feat(benchmarks): add hashline-edit benchmark agent and deps
Standalone headless agent using Vercel AI SDK v6 with FriendliAI provider. Imports hashline-edit pure functions directly from src/ for benchmarking the edit tool against LLMs (Minimax M2.5 via FriendliAI). Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
minpeter
parent
b1203b9501
commit
d1a0a66dde
62
benchmarks/bun.lock
Normal file
62
benchmarks/bun.lock
Normal file
@ -0,0 +1,62 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "hashline-edit-benchmark",
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^1.3.0",
|
||||
"@friendliai/ai-provider": "^1.0.9",
|
||||
"ai": "^6.0.94",
|
||||
"zod": "^4.1.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.55", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15", "@vercel/oidc": "3.1.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7xMeTJnCjwRwXKVCiv4Ly4qzWvDuW3+W1WIV0X1EFu6W83d4mEhV9bFArto10MeTw40ewuDjrbrZd21mXKohkw=="],
|
||||
|
||||
"@ai-sdk/openai": ["@ai-sdk/openai@1.3.24", "", { "dependencies": { "@ai-sdk/provider": "1.1.3", "@ai-sdk/provider-utils": "2.2.8" }, "peerDependencies": { "zod": "^3.0.0" } }, "sha512-GYXnGJTHRTZc4gJMSmFRgEQudjqd4PUN0ZjQhPwOAYH1yOAvQoG/Ikqs+HyISRbLPCrhbZnPKCNHuRU4OfpW0Q=="],
|
||||
|
||||
"@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.30", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-iTjumHf1/u4NhjXYFn/aONM2GId3/o7J1Lp5ql8FCbgIMyRwrmanR5xy1S3aaVkfTscuDvLTzWiy1mAbGzK3nQ=="],
|
||||
|
||||
"@ai-sdk/provider": ["@ai-sdk/provider@1.1.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg=="],
|
||||
|
||||
"@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@2.2.8", "", { "dependencies": { "@ai-sdk/provider": "1.1.3", "nanoid": "^3.3.8", "secure-json-parse": "^2.7.0" }, "peerDependencies": { "zod": "^3.23.8" } }, "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA=="],
|
||||
|
||||
"@friendliai/ai-provider": ["@friendliai/ai-provider@1.1.4", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.30", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.12" } }, "sha512-9TU4B1QFqPhbkONjI5afCF7Ox4jOqtGg1xw8mA9QHZdtlEbZxU+mBNvMPlI5pU5kPoN6s7wkXmFmxpID+own1A=="],
|
||||
|
||||
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
|
||||
|
||||
"@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
|
||||
|
||||
"@vercel/oidc": ["@vercel/oidc@3.1.0", "", {}, "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w=="],
|
||||
|
||||
"ai": ["ai@6.0.101", "", { "dependencies": { "@ai-sdk/gateway": "3.0.55", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15", "@opentelemetry/api": "1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Ur/NgbgOp1rdhyDiKDk6EOpSgd1g5ADlbcD1cjQJtQsnmhEngz3Rf8nK5JetDh0vnbLy2aEBpaQeL+zvLRWuaA=="],
|
||||
|
||||
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],
|
||||
|
||||
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||
|
||||
"secure-json-parse": ["secure-json-parse@2.7.0", "", {}, "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="],
|
||||
|
||||
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
|
||||
|
||||
"@ai-sdk/gateway/@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="],
|
||||
|
||||
"@ai-sdk/gateway/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
|
||||
|
||||
"@ai-sdk/openai-compatible/@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="],
|
||||
|
||||
"@ai-sdk/openai-compatible/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
|
||||
|
||||
"@friendliai/ai-provider/@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="],
|
||||
|
||||
"@friendliai/ai-provider/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
|
||||
|
||||
"ai/@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="],
|
||||
|
||||
"ai/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
|
||||
}
|
||||
}
|
||||
190
benchmarks/headless.ts
Normal file
190
benchmarks/headless.ts
Normal file
@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env bun
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises"
|
||||
import { join, dirname } from "node:path"
|
||||
import { stepCountIs, streamText, type CoreMessage } from "ai"
|
||||
import { tool } from "ai"
|
||||
import { createFriendli } from "@friendliai/ai-provider"
|
||||
import { z } from "zod"
|
||||
import { formatHashLines } from "../src/tools/hashline-edit/hash-computation"
|
||||
import { normalizeHashlineEdits } from "../src/tools/hashline-edit/normalize-edits"
|
||||
import { applyHashlineEditsWithReport } from "../src/tools/hashline-edit/edit-operations"
|
||||
import { canonicalizeFileText, restoreFileText } from "../src/tools/hashline-edit/file-text-canonicalization"
|
||||
|
||||
const DEFAULT_MODEL = "MiniMaxAI/MiniMax-M2.5"
|
||||
const MAX_STEPS = 50
|
||||
const sessionId = `bench-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
|
||||
const emit = (event: Record<string, unknown>) =>
|
||||
console.log(JSON.stringify({ sessionId, timestamp: new Date().toISOString(), ...event }))
|
||||
|
||||
// ── CLI ──────────────────────────────────────────────────────
|
||||
function parseArgs(): { prompt: string; modelId: string } {
|
||||
const args = process.argv.slice(2)
|
||||
let prompt = ""
|
||||
let modelId = DEFAULT_MODEL
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if ((args[i] === "-p" || args[i] === "--prompt") && args[i + 1]) {
|
||||
prompt = args[++i]
|
||||
} else if ((args[i] === "-m" || args[i] === "--model") && args[i + 1]) {
|
||||
modelId = args[++i]
|
||||
} else if (args[i] === "--reasoning-mode" && args[i + 1]) {
|
||||
i++ // consume
|
||||
}
|
||||
// --no-translate, --think consumed silently
|
||||
}
|
||||
if (!prompt) {
|
||||
console.error("Usage: bun run benchmarks/headless.ts -p <prompt> [-m <model>]")
|
||||
process.exit(1)
|
||||
}
|
||||
return { prompt, modelId }
|
||||
}
|
||||
|
||||
// ── Tools ────────────────────────────────────────────────────
|
||||
const readFileTool = tool({
|
||||
description: "Read a file with hashline-tagged content (LINE#ID format)",
|
||||
inputSchema: z.object({ path: z.string().describe("File path") }),
|
||||
execute: async ({ path }) => {
|
||||
const fullPath = join(process.cwd(), path)
|
||||
try {
|
||||
const content = await readFile(fullPath, "utf-8")
|
||||
const lines = content.split("\n")
|
||||
const tagged = formatHashLines(content)
|
||||
return `OK - read file\npath: ${path}\nlines: ${lines.length}\n\n${tagged}`
|
||||
} catch {
|
||||
return `Error: File not found: ${path}`
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const editFileTool = tool({
|
||||
description: "Edit a file using hashline anchors (LINE#ID format)",
|
||||
inputSchema: z.object({
|
||||
path: z.string(),
|
||||
edits: z.array(
|
||||
z.object({
|
||||
op: z.enum(["replace", "append", "prepend"]),
|
||||
pos: z.string().optional(),
|
||||
end: z.string().optional(),
|
||||
lines: z.union([z.array(z.string()), z.string(), z.null()]),
|
||||
})
|
||||
).min(1),
|
||||
}),
|
||||
execute: async ({ path, edits }) => {
|
||||
const fullPath = join(process.cwd(), path)
|
||||
try {
|
||||
let rawContent = ""
|
||||
let exists = true
|
||||
try {
|
||||
rawContent = await readFile(fullPath, "utf-8")
|
||||
} catch {
|
||||
exists = false
|
||||
}
|
||||
|
||||
const normalized = normalizeHashlineEdits(edits)
|
||||
|
||||
if (!exists) {
|
||||
const canCreate = normalized.every(
|
||||
(e) => (e.op === "append" || e.op === "prepend") && !e.pos
|
||||
)
|
||||
if (!canCreate) return `Error: File not found: ${path}`
|
||||
}
|
||||
|
||||
const envelope = canonicalizeFileText(rawContent)
|
||||
const result = applyHashlineEditsWithReport(envelope.content, normalized)
|
||||
|
||||
if (result.content === envelope.content) {
|
||||
return `Error: No changes made to ${path}. The edits produced identical content.`
|
||||
}
|
||||
|
||||
const writeContent = restoreFileText(result.content, envelope)
|
||||
await mkdir(dirname(fullPath), { recursive: true })
|
||||
await writeFile(fullPath, writeContent, "utf-8")
|
||||
|
||||
const oldLineCount = rawContent.split("\n").length
|
||||
const newLineCount = writeContent.split("\n").length
|
||||
const delta = newLineCount - oldLineCount
|
||||
const sign = delta > 0 ? "+" : ""
|
||||
const action = exists ? "Updated" : "Created"
|
||||
return `${action} ${path}\n${edits.length} edit(s) applied, ${sign}${delta} line(s)`
|
||||
} catch (error) {
|
||||
return `Error: ${error instanceof Error ? error.message : String(error)}`
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
// ── Agent Loop ───────────────────────────────────────────────
|
||||
async function run() {
|
||||
const { prompt, modelId } = parseArgs()
|
||||
|
||||
const friendli = createFriendli({ apiKey: process.env.FRIENDLI_TOKEN! })
|
||||
const model = friendli(modelId)
|
||||
const tools = { read_file: readFileTool, edit_file: editFileTool }
|
||||
|
||||
emit({ type: "user", content: prompt })
|
||||
|
||||
const messages: CoreMessage[] = [{ role: "user", content: prompt }]
|
||||
const system =
|
||||
"You are a code editing assistant. Use read_file to read files and edit_file to edit them. " +
|
||||
"Always read a file before editing it to get fresh LINE#ID anchors."
|
||||
|
||||
for (let step = 0; step < MAX_STEPS; step++) {
|
||||
const stream = streamText({
|
||||
model,
|
||||
tools,
|
||||
messages,
|
||||
system,
|
||||
stopWhen: stepCountIs(1),
|
||||
})
|
||||
|
||||
let currentText = ""
|
||||
for await (const part of stream.fullStream) {
|
||||
switch (part.type) {
|
||||
case "text-delta":
|
||||
currentText += part.text
|
||||
break
|
||||
case "tool-call":
|
||||
emit({
|
||||
type: "tool_call",
|
||||
tool_call_id: part.toolCallId,
|
||||
tool_name: part.toolName,
|
||||
tool_input: part.args,
|
||||
model: modelId,
|
||||
})
|
||||
break
|
||||
case "tool-result":
|
||||
emit({
|
||||
type: "tool_result",
|
||||
tool_call_id: part.toolCallId,
|
||||
output: typeof part.result === "string" ? part.result : JSON.stringify(part.result),
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const response = await stream.response
|
||||
messages.push(...response.messages)
|
||||
|
||||
const finishReason = await stream.finishReason
|
||||
if (finishReason !== "tool-calls") {
|
||||
if (currentText.trim()) {
|
||||
emit({ type: "assistant", content: currentText, model: modelId })
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Signal + Startup ─────────────────────────────────────────
|
||||
process.once("SIGINT", () => process.exit(0))
|
||||
process.once("SIGTERM", () => process.exit(143))
|
||||
|
||||
const startTime = Date.now()
|
||||
run()
|
||||
.catch((error) => {
|
||||
emit({ type: "error", error: error instanceof Error ? error.message : String(error) })
|
||||
process.exit(1)
|
||||
})
|
||||
.then(() => {
|
||||
const elapsed = ((Date.now() - startTime) / 1000).toFixed(2)
|
||||
console.error(`[headless] Completed in ${elapsed}s`)
|
||||
})
|
||||
19
benchmarks/package.json
Normal file
19
benchmarks/package.json
Normal file
@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "hashline-edit-benchmark",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Hashline edit tool benchmark using Vercel AI SDK with FriendliAI provider",
|
||||
"scripts": {
|
||||
"bench:basic": "bun run test-edit-ops.ts",
|
||||
"bench:edge": "bun run test-edge-cases.ts",
|
||||
"bench:multi": "bun run test-multi-model.ts",
|
||||
"bench:all": "bun run bench:basic && bun run bench:edge"
|
||||
},
|
||||
"dependencies": {
|
||||
"ai": "^6.0.94",
|
||||
"@ai-sdk/openai": "^1.3.0",
|
||||
"@friendliai/ai-provider": "^1.0.9",
|
||||
"zod": "^4.1.0"
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user