From 20cca35157fd51c06e9606e6456add518de74601 Mon Sep 17 00:00:00 2001 From: Sisyphus Date: Sun, 25 Jan 2026 12:34:42 +0900 Subject: [PATCH] fix(ralph-loop): skip user messages in transcript completion detection (#622) (#1086) * fix(ralph-loop): skip user messages in transcript completion detection (#622) The transcript-based completion detection was searching the entire JSONL file for DONE, including user message entries. The RALPH_LOOP_TEMPLATE instructional text contains this literal pattern, which gets recorded as a user message, causing false positive completion detection on every iteration. This made the loop always terminate at iteration 1. Fix: Parse JSONL entries line-by-line and skip entries with type 'user' so only tool_result/assistant entries are checked for the completion promise. Also remove the hardcoded DONE from the template exit conditions as defense-in-depth. * chore: changes by sisyphus-dev-ai --------- Co-authored-by: sisyphus-dev-ai --- .../builtin-commands/templates/ralph-loop.ts | 2 +- src/hooks/ralph-loop/index.test.ts | 103 +++++++++++++++++- src/hooks/ralph-loop/index.ts | 13 ++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/features/builtin-commands/templates/ralph-loop.ts b/src/features/builtin-commands/templates/ralph-loop.ts index 65846393..de4b8ca0 100644 --- a/src/features/builtin-commands/templates/ralph-loop.ts +++ b/src/features/builtin-commands/templates/ralph-loop.ts @@ -17,7 +17,7 @@ export const RALPH_LOOP_TEMPLATE = `You are starting a Ralph Loop - a self-refer ## Exit Conditions -1. **Completion**: Output \`DONE\` (or custom promise text) when fully complete +1. **Completion**: Output your completion promise tag when fully complete 2. **Max Iterations**: Loop stops automatically at limit 3. **Cancel**: User runs \`/cancel-ralph\` command diff --git a/src/hooks/ralph-loop/index.test.ts b/src/hooks/ralph-loop/index.test.ts index 3a6a77ed..bf02831b 100644 --- a/src/hooks/ralph-loop/index.test.ts +++ b/src/hooks/ralph-loop/index.test.ts @@ -459,7 +459,7 @@ describe("ralph-loop", () => { }) hook.startLoop("session-123", "Build something", { completionPromise: "COMPLETE" }) - writeFileSync(transcriptPath, JSON.stringify({ content: "Task done COMPLETE" })) + writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "Task done COMPLETE" } }) + "\n") // #when - session goes idle (transcriptPath now derived from sessionID via getTranscriptPath) await hook.event({ @@ -703,10 +703,105 @@ describe("ralph-loop", () => { expect(promptCalls[0].text).toContain("2/50") }) + test("should NOT detect completion from user message in transcript (issue #622)", async () => { + // #given - transcript contains user message with template text that includes completion promise + // This reproduces the bug where the RALPH_LOOP_TEMPLATE instructional text + // containing `DONE` is recorded as a user message and + // falsely triggers completion detection + const transcriptPath = join(TEST_DIR, "transcript.jsonl") + const templateText = `You are starting a Ralph Loop... +Output DONE when fully complete` + const userEntry = JSON.stringify({ + type: "user", + timestamp: new Date().toISOString(), + content: templateText, + }) + writeFileSync(transcriptPath, userEntry + "\n") + + const hook = createRalphLoopHook(createMockPluginInput(), { + getTranscriptPath: () => transcriptPath, + }) + hook.startLoop("session-123", "Build something", { completionPromise: "DONE" }) + + // #when - session goes idle + await hook.event({ + event: { + type: "session.idle", + properties: { sessionID: "session-123" }, + }, + }) + + // #then - loop should CONTINUE (user message completion promise is instructional, not actual) + expect(promptCalls.length).toBe(1) + expect(hook.getState()?.iteration).toBe(2) + }) + + test("should NOT detect completion from continuation prompt in transcript (issue #622)", async () => { + // #given - transcript contains continuation prompt (also a user message) with completion promise + const transcriptPath = join(TEST_DIR, "transcript.jsonl") + const continuationText = `RALPH LOOP 2/100 +When FULLY complete, output: DONE +Original task: Build something` + const userEntry = JSON.stringify({ + type: "user", + timestamp: new Date().toISOString(), + content: continuationText, + }) + writeFileSync(transcriptPath, userEntry + "\n") + + const hook = createRalphLoopHook(createMockPluginInput(), { + getTranscriptPath: () => transcriptPath, + }) + hook.startLoop("session-123", "Build something", { completionPromise: "DONE" }) + + // #when - session goes idle + await hook.event({ + event: { + type: "session.idle", + properties: { sessionID: "session-123" }, + }, + }) + + // #then - loop should CONTINUE (continuation prompt text is not actual completion) + expect(promptCalls.length).toBe(1) + expect(hook.getState()?.iteration).toBe(2) + }) + + test("should detect completion from tool_result entry in transcript", async () => { + // #given - transcript contains a tool_result with completion promise + const transcriptPath = join(TEST_DIR, "transcript.jsonl") + const toolResultEntry = JSON.stringify({ + type: "tool_result", + timestamp: new Date().toISOString(), + tool_name: "write", + tool_input: {}, + tool_output: { output: "Task complete! DONE" }, + }) + writeFileSync(transcriptPath, toolResultEntry + "\n") + + const hook = createRalphLoopHook(createMockPluginInput(), { + getTranscriptPath: () => transcriptPath, + }) + hook.startLoop("session-123", "Build something", { completionPromise: "DONE" }) + + // #when - session goes idle + await hook.event({ + event: { + type: "session.idle", + properties: { sessionID: "session-123" }, + }, + }) + + // #then - loop should complete (tool_result contains actual completion output) + expect(promptCalls.length).toBe(0) + expect(toastCalls.some((t) => t.title === "Ralph Loop Complete!")).toBe(true) + expect(hook.getState()).toBeNull() + }) + test("should check transcript BEFORE API to optimize performance", async () => { // #given - transcript has completion promise const transcriptPath = join(TEST_DIR, "transcript.jsonl") - writeFileSync(transcriptPath, JSON.stringify({ content: "DONE" })) + writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "DONE" } }) + "\n") mockSessionMessages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "No promise here" }] }, ] @@ -736,7 +831,7 @@ describe("ralph-loop", () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: () => transcriptPath, }) - writeFileSync(transcriptPath, JSON.stringify({ content: "DONE" })) + writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "DONE" } }) + "\n") hook.startLoop("test-id", "Build API", { ultrawork: true }) // #when - idle event triggered @@ -754,7 +849,7 @@ describe("ralph-loop", () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: () => transcriptPath, }) - writeFileSync(transcriptPath, JSON.stringify({ content: "DONE" })) + writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "DONE" } }) + "\n") hook.startLoop("test-id", "Build API") // #when - idle event triggered diff --git a/src/hooks/ralph-loop/index.ts b/src/hooks/ralph-loop/index.ts index 9f27f201..c961fe66 100644 --- a/src/hooks/ralph-loop/index.ts +++ b/src/hooks/ralph-loop/index.ts @@ -100,7 +100,18 @@ export function createRalphLoopHook( const content = readFileSync(transcriptPath, "utf-8") const pattern = new RegExp(`\\s*${escapeRegex(promise)}\\s*`, "is") - return pattern.test(content) + const lines = content.split("\n").filter(l => l.trim()) + + for (const line of lines) { + try { + const entry = JSON.parse(line) + if (entry.type === "user") continue + if (pattern.test(line)) return true + } catch { + continue + } + } + return false } catch { return false }