From 086d9a302668ed7942e1452a7156aca1bae0dde0 Mon Sep 17 00:00:00 2001 From: Azreen Jamal Date: Fri, 6 Mar 2026 15:57:39 +0800 Subject: [PATCH] feat: add pre/post verification to task execution pipeline - Before executing: verify if work is already done on server - If already done: skip execution, comment evidence, mark complete - After executing: run verification again to confirm work was applied - Verification agent uses Pi with read-only investigation prompt - Adds verification status (PASSED/COULD NOT CONFIRM) to Asana comments - Prevents redundant work and catches false completions --- pi-worker/src/pi/executor.ts | 17 +++--- pi-worker/src/scheduler/task-loop.ts | 90 ++++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/pi-worker/src/pi/executor.ts b/pi-worker/src/pi/executor.ts index f65f1df..400aacc 100644 --- a/pi-worker/src/pi/executor.ts +++ b/pi-worker/src/pi/executor.ts @@ -39,15 +39,16 @@ export class PiExecutor { }); try { + // Don't pass ANTHROPIC_API_KEY — Pi uses OAuth from ~/.pi/agent/auth.json + // Passing the API key overrides OAuth and may hit a depleted prepaid balance + const { ANTHROPIC_API_KEY: _removed, ...cleanEnv } = process.env; + const proc = spawn({ cmd: args, cwd: this.config.targetProjectPath, stdout: "pipe", stderr: "pipe", - env: { - ...process.env, - ANTHROPIC_API_KEY: this.config.anthropicApiKey, - }, + env: cleanEnv, }); this.runningProcess = proc; @@ -172,15 +173,15 @@ export class PiExecutor { logger.info("pi-executor", `Executing with agent-team: ${task.substring(0, 100)}...`); try { + // Don't pass ANTHROPIC_API_KEY — Pi uses OAuth from ~/.pi/agent/auth.json + const { ANTHROPIC_API_KEY: _removed, ...cleanEnv } = process.env; + const proc = spawn({ cmd: args, cwd: this.config.targetProjectPath, stdout: "pipe", stderr: "pipe", - env: { - ...process.env, - ANTHROPIC_API_KEY: this.config.anthropicApiKey, - }, + env: cleanEnv, }); this.runningProcess = proc; diff --git a/pi-worker/src/scheduler/task-loop.ts b/pi-worker/src/scheduler/task-loop.ts index 6396112..b869235 100644 --- a/pi-worker/src/scheduler/task-loop.ts +++ b/pi-worker/src/scheduler/task-loop.ts @@ -129,25 +129,62 @@ export class TaskLoop { private async executeTask(task: AsanaTask) { try { - // Move to "In Progress" + // STEP 1: Check if the work is already done + const alreadyDone = await this.verifyTaskAlreadyDone(task); + + if (alreadyDone.done) { + logger.info("task-loop", `Task already done: ${task.name}`, { evidence: alreadyDone.evidence }); + + await this.asana.addComment( + task.gid, + `✅ Pi Worker verified this task is **already completed** on the server.\n\nEvidence:\n${alreadyDone.evidence}` + ); + + // Move to Done and mark complete + const doneSection = await this.findFirstSection(SECTIONS.DONE); + if (doneSection) { + await this.asana.moveTaskToSection(task.gid, doneSection.gid); + } + await this.asana.updateTask(task.gid, { completed: true }); + logger.info("task-loop", `Task marked done (was already complete): ${task.name}`); + return; + } + + // STEP 2: Move to "In Progress" const inProgressSection = await this.findFirstSection(SECTIONS.IN_PROGRESS); if (inProgressSection) { await this.asana.moveTaskToSection(task.gid, inProgressSection.gid); } // Add starting comment - await this.asana.addComment(task.gid, `🤖 Pi Worker picked up this task.\nStarting execution at ${new Date().toISOString()}`); + await this.asana.addComment(task.gid, `🤖 Pi Worker picked up this task.\nVerification: Not yet done on server.\nStarting execution at ${new Date().toISOString()}`); // Build context from task const context = this.buildTaskContext(task); - // Execute with Pi agent + // STEP 3: Execute with Pi agent const result = await this.pi.execute(task.name + (task.notes ? `\n\nDetails:\n${task.notes}` : ""), context); + // STEP 4: After execution, verify the work was actually done + let verified = false; + if (result.success && result.toolCalls > 0) { + const postCheck = await this.verifyTaskAlreadyDone(task); + verified = postCheck.done; + + if (!verified) { + logger.warn("task-loop", `Task executed but verification failed: ${task.name}`); + } + } + // Add completion comment - const statusEmoji = result.success ? "✅" : "❌"; + const statusEmoji = result.success ? (verified ? "✅" : "⚠️") : "❌"; + const verificationNote = result.success + ? (verified ? "Post-execution verification: PASSED" : "Post-execution verification: COULD NOT CONFIRM — please review manually") + : ""; + const comment = [ `${statusEmoji} Pi Worker ${result.success ? "completed" : "failed"} this task.`, + verificationNote, `Duration: ${(result.durationMs / 1000).toFixed(1)}s`, `Tool calls: ${result.toolCalls}`, result.error ? `Error: ${result.error}` : "", @@ -161,13 +198,12 @@ export class TaskLoop { await this.asana.addComment(task.gid, comment); if (result.success) { - // Move to "Done" and mark complete const doneSection = await this.findFirstSection(SECTIONS.DONE); if (doneSection) { await this.asana.moveTaskToSection(task.gid, doneSection.gid); } await this.asana.updateTask(task.gid, { completed: true }); - logger.info("task-loop", `Task completed: ${task.name}`); + logger.info("task-loop", `Task completed: ${task.name}`, { verified }); } else { // Move back to To Do on failure const todoSection = await this.findFirstSection(SECTIONS.TODO); @@ -184,6 +220,48 @@ export class TaskLoop { } } + private async verifyTaskAlreadyDone(task: AsanaTask): Promise<{ done: boolean; evidence: string }> { + try { + const verifyPrompt = `You are a verification agent. Your ONLY job is to check if the following task/bug/feature is ALREADY done on this server/codebase. Do NOT fix anything. Do NOT make any changes. ONLY investigate and report. + +Task: ${task.name} +${task.notes ? `Details: ${task.notes}` : ""} + +INSTRUCTIONS: +1. Use your tools (bash, read, grep, find, ls) to investigate if this work is already completed +2. Check the codebase, running services, server state, logs — whatever is relevant +3. Be thorough but quick + +RESPOND WITH EXACTLY THIS FORMAT (nothing else): +DONE: true +EVIDENCE: + +OR: +DONE: false +EVIDENCE: `; + + const result = await this.pi.execute(verifyPrompt); + + if (!result.success) { + return { done: false, evidence: "Verification check failed to run" }; + } + + const output = result.output.trim(); + const doneMatch = output.match(/DONE:\s*(true|false)/i); + const evidenceMatch = output.match(/EVIDENCE:\s*(.+)/is); + + const isDone = doneMatch ? doneMatch[1].toLowerCase() === "true" : false; + const evidence = evidenceMatch ? evidenceMatch[1].trim().substring(0, 1000) : output.substring(0, 1000); + + logger.info("task-loop", `Verification result for "${task.name}": done=${isDone}`, { evidence: evidence.substring(0, 200) }); + + return { done: isDone, evidence }; + } catch (error: any) { + logger.error("task-loop", `Verification error: ${error.message}`); + return { done: false, evidence: `Verification error: ${error.message}` }; + } + } + private buildTaskContext(task: AsanaTask): string { const parts = [ `Asana Task: ${task.name}`,