feat: add pre/post verification to task execution pipeline

- Before executing: verify if work is already done on server
- If already done: skip execution, comment evidence, mark complete
- After executing: run verification again to confirm work was applied
- Verification agent uses Pi with read-only investigation prompt
- Adds verification status (PASSED/COULD NOT CONFIRM) to Asana comments
- Prevents redundant work and catches false completions
This commit is contained in:
Azreen Jamal
2026-03-06 15:57:39 +08:00
parent ccdbf94450
commit 086d9a3026
2 changed files with 93 additions and 14 deletions
+9 -8
View File
@@ -39,15 +39,16 @@ export class PiExecutor {
}); });
try { try {
// Don't pass ANTHROPIC_API_KEY — Pi uses OAuth from ~/.pi/agent/auth.json
// Passing the API key overrides OAuth and may hit a depleted prepaid balance
const { ANTHROPIC_API_KEY: _removed, ...cleanEnv } = process.env;
const proc = spawn({ const proc = spawn({
cmd: args, cmd: args,
cwd: this.config.targetProjectPath, cwd: this.config.targetProjectPath,
stdout: "pipe", stdout: "pipe",
stderr: "pipe", stderr: "pipe",
env: { env: cleanEnv,
...process.env,
ANTHROPIC_API_KEY: this.config.anthropicApiKey,
},
}); });
this.runningProcess = proc; this.runningProcess = proc;
@@ -172,15 +173,15 @@ export class PiExecutor {
logger.info("pi-executor", `Executing with agent-team: ${task.substring(0, 100)}...`); logger.info("pi-executor", `Executing with agent-team: ${task.substring(0, 100)}...`);
try { try {
// Don't pass ANTHROPIC_API_KEY — Pi uses OAuth from ~/.pi/agent/auth.json
const { ANTHROPIC_API_KEY: _removed, ...cleanEnv } = process.env;
const proc = spawn({ const proc = spawn({
cmd: args, cmd: args,
cwd: this.config.targetProjectPath, cwd: this.config.targetProjectPath,
stdout: "pipe", stdout: "pipe",
stderr: "pipe", stderr: "pipe",
env: { env: cleanEnv,
...process.env,
ANTHROPIC_API_KEY: this.config.anthropicApiKey,
},
}); });
this.runningProcess = proc; this.runningProcess = proc;
+84 -6
View File
@@ -129,25 +129,62 @@ export class TaskLoop {
private async executeTask(task: AsanaTask) { private async executeTask(task: AsanaTask) {
try { try {
// Move to "In Progress" // STEP 1: Check if the work is already done
const alreadyDone = await this.verifyTaskAlreadyDone(task);
if (alreadyDone.done) {
logger.info("task-loop", `Task already done: ${task.name}`, { evidence: alreadyDone.evidence });
await this.asana.addComment(
task.gid,
`✅ Pi Worker verified this task is **already completed** on the server.\n\nEvidence:\n${alreadyDone.evidence}`
);
// Move to Done and mark complete
const doneSection = await this.findFirstSection(SECTIONS.DONE);
if (doneSection) {
await this.asana.moveTaskToSection(task.gid, doneSection.gid);
}
await this.asana.updateTask(task.gid, { completed: true });
logger.info("task-loop", `Task marked done (was already complete): ${task.name}`);
return;
}
// STEP 2: Move to "In Progress"
const inProgressSection = await this.findFirstSection(SECTIONS.IN_PROGRESS); const inProgressSection = await this.findFirstSection(SECTIONS.IN_PROGRESS);
if (inProgressSection) { if (inProgressSection) {
await this.asana.moveTaskToSection(task.gid, inProgressSection.gid); await this.asana.moveTaskToSection(task.gid, inProgressSection.gid);
} }
// Add starting comment // Add starting comment
await this.asana.addComment(task.gid, `🤖 Pi Worker picked up this task.\nStarting execution at ${new Date().toISOString()}`); await this.asana.addComment(task.gid, `🤖 Pi Worker picked up this task.\nVerification: Not yet done on server.\nStarting execution at ${new Date().toISOString()}`);
// Build context from task // Build context from task
const context = this.buildTaskContext(task); const context = this.buildTaskContext(task);
// Execute with Pi agent // STEP 3: Execute with Pi agent
const result = await this.pi.execute(task.name + (task.notes ? `\n\nDetails:\n${task.notes}` : ""), context); const result = await this.pi.execute(task.name + (task.notes ? `\n\nDetails:\n${task.notes}` : ""), context);
// STEP 4: After execution, verify the work was actually done
let verified = false;
if (result.success && result.toolCalls > 0) {
const postCheck = await this.verifyTaskAlreadyDone(task);
verified = postCheck.done;
if (!verified) {
logger.warn("task-loop", `Task executed but verification failed: ${task.name}`);
}
}
// Add completion comment // Add completion comment
const statusEmoji = result.success ? "✅" : "❌"; const statusEmoji = result.success ? (verified ? "✅" : "⚠️") : "❌";
const verificationNote = result.success
? (verified ? "Post-execution verification: PASSED" : "Post-execution verification: COULD NOT CONFIRM — please review manually")
: "";
const comment = [ const comment = [
`${statusEmoji} Pi Worker ${result.success ? "completed" : "failed"} this task.`, `${statusEmoji} Pi Worker ${result.success ? "completed" : "failed"} this task.`,
verificationNote,
`Duration: ${(result.durationMs / 1000).toFixed(1)}s`, `Duration: ${(result.durationMs / 1000).toFixed(1)}s`,
`Tool calls: ${result.toolCalls}`, `Tool calls: ${result.toolCalls}`,
result.error ? `Error: ${result.error}` : "", result.error ? `Error: ${result.error}` : "",
@@ -161,13 +198,12 @@ export class TaskLoop {
await this.asana.addComment(task.gid, comment); await this.asana.addComment(task.gid, comment);
if (result.success) { if (result.success) {
// Move to "Done" and mark complete
const doneSection = await this.findFirstSection(SECTIONS.DONE); const doneSection = await this.findFirstSection(SECTIONS.DONE);
if (doneSection) { if (doneSection) {
await this.asana.moveTaskToSection(task.gid, doneSection.gid); await this.asana.moveTaskToSection(task.gid, doneSection.gid);
} }
await this.asana.updateTask(task.gid, { completed: true }); await this.asana.updateTask(task.gid, { completed: true });
logger.info("task-loop", `Task completed: ${task.name}`); logger.info("task-loop", `Task completed: ${task.name}`, { verified });
} else { } else {
// Move back to To Do on failure // Move back to To Do on failure
const todoSection = await this.findFirstSection(SECTIONS.TODO); const todoSection = await this.findFirstSection(SECTIONS.TODO);
@@ -184,6 +220,48 @@ export class TaskLoop {
} }
} }
private async verifyTaskAlreadyDone(task: AsanaTask): Promise<{ done: boolean; evidence: string }> {
try {
const verifyPrompt = `You are a verification agent. Your ONLY job is to check if the following task/bug/feature is ALREADY done on this server/codebase. Do NOT fix anything. Do NOT make any changes. ONLY investigate and report.
Task: ${task.name}
${task.notes ? `Details: ${task.notes}` : ""}
INSTRUCTIONS:
1. Use your tools (bash, read, grep, find, ls) to investigate if this work is already completed
2. Check the codebase, running services, server state, logs — whatever is relevant
3. Be thorough but quick
RESPOND WITH EXACTLY THIS FORMAT (nothing else):
DONE: true
EVIDENCE: <one paragraph explaining what you found that proves it's done>
OR:
DONE: false
EVIDENCE: <one paragraph explaining what's missing or broken>`;
const result = await this.pi.execute(verifyPrompt);
if (!result.success) {
return { done: false, evidence: "Verification check failed to run" };
}
const output = result.output.trim();
const doneMatch = output.match(/DONE:\s*(true|false)/i);
const evidenceMatch = output.match(/EVIDENCE:\s*(.+)/is);
const isDone = doneMatch ? doneMatch[1].toLowerCase() === "true" : false;
const evidence = evidenceMatch ? evidenceMatch[1].trim().substring(0, 1000) : output.substring(0, 1000);
logger.info("task-loop", `Verification result for "${task.name}": done=${isDone}`, { evidence: evidence.substring(0, 200) });
return { done: isDone, evidence };
} catch (error: any) {
logger.error("task-loop", `Verification error: ${error.message}`);
return { done: false, evidence: `Verification error: ${error.message}` };
}
}
private buildTaskContext(task: AsanaTask): string { private buildTaskContext(task: AsanaTask): string {
const parts = [ const parts = [
`Asana Task: ${task.name}`, `Asana Task: ${task.name}`,