feat: add pre/post verification to task execution pipeline
- Before executing: verify if work is already done on server - If already done: skip execution, comment evidence, mark complete - After executing: run verification again to confirm work was applied - Verification agent uses Pi with read-only investigation prompt - Adds verification status (PASSED/COULD NOT CONFIRM) to Asana comments - Prevents redundant work and catches false completions
This commit is contained in:
@@ -39,15 +39,16 @@ export class PiExecutor {
|
||||
});
|
||||
|
||||
try {
|
||||
// Don't pass ANTHROPIC_API_KEY — Pi uses OAuth from ~/.pi/agent/auth.json
|
||||
// Passing the API key overrides OAuth and may hit a depleted prepaid balance
|
||||
const { ANTHROPIC_API_KEY: _removed, ...cleanEnv } = process.env;
|
||||
|
||||
const proc = spawn({
|
||||
cmd: args,
|
||||
cwd: this.config.targetProjectPath,
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
env: {
|
||||
...process.env,
|
||||
ANTHROPIC_API_KEY: this.config.anthropicApiKey,
|
||||
},
|
||||
env: cleanEnv,
|
||||
});
|
||||
|
||||
this.runningProcess = proc;
|
||||
@@ -172,15 +173,15 @@ export class PiExecutor {
|
||||
logger.info("pi-executor", `Executing with agent-team: ${task.substring(0, 100)}...`);
|
||||
|
||||
try {
|
||||
// Don't pass ANTHROPIC_API_KEY — Pi uses OAuth from ~/.pi/agent/auth.json
|
||||
const { ANTHROPIC_API_KEY: _removed, ...cleanEnv } = process.env;
|
||||
|
||||
const proc = spawn({
|
||||
cmd: args,
|
||||
cwd: this.config.targetProjectPath,
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
env: {
|
||||
...process.env,
|
||||
ANTHROPIC_API_KEY: this.config.anthropicApiKey,
|
||||
},
|
||||
env: cleanEnv,
|
||||
});
|
||||
|
||||
this.runningProcess = proc;
|
||||
|
||||
@@ -129,25 +129,62 @@ export class TaskLoop {
|
||||
|
||||
private async executeTask(task: AsanaTask) {
|
||||
try {
|
||||
// Move to "In Progress"
|
||||
// STEP 1: Check if the work is already done
|
||||
const alreadyDone = await this.verifyTaskAlreadyDone(task);
|
||||
|
||||
if (alreadyDone.done) {
|
||||
logger.info("task-loop", `Task already done: ${task.name}`, { evidence: alreadyDone.evidence });
|
||||
|
||||
await this.asana.addComment(
|
||||
task.gid,
|
||||
`✅ Pi Worker verified this task is **already completed** on the server.\n\nEvidence:\n${alreadyDone.evidence}`
|
||||
);
|
||||
|
||||
// Move to Done and mark complete
|
||||
const doneSection = await this.findFirstSection(SECTIONS.DONE);
|
||||
if (doneSection) {
|
||||
await this.asana.moveTaskToSection(task.gid, doneSection.gid);
|
||||
}
|
||||
await this.asana.updateTask(task.gid, { completed: true });
|
||||
logger.info("task-loop", `Task marked done (was already complete): ${task.name}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// STEP 2: Move to "In Progress"
|
||||
const inProgressSection = await this.findFirstSection(SECTIONS.IN_PROGRESS);
|
||||
if (inProgressSection) {
|
||||
await this.asana.moveTaskToSection(task.gid, inProgressSection.gid);
|
||||
}
|
||||
|
||||
// Add starting comment
|
||||
await this.asana.addComment(task.gid, `🤖 Pi Worker picked up this task.\nStarting execution at ${new Date().toISOString()}`);
|
||||
await this.asana.addComment(task.gid, `🤖 Pi Worker picked up this task.\nVerification: Not yet done on server.\nStarting execution at ${new Date().toISOString()}`);
|
||||
|
||||
// Build context from task
|
||||
const context = this.buildTaskContext(task);
|
||||
|
||||
// Execute with Pi agent
|
||||
// STEP 3: Execute with Pi agent
|
||||
const result = await this.pi.execute(task.name + (task.notes ? `\n\nDetails:\n${task.notes}` : ""), context);
|
||||
|
||||
// STEP 4: After execution, verify the work was actually done
|
||||
let verified = false;
|
||||
if (result.success && result.toolCalls > 0) {
|
||||
const postCheck = await this.verifyTaskAlreadyDone(task);
|
||||
verified = postCheck.done;
|
||||
|
||||
if (!verified) {
|
||||
logger.warn("task-loop", `Task executed but verification failed: ${task.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Add completion comment
|
||||
const statusEmoji = result.success ? "✅" : "❌";
|
||||
const statusEmoji = result.success ? (verified ? "✅" : "⚠️") : "❌";
|
||||
const verificationNote = result.success
|
||||
? (verified ? "Post-execution verification: PASSED" : "Post-execution verification: COULD NOT CONFIRM — please review manually")
|
||||
: "";
|
||||
|
||||
const comment = [
|
||||
`${statusEmoji} Pi Worker ${result.success ? "completed" : "failed"} this task.`,
|
||||
verificationNote,
|
||||
`Duration: ${(result.durationMs / 1000).toFixed(1)}s`,
|
||||
`Tool calls: ${result.toolCalls}`,
|
||||
result.error ? `Error: ${result.error}` : "",
|
||||
@@ -161,13 +198,12 @@ export class TaskLoop {
|
||||
await this.asana.addComment(task.gid, comment);
|
||||
|
||||
if (result.success) {
|
||||
// Move to "Done" and mark complete
|
||||
const doneSection = await this.findFirstSection(SECTIONS.DONE);
|
||||
if (doneSection) {
|
||||
await this.asana.moveTaskToSection(task.gid, doneSection.gid);
|
||||
}
|
||||
await this.asana.updateTask(task.gid, { completed: true });
|
||||
logger.info("task-loop", `Task completed: ${task.name}`);
|
||||
logger.info("task-loop", `Task completed: ${task.name}`, { verified });
|
||||
} else {
|
||||
// Move back to To Do on failure
|
||||
const todoSection = await this.findFirstSection(SECTIONS.TODO);
|
||||
@@ -184,6 +220,48 @@ export class TaskLoop {
|
||||
}
|
||||
}
|
||||
|
||||
private async verifyTaskAlreadyDone(task: AsanaTask): Promise<{ done: boolean; evidence: string }> {
|
||||
try {
|
||||
const verifyPrompt = `You are a verification agent. Your ONLY job is to check if the following task/bug/feature is ALREADY done on this server/codebase. Do NOT fix anything. Do NOT make any changes. ONLY investigate and report.
|
||||
|
||||
Task: ${task.name}
|
||||
${task.notes ? `Details: ${task.notes}` : ""}
|
||||
|
||||
INSTRUCTIONS:
|
||||
1. Use your tools (bash, read, grep, find, ls) to investigate if this work is already completed
|
||||
2. Check the codebase, running services, server state, logs — whatever is relevant
|
||||
3. Be thorough but quick
|
||||
|
||||
RESPOND WITH EXACTLY THIS FORMAT (nothing else):
|
||||
DONE: true
|
||||
EVIDENCE: <one paragraph explaining what you found that proves it's done>
|
||||
|
||||
OR:
|
||||
DONE: false
|
||||
EVIDENCE: <one paragraph explaining what's missing or broken>`;
|
||||
|
||||
const result = await this.pi.execute(verifyPrompt);
|
||||
|
||||
if (!result.success) {
|
||||
return { done: false, evidence: "Verification check failed to run" };
|
||||
}
|
||||
|
||||
const output = result.output.trim();
|
||||
const doneMatch = output.match(/DONE:\s*(true|false)/i);
|
||||
const evidenceMatch = output.match(/EVIDENCE:\s*(.+)/is);
|
||||
|
||||
const isDone = doneMatch ? doneMatch[1].toLowerCase() === "true" : false;
|
||||
const evidence = evidenceMatch ? evidenceMatch[1].trim().substring(0, 1000) : output.substring(0, 1000);
|
||||
|
||||
logger.info("task-loop", `Verification result for "${task.name}": done=${isDone}`, { evidence: evidence.substring(0, 200) });
|
||||
|
||||
return { done: isDone, evidence };
|
||||
} catch (error: any) {
|
||||
logger.error("task-loop", `Verification error: ${error.message}`);
|
||||
return { done: false, evidence: `Verification error: ${error.message}` };
|
||||
}
|
||||
}
|
||||
|
||||
private buildTaskContext(task: AsanaTask): string {
|
||||
const parts = [
|
||||
`Asana Task: ${task.name}`,
|
||||
|
||||
Reference in New Issue
Block a user