justvitamin/scripts/validate-apify-task-preflight.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
mkdirSync(outRoot, { recursive: true })

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}

function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (value === undefined || value === null) value = ''
    if (Array.isArray(value)) value = value.join(' | ')
    value = String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}

const generatedAt = new Date().toISOString()
const apifyGapfill = readJson(join(outRoot, 'apify_gapfill_manifest.json'), {})
const spendGate = readJson(join(outRoot, 'source_acquisition_spend_gate.json'), {})
const budgetGate = readJson(join(outRoot, 'source_acquisition_budget_gate.json'), {})
const dataforseoReview = readJson(join(outRoot, 'dataforseo_pilot_review.json'), {})
const budget = budgetGate.budget || {}
const apifyBudget = budget.apifyStage2 || {}
const allowFlag = process.env.JV_SOURCE_ALLOW_PAID || ''
const explicitAllow = allowFlag === 'apify-stage2'
const tokenPresent = Boolean(process.env.APIFY_TOKEN)
const firstBatchTargets = Number(apifyGapfill.counts?.firstBatchTargets || 0)
const holdTargets = Number(apifyGapfill.counts?.holdTargets || 0)
const maxActorRuns = Number(apifyBudget.maxActorRuns || 0)
const maxTargets = Number(apifyBudget.maxTargets || 0)
const maxRequestsPerBrand = Number(apifyBudget.maxRequestsPerBrand || 0)
const maxCrawlDepth = Number(apifyGapfill.guardrails ? 1 : 0)
const decision = dataforseoReview.review?.decision || dataforseoReview.decision || 'pending'
const namedGap = String(dataforseoReview.review?.namedGapForApify || '').trim()
const apifySelected = decision === 'run_apify_gapfill'
const targetRows = Array.isArray(apifyGapfill.targets) ? apifyGapfill.targets : []
const runNowTargets = targetRows.filter((row: any) => row.RunNow === 'yes')
const capsPreserved = firstBatchTargets > 0 && firstBatchTargets <= 3 && holdTargets >= 0 && maxActorRuns === 1 && maxTargets > 0 && firstBatchTargets <= maxTargets && maxRequestsPerBrand > 0 && maxRequestsPerBrand <= 3 && runNowTargets.every((row: any) => Number(row.MaxRequestsPerBrand || 0) <= maxRequestsPerBrand && Number(row.MaxDepth || 0) <= 1)
const inputTemplate = readJson(join(outRoot, 'apify_competitor_pdp_input_template.json'), {})
const templateUrls = Array.isArray(inputTemplate.startUrls) ? inputTemplate.startUrls : []
const templateMatchesFirstBatch = templateUrls.length === firstBatchTargets && Number(inputTemplate.maxRequestsPerCrawl || 0) <= firstBatchTargets * maxRequestsPerBrand && Number(inputTemplate.maxCrawlDepth || 0) <= 1

const checks = [
  {
    Check: 'Explicit paid-run flag set',
    Status: explicitAllow ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: explicitAllow ? 'JV_SOURCE_ALLOW_PAID=apify-stage2' : 'set JV_SOURCE_ALLOW_PAID=apify-stage2 only when intentionally running Apify Stage 2'
  },
  {
    Check: 'Apify token present in environment',
    Status: tokenPresent ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `APIFY_TOKEN=${tokenPresent ? 'present' : 'missing'}`
  },
  {
    Check: 'Spend gate allows Apify Stage 2',
    Status: spendGate.spendAllowed?.apifyStage2 === true ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${spendGate.counts?.passing || 0}/${spendGate.counts?.checks || 0} spend checks passing`
  },
  {
    Check: 'Budget gate allows Apify Stage 2',
    Status: budgetGate.budgetAllowed?.apifyStage2 === true ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${budgetGate.counts?.passing || 0}/${budgetGate.counts?.checks || 0} budget checks passing`
  },
  {
    Check: 'DataForSEO review explicitly selected Apify',
    Status: apifySelected ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `decision=${decision}`
  },
  {
    Check: 'Named Apify gap documented',
    Status: apifySelected && namedGap.length > 0 ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: namedGap || 'no named gap from DataForSEO pilot review'
  },
  {
    Check: 'First batch stays inside caps',
    Status: capsPreserved ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${firstBatchTargets} first-batch targets, ${holdTargets} held, maxActorRuns=${maxActorRuns || 'unset'}, maxTargets=${maxTargets || 'unset'}, maxRequestsPerBrand=${maxRequestsPerBrand || 'unset'}`
  },
  {
    Check: 'Actor input matches first batch only',
    Status: templateMatchesFirstBatch ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${templateUrls.length} start URLs / ${firstBatchTargets} first-batch targets, maxRequestsPerCrawl=${inputTemplate.maxRequestsPerCrawl || 'unset'}, maxCrawlDepth=${inputTemplate.maxCrawlDepth ?? 'unset'}`
  }
]

const ready = checks.every(row => row.Status === 'pass')
const manifest = {
  generatedAt,
  ready,
  command: 'bun run apify:task-preflight',
  paidRunCommand: 'Set APIFY_TOKEN, JV_SOURCE_ALLOW_PAID=apify-stage2, confirm the chosen actor in Apify UI/API, paste only the generated first-batch input template, then stop after one dataset inspection.',
  counts: {
    checks: checks.length,
    passing: checks.filter(row => row.Status === 'pass').length,
    blocked: checks.filter(row => row.Status !== 'pass').length,
    firstBatchTargets,
    holdTargets,
    startUrls: templateUrls.length
  },
  files: {
    actorInputTemplate: 'content_population_exports/apify_competitor_pdp_input_template.json',
    gapfillManifest: 'content_population_exports/apify_gapfill_manifest.json',
    preflightMd: 'content_population_exports/apify_task_preflight.md',
    preflightCsv: 'content_population_exports/apify_task_preflight.csv',
    preflightJson: 'content_population_exports/apify_task_preflight.json'
  },
  checks,
  blockers: checks.filter(row => row.Status !== 'pass').map(row => `${row.Check}: ${row.Evidence}`),
  targetPreview: runNowTargets.slice(0, 6).map((row: any) => ({ brand: row.Brand, url: row.StartUrl, maxRequestsPerBrand: row.MaxRequestsPerBrand, maxDepth: row.MaxDepth }))
}

writeFileSync(join(outRoot, 'apify_task_preflight.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'apify_task_preflight.csv'), csv(checks, ['Check', 'Status', 'Blocking', 'Evidence']), 'utf8')
writeFileSync(join(outRoot, 'apify_task_preflight.md'), [
  '# Apify paid actor preflight',
  '',
  `Generated: ${generatedAt}`,
  '',
  `Ready to run Apify Stage 2 actor: ${ready ? 'YES' : 'NO'}`,
  '',
  'This preflight is a final local guard before any Apify paid actor run. It does not call Apify.',
  '',
  '## Checks',
  ...checks.map(row => `- ${row.Status === 'pass' ? '[pass]' : '[blocked]'} ${row.Check} — ${row.Evidence}`),
  '',
  '## Paid-run rule',
  '- All checks must pass.',
  '- `APIFY_TOKEN` must be a shell environment variable only.',
  '- `JV_SOURCE_ALLOW_PAID` must equal `apify-stage2` for the run window.',
  '- DataForSEO pilot review must explicitly choose `run_apify_gapfill` and name the gap.',
  '- Run only the first-batch input template, then stop and inspect dataset quality/cost before any scale-up.',
  ''
].join('\n'), 'utf8')

console.log(`Apify task preflight ready: ${ready}`)
console.log(`Checks passing: ${manifest.counts.passing}/${manifest.counts.checks}`)
console.log(`First-batch targets: ${manifest.counts.firstBatchTargets}`)