justvitamin/scripts/validate-source-acquisition-spend.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
const approvalRoot = join(root, 'data', 'sources', 'approvals')
mkdirSync(outRoot, { recursive: true })
mkdirSync(approvalRoot, { recursive: true })

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (value === undefined || value === null) value = ''
    if (Array.isArray(value)) value = value.join(' | ')
    value = String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}

const generatedAt = new Date().toISOString()
const sourcePlan = readJson(join(outRoot, 'source_acquisition_plan.json'), {})
const dataforseoPilot = readJson(join(outRoot, 'dataforseo_pilot_manifest.json'), {})
const asinLookup = readJson(join(outRoot, 'dataforseo_asin_lookup_queue.json'), {})
const dataforseoIngest = readJson(join(outRoot, 'dataforseo_ingest_validation.json'), {})
const apifyGapfill = readJson(join(outRoot, 'apify_gapfill_manifest.json'), {})
const budgetGate = readJson(join(outRoot, 'source_acquisition_budget_gate.json'), {})
const costPlan = readJson(join(outRoot, 'source_acquisition_cost_plan.json'), {})
const approvalGate = readJson(join(outRoot, 'source_acquisition_approval_gate.json'), {})
const supportResponseGate = readJson(join(outRoot, 'dataforseo_support_response_gate.json'), {})
const reviewsDocsStatus = readJson(join(outRoot, 'dataforseo_reviews_docs_status.json'), {})
const dataforseoApprovalPath = join(approvalRoot, 'dataforseo-stage1-approval.json')
const apifyApprovalPath = join(approvalRoot, 'apify-stage2-approval.json')

const approvalTemplate = {
  approved: false,
  approvedBy: '',
  approvedAt: '',
  scope: 'fill before spending provider credits',
  notes: ''
}
if (!existsSync(dataforseoApprovalPath)) writeFileSync(dataforseoApprovalPath, JSON.stringify({ ...approvalTemplate, provider: 'DataForSEO', stage: 1 }, null, 2) + '\n', 'utf8')
if (!existsSync(apifyApprovalPath)) writeFileSync(apifyApprovalPath, JSON.stringify({ ...approvalTemplate, provider: 'Apify', stage: 2 }, null, 2) + '\n', 'utf8')
const dataforseoApproval = readJson(dataforseoApprovalPath, {})
const apifyApproval = readJson(apifyApprovalPath, {})
const dataforseoApprovalValid = (approvalGate.approvals || []).some((row: any) => row.key === 'dataforseoStage1' && row.valid === true)
const apifyApprovalValid = (approvalGate.approvals || []).some((row: any) => row.key === 'apifyStage2' && row.valid === true)
const dataforseoCostReady = costPlan.costReady?.dataforseoStage1 === true
const apifyCostReady = costPlan.costReady?.apifyStage2 === true
const readyTasks = Number(dataforseoPilot.counts?.readyTasks || 0)
const missingAsins = Number(dataforseoPilot.counts?.missingAsins ?? asinLookup.counts?.missingAsins ?? 0)

const dataforseoChecks = [
  {
    Check: 'Stage 1 targets capped',
    Status: (dataforseoPilot.counts?.stageOneTargets || 0) > 0 && (dataforseoPilot.counts?.stageOneTargets || 99) <= (dataforseoPilot.guardrails?.maxStageOneTargets || 12) ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${dataforseoPilot.counts?.stageOneTargets || 0}/${dataforseoPilot.guardrails?.maxStageOneTargets || 12} targets`
  },
  {
    Check: 'ASIN payload scoped',
    Status: readyTasks > 0 ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${readyTasks} ready payload row(s); ${missingAsins} unresolved ASIN(s) remain outside the paid pilot payload`
  },
  {
    Check: 'Task payload ready',
    Status: readyTasks > 0 ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${readyTasks} ready task rows`
  },
  {
    Check: 'Budget cap recorded',
    Status: budgetGate.budgetAllowed?.dataforseoStage1 === true ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: budgetGate.budgetAllowed?.dataforseoStage1 === true ? `${budgetGate.currency || 'GBP'} cap recorded` : 'data/sources/budgets/source-acquisition-budget.json not enabled/capped for DataForSEO'
  },
  {
    Check: 'Cost plan accepted',
    Status: dataforseoCostReady ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: dataforseoCostReady ? `${costPlan.currency || 'GBP'} ${costPlan.estimates?.dataforseoStage1?.estimatedTotal || 0} estimated / ${costPlan.estimates?.dataforseoStage1?.maxSpend || 0} max spend` : 'source_acquisition_cost_plan.json not ready for DataForSEO'
  },
  {
    Check: 'Reviews endpoint cleared by support',
    Status: supportResponseGate.ready === true ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: supportResponseGate.ready === true ? 'support response gate ready for capped retry' : `${supportResponseGate.counts?.passing || 0}/${supportResponseGate.counts?.checks || 0} support-response checks passing; endpoint/retry path not confirmed`
  },
  {
    Check: 'Official Reviews docs status allows retry',
    Status: reviewsDocsStatus.ready === true ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: reviewsDocsStatus.ready === true
      ? 'official docs no longer mark Reviews endpoint unavailable'
      : `endpointTemporarilyUnavailable=${reviewsDocsStatus.endpointTemporarilyUnavailable === true}; ${reviewsDocsStatus.counts?.passing || 0}/${reviewsDocsStatus.counts?.checks || 0} docs-status checks passing`
  },
  {
    Check: 'Human spend approval recorded',
    Status: dataforseoApproval.approved === true && dataforseoApprovalValid ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: dataforseoApproval.approved && dataforseoApprovalValid ? `${dataforseoApproval.approvedBy || 'unknown'} at ${dataforseoApproval.approvedAt || 'unknown'}` : 'data/sources/approvals/dataforseo-stage1-approval.json not approved or approval gate invalid'
  }
]

const apifyChecks = [
  {
    Check: 'DataForSEO pilot inspected first',
    Status: (dataforseoIngest.counts?.convertedReviews || 0) > 0 ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${dataforseoIngest.counts?.convertedReviews || 0} converted DataForSEO reviews`
  },
  {
    Check: 'Apify first batch capped',
    Status: (apifyGapfill.counts?.firstBatchTargets || 99) <= 18 && (apifyGapfill.targets || []).every((row: any) => Number(row.MaxRequestsPerBrand || 0) >= 100 && Number(row.MaxRequestsPerBrand || 9999) <= 1000 && Number(row.MaxDepth || 99) <= 2) ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: `${apifyGapfill.counts?.firstBatchTargets || 0} first-batch targets, ${apifyGapfill.counts?.holdTargets || 0} held`
  },
  {
    Check: 'Named gap documented',
    Status: dataforseoIngest.ready === true && (sourcePlan.counts?.amazonFiles || 0) > 0 ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: dataforseoIngest.ready === true ? 'DataForSEO ingest ready; inspect source plan for gap' : 'DataForSEO pilot not ready/inspected yet'
  },
  {
    Check: 'Budget cap recorded',
    Status: budgetGate.budgetAllowed?.apifyStage2 === true ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: budgetGate.budgetAllowed?.apifyStage2 === true ? `${budgetGate.currency || 'GBP'} cap recorded` : 'data/sources/budgets/source-acquisition-budget.json not enabled/capped for Apify'
  },
  {
    Check: 'Cost plan accepted',
    Status: apifyCostReady ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: apifyCostReady ? `${costPlan.currency || 'GBP'} ${costPlan.estimates?.apifyStage2?.estimatedTotal || 0} estimated / ${costPlan.estimates?.apifyStage2?.maxSpend || 0} max spend` : 'source_acquisition_cost_plan.json not ready for Apify'
  },
  {
    Check: 'Human spend approval recorded',
    Status: apifyApproval.approved === true && apifyApprovalValid ? 'pass' : 'blocked',
    Blocking: 'yes',
    Evidence: apifyApproval.approved && apifyApprovalValid ? `${apifyApproval.approvedBy || 'unknown'} at ${apifyApproval.approvedAt || 'unknown'}` : 'data/sources/approvals/apify-stage2-approval.json not approved or approval gate invalid'
  }
]

const rows = [
  ...dataforseoChecks.map(row => ({ Provider: 'DataForSEO', Stage: '1', ...row })),
  ...apifyChecks.map(row => ({ Provider: 'Apify', Stage: '2', ...row }))
]
const dataforseoAllowed = dataforseoChecks.every(row => row.Status === 'pass')
const apifyAllowed = apifyChecks.every(row => row.Status === 'pass')
const manifest = {
  generatedAt,
  ready: dataforseoAllowed && apifyAllowed,
  spendAllowed: {
    dataforseoStage1: dataforseoAllowed,
    apifyStage2: apifyAllowed
  },
  counts: {
    checks: rows.length,
    passing: rows.filter(row => row.Status === 'pass').length,
    blocked: rows.filter(row => row.Status !== 'pass').length
  },
  approvalFiles: {
    dataforseoStage1: 'data/sources/approvals/dataforseo-stage1-approval.json',
    apifyStage2: 'data/sources/approvals/apify-stage2-approval.json'
  },
  checks: rows,
  nextActions: rows.filter(row => row.Status !== 'pass').map(row => `${row.Provider} Stage ${row.Stage}: ${row.Check} - ${row.Evidence}`)
}
writeFileSync(join(outRoot, 'source_acquisition_spend_gate.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'source_acquisition_spend_gate.csv'), csv(rows, ['Provider', 'Stage', 'Check', 'Status', 'Blocking', 'Evidence']), 'utf8')
writeFileSync(join(outRoot, 'source_acquisition_spend_gate.md'), [
  '# Source acquisition spend gate',
  '',
  `Generated: ${generatedAt}`,
  '',
  `DataForSEO Stage 1 spend allowed: ${dataforseoAllowed ? 'YES' : 'NO'}`,
  `Apify Stage 2 spend allowed: ${apifyAllowed ? 'YES' : 'NO'}`,
  '',
  'This gate intentionally blocks provider spend until prerequisites and explicit local approval files are satisfied.',
  '',
  '## Checks',
  ...rows.map(row => `- ${row.Status === 'pass' ? '[pass]' : '[blocked]'} ${row.Provider} Stage ${row.Stage}: ${row.Check} — ${row.Evidence}`),
  '',
  '## Approval files',
  `- \`${manifest.approvalFiles.dataforseoStage1}\``,
  `- \`${manifest.approvalFiles.apifyStage2}\``,
  ''
].join('\n'), 'utf8')

console.log(`Source acquisition spend gate ready: ${manifest.ready}`)
console.log(`DataForSEO spend allowed: ${dataforseoAllowed}`)
console.log(`Apify spend allowed: ${apifyAllowed}`)
console.log(`Checks passing: ${manifest.counts.passing}/${manifest.counts.checks}`)