#!/usr/bin/env bun import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs' import { join } from 'path' const root = process.cwd() const outRoot = join(root, 'content_population_exports') mkdirSync(outRoot, { recursive: true }) function readJson(path: string, fallback: any = null) { if (!existsSync(path)) return fallback return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, '')) } function csv(rows: any[], headers: string[]) { const esc = (value: any) => { if (value === undefined || value === null) value = '' if (Array.isArray(value)) value = value.join(' | ') value = String(value) return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value } return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n' } const generatedAt = new Date().toISOString() const apifyGapfill = readJson(join(outRoot, 'apify_gapfill_manifest.json'), {}) const spendGate = readJson(join(outRoot, 'source_acquisition_spend_gate.json'), {}) const budgetGate = readJson(join(outRoot, 'source_acquisition_budget_gate.json'), {}) const dataforseoReview = readJson(join(outRoot, 'dataforseo_pilot_review.json'), {}) const budget = budgetGate.budget || {} const apifyBudget = budget.apifyStage2 || {} const allowFlag = process.env.JV_SOURCE_ALLOW_PAID || '' const explicitAllow = allowFlag === 'apify-stage2' const tokenPresent = Boolean(process.env.APIFY_TOKEN) const firstBatchTargets = Number(apifyGapfill.counts?.firstBatchTargets || 0) const holdTargets = Number(apifyGapfill.counts?.holdTargets || 0) const maxActorRuns = Number(apifyBudget.maxActorRuns || 0) const maxTargets = Number(apifyBudget.maxTargets || 0) const maxRequestsPerBrand = Number(apifyBudget.maxRequestsPerBrand || 0) const maxCrawlDepth = Number(apifyGapfill.guardrails ? 1 : 0) const decision = dataforseoReview.review?.decision || dataforseoReview.decision || 'pending' const namedGap = String(dataforseoReview.review?.namedGapForApify || '').trim() const apifySelected = decision === 'run_apify_gapfill' const targetRows = Array.isArray(apifyGapfill.targets) ? apifyGapfill.targets : [] const runNowTargets = targetRows.filter((row: any) => row.RunNow === 'yes') const capsPreserved = firstBatchTargets > 0 && firstBatchTargets <= 3 && holdTargets >= 0 && maxActorRuns === 1 && maxTargets > 0 && firstBatchTargets <= maxTargets && maxRequestsPerBrand > 0 && maxRequestsPerBrand <= 3 && runNowTargets.every((row: any) => Number(row.MaxRequestsPerBrand || 0) <= maxRequestsPerBrand && Number(row.MaxDepth || 0) <= 1) const inputTemplate = readJson(join(outRoot, 'apify_competitor_pdp_input_template.json'), {}) const templateUrls = Array.isArray(inputTemplate.startUrls) ? inputTemplate.startUrls : [] const templateMatchesFirstBatch = templateUrls.length === firstBatchTargets && Number(inputTemplate.maxRequestsPerCrawl || 0) <= firstBatchTargets * maxRequestsPerBrand && Number(inputTemplate.maxCrawlDepth || 0) <= 1 const checks = [ { Check: 'Explicit paid-run flag set', Status: explicitAllow ? 'pass' : 'blocked', Blocking: 'yes', Evidence: explicitAllow ? 'JV_SOURCE_ALLOW_PAID=apify-stage2' : 'set JV_SOURCE_ALLOW_PAID=apify-stage2 only when intentionally running Apify Stage 2' }, { Check: 'Apify token present in environment', Status: tokenPresent ? 'pass' : 'blocked', Blocking: 'yes', Evidence: `APIFY_TOKEN=${tokenPresent ? 'present' : 'missing'}` }, { Check: 'Spend gate allows Apify Stage 2', Status: spendGate.spendAllowed?.apifyStage2 === true ? 'pass' : 'blocked', Blocking: 'yes', Evidence: `${spendGate.counts?.passing || 0}/${spendGate.counts?.checks || 0} spend checks passing` }, { Check: 'Budget gate allows Apify Stage 2', Status: budgetGate.budgetAllowed?.apifyStage2 === true ? 'pass' : 'blocked', Blocking: 'yes', Evidence: `${budgetGate.counts?.passing || 0}/${budgetGate.counts?.checks || 0} budget checks passing` }, { Check: 'DataForSEO review explicitly selected Apify', Status: apifySelected ? 'pass' : 'blocked', Blocking: 'yes', Evidence: `decision=${decision}` }, { Check: 'Named Apify gap documented', Status: apifySelected && namedGap.length > 0 ? 'pass' : 'blocked', Blocking: 'yes', Evidence: namedGap || 'no named gap from DataForSEO pilot review' }, { Check: 'First batch stays inside caps', Status: capsPreserved ? 'pass' : 'blocked', Blocking: 'yes', Evidence: `${firstBatchTargets} first-batch targets, ${holdTargets} held, maxActorRuns=${maxActorRuns || 'unset'}, maxTargets=${maxTargets || 'unset'}, maxRequestsPerBrand=${maxRequestsPerBrand || 'unset'}` }, { Check: 'Actor input matches first batch only', Status: templateMatchesFirstBatch ? 'pass' : 'blocked', Blocking: 'yes', Evidence: `${templateUrls.length} start URLs / ${firstBatchTargets} first-batch targets, maxRequestsPerCrawl=${inputTemplate.maxRequestsPerCrawl || 'unset'}, maxCrawlDepth=${inputTemplate.maxCrawlDepth ?? 'unset'}` } ] const ready = checks.every(row => row.Status === 'pass') const manifest = { generatedAt, ready, command: 'bun run apify:task-preflight', paidRunCommand: 'Set APIFY_TOKEN, JV_SOURCE_ALLOW_PAID=apify-stage2, confirm the chosen actor in Apify UI/API, paste only the generated first-batch input template, then stop after one dataset inspection.', counts: { checks: checks.length, passing: checks.filter(row => row.Status === 'pass').length, blocked: checks.filter(row => row.Status !== 'pass').length, firstBatchTargets, holdTargets, startUrls: templateUrls.length }, files: { actorInputTemplate: 'content_population_exports/apify_competitor_pdp_input_template.json', gapfillManifest: 'content_population_exports/apify_gapfill_manifest.json', preflightMd: 'content_population_exports/apify_task_preflight.md', preflightCsv: 'content_population_exports/apify_task_preflight.csv', preflightJson: 'content_population_exports/apify_task_preflight.json' }, checks, blockers: checks.filter(row => row.Status !== 'pass').map(row => `${row.Check}: ${row.Evidence}`), targetPreview: runNowTargets.slice(0, 6).map((row: any) => ({ brand: row.Brand, url: row.StartUrl, maxRequestsPerBrand: row.MaxRequestsPerBrand, maxDepth: row.MaxDepth })) } writeFileSync(join(outRoot, 'apify_task_preflight.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8') writeFileSync(join(outRoot, 'apify_task_preflight.csv'), csv(checks, ['Check', 'Status', 'Blocking', 'Evidence']), 'utf8') writeFileSync(join(outRoot, 'apify_task_preflight.md'), [ '# Apify paid actor preflight', '', `Generated: ${generatedAt}`, '', `Ready to run Apify Stage 2 actor: ${ready ? 'YES' : 'NO'}`, '', 'This preflight is a final local guard before any Apify paid actor run. It does not call Apify.', '', '## Checks', ...checks.map(row => `- ${row.Status === 'pass' ? '[pass]' : '[blocked]'} ${row.Check} — ${row.Evidence}`), '', '## Paid-run rule', '- All checks must pass.', '- `APIFY_TOKEN` must be a shell environment variable only.', '- `JV_SOURCE_ALLOW_PAID` must equal `apify-stage2` for the run window.', '- DataForSEO pilot review must explicitly choose `run_apify_gapfill` and name the gap.', '- Run only the first-batch input template, then stop and inspect dataset quality/cost before any scale-up.', '' ].join('\n'), 'utf8') console.log(`Apify task preflight ready: ${ready}`) console.log(`Checks passing: ${manifest.counts.passing}/${manifest.counts.checks}`) console.log(`First-batch targets: ${manifest.counts.firstBatchTargets}`)