justvitamin/scripts/generate-apify-gapfill-packet.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
const sourcesRoot = join(root, 'data', 'sources')
const apifyRoot = join(sourcesRoot, 'apify')
const rawRoot = join(apifyRoot, 'raw')

mkdirSync(outRoot, { recursive: true })
mkdirSync(rawRoot, { recursive: true })

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}

function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (value === undefined || value === null) value = ''
    if (Array.isArray(value)) value = value.join(' | ')
    value = String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}

function slug(value: string) {
  return String(value).toLowerCase().replace(/&/g, 'and').replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
}

const generatedAt = new Date().toISOString()
const acquisition = readJson(join(outRoot, 'source_acquisition_plan.json'), { targets: [] })
const competitors = readJson(join(sourcesRoot, 'competitor-sources.json'), { competitors: [] }).competitors || []
const requestsPerBrand = Math.min(1000, Math.max(100, Number(Bun.env.APIFY_REQUESTS_PER_BRAND || 100)))
const maxDepth = Math.min(2, Math.max(1, Number(Bun.env.APIFY_MAX_DEPTH || 1)))
const stageTwoTargets = (acquisition.targets || []).filter((row: any) => row.Stage === '2' && /Apify/i.test(row.Provider || ''))
const targetByBrand = new Map(stageTwoTargets.map((row: any) => [row.Product, row]))

const rows = competitors.map((competitor: any, index: number) => {
  const stageTarget = targetByBrand.get(competitor.brand) || {}
  const urls = competitor.urls || [`https://${competitor.domain}/`]
  const pdpPath = join(sourcesRoot, 'competitor-pdps', `${slug(competitor.brand)}.json`)
  const existingPdp = existsSync(pdpPath)
  return {
    Stage: '2',
    Provider: 'Apify selective',
    Priority: index < 3 ? 'pilot_gapfill' : 'hold_until_pilot_review',
    Brand: competitor.brand,
    Domain: competitor.domain,
    StartUrl: urls[0],
    ExistingPdpProbe: existingPdp ? 'yes' : 'no',
    RunNow: index < 3 ? 'yes' : 'no',
    MaxRequestsPerBrand: requestsPerBrand,
    MaxDepth: maxDepth,
    TargetRawPath: `data/sources/apify/raw/${slug(competitor.brand)}.json`,
    IntendedOutputPath: existingPdp ? `data/sources/competitor-pdps/${slug(competitor.brand)}.json` : `data/sources/competitor-pdps/${slug(competitor.brand)}.json`,
    Reason: stageTarget.Reason || `Competitor PDP/domain: ${competitor.domain}`
  }
})

const firstBatch = rows.filter(row => row.RunNow === 'yes')
const actorInputTemplate = {
  note: 'No-spend template. Paste this into the chosen Apify actor only after DataForSEO pilot review shows a real PDP/social gap.',
  spendGuardrail: {
    firstBatchBrands: firstBatch.map(row => row.Brand),
    maxRequestsPerBrand: requestsPerBrand,
    maxDepth,
    stopAfterDatasetInspection: true
  },
  startUrls: firstBatch.map(row => ({ url: row.StartUrl })),
  maxRequestsPerCrawl: firstBatch.length * requestsPerBrand,
  maxCrawlDepth: maxDepth,
  proxyConfiguration: { useApifyProxy: true },
  outputExpectation: 'Keep title, URL, visible product claims, pricing cues, review/testimonial snippets if present, and source URLs. Reject shallow/duplicate/blocked output before scaling.'
}

const rawOutputTemplate = {
  source: 'apify',
  brand: 'Feel',
  domain: 'wearefeel.com',
  capturedAt: generatedAt,
  actor: 'fill-after-run',
  datasetId: 'fill-after-run',
  costUsd: null,
  pages: [
    {
      url: 'https://example.com/product',
      title: '',
      text: '',
      productClaims: [],
      reviewSnippets: [],
      imageUrls: []
    }
  ]
}

const manifest = {
  generatedAt,
  ready: false,
  recommendation: 'Top-3 scraping budget raised for hundreds of items. Run capped first-batch targets, inspect the dataset, then decide whether to scale toward 1000 items per product/source.',
  counts: {
    targets: rows.length,
    firstBatchTargets: firstBatch.length,
    holdTargets: rows.length - firstBatch.length,
    existingPdpProbes: rows.filter(row => row.ExistingPdpProbe === 'yes').length
  },
  guardrails: {
    runOrder: 'DataForSEO Stage 1 first; Apify Stage 2 only for named gaps.',
    hardCap: `One actor lane, first ${firstBatch.length} brands, max ${requestsPerBrand} requests per brand, max depth ${maxDepth}, then stop for dataset inspection.`,
    rejectIf: ['blocked pages', 'duplicate content', 'no source URLs', 'no useful PDP/review/claim text', 'cost unclear before scaling']
  },
  files: {
    targetsCsv: 'content_population_exports/apify_gapfill_targets.csv',
    runbook: 'content_population_exports/apify_gapfill_runbook.md',
    actorInputTemplate: 'content_population_exports/apify_competitor_pdp_input_template.json',
    rawOutputTemplate: 'data/sources/apify/raw/_apify-output-template.json'
  },
  targets: rows
}

writeFileSync(join(outRoot, 'apify_gapfill_manifest.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'apify_gapfill_targets.csv'), csv(rows, ['Stage', 'Provider', 'Priority', 'Brand', 'Domain', 'StartUrl', 'ExistingPdpProbe', 'RunNow', 'MaxRequestsPerBrand', 'MaxDepth', 'TargetRawPath', 'IntendedOutputPath', 'Reason']), 'utf8')
writeFileSync(join(outRoot, 'apify_competitor_pdp_input_template.json'), JSON.stringify(actorInputTemplate, null, 2) + '\n', 'utf8')
writeFileSync(join(rawRoot, '_apify-output-template.json'), JSON.stringify(rawOutputTemplate, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'apify_gapfill_runbook.md'), [
  '# Apify selective gap-fill packet',
  '',
  `Generated: ${generatedAt}`,
  '',
  manifest.recommendation,
  '',
  '## Spend guardrails',
  `- ${manifest.guardrails.runOrder}`,
  `- ${manifest.guardrails.hardCap}`,
  '- Do not run all 7 competitor domains in one go.',
  '- Save raw datasets under `data/sources/apify/raw/` and inspect before any normalization/scaling.',
  '',
  '## First batch only if needed',
  ...firstBatch.map(row => `- ${row.Brand}: ${row.StartUrl} (${row.MaxRequestsPerBrand} requests max, depth ${row.MaxDepth})`),
  '',
  '## Hold until first-batch review',
  ...rows.filter(row => row.RunNow !== 'yes').map(row => `- ${row.Brand}: ${row.StartUrl}`),
  '',
  '## Files',
  `- \`${manifest.files.targetsCsv}\` - all Stage 2 targets with run/hold status.`,
  `- \`${manifest.files.actorInputTemplate}\` - no-spend actor input template for the first batch only.`,
  `- \`${manifest.files.rawOutputTemplate}\` - expected local raw output shape after a run.`,
  '',
  '## Stop / scale rule',
  'Scale only if the first dataset contains useful PDP/review/claim evidence with source URLs and the cost is acceptable. Otherwise keep Apify off and stay with DataForSEO/manual source drops.',
  ''
].join('\n'), 'utf8')

console.log(`Apify gap-fill packet ready: ${manifest.ready}`)
console.log(`Targets: ${manifest.counts.targets}`)
console.log(`First-batch targets: ${manifest.counts.firstBatchTargets}`)
console.log(`Hold targets: ${manifest.counts.holdTargets}`)
console.log(`Existing PDP probes: ${manifest.counts.existingPdpProbes}`)