justvitamin/scripts/generate-source-acquisition-plan.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
const contentRoot = join(root, 'data', 'content')
const sourcesRoot = join(root, 'data', 'sources')
const catalogPath = join(root, 'data', 'pipeline', 'catalog.json')

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}

function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (Array.isArray(value)) value = value.join(' | ')
    if (value === undefined || value === null) value = ''
    value = String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}

function listJson(dir: string) {
  if (!existsSync(dir)) return []
  return readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
    const path = join(dir, entry.name)
    if (entry.isDirectory()) return listJson(path)
    return entry.isFile() && entry.name.endsWith('.json') ? [path] : []
  })
}

mkdirSync(outRoot, { recursive: true })

const generatedAt = new Date().toISOString()
const catalog = readJson(catalogPath, [])
const productsBySku = new Map<string, any>()
for (const file of readdirSync(join(contentRoot, 'products')).filter(file => file.endsWith('.json'))) {
  const product = readJson(join(contentRoot, 'products', file), {})
  productsBySku.set(product.sku, product)
}
const competitorConfig = readJson(join(sourcesRoot, 'competitor-sources.json'), { competitors: [] })
const pilotSkus = [...productsBySku.values()].filter(product => product.status !== 'pending_intelligence').map(product => product.sku)
const topMissing = catalog
  .filter((row: any) => !pilotSkus.includes(row.sku))
  .sort((a: any, b: any) => Number(b.total_reviews || 0) - Number(a.total_reviews || 0))
  .slice(0, 8)
  .map((row: any) => row.sku)
const collagenSkus = [...productsBySku.values()]
  .filter(product => /collagen/i.test(`${product.sku} ${product.name || ''}`))
  .map(product => product.sku)

const stageRows = [
  {
    Stage: '0',
    Provider: 'No paid provider',
    Scope: 'Use existing Feefo-derived pilot intelligence, competitor PDP probes, source request pack, and queues.',
    Targets: `pilot SKUs: ${pilotSkus.join(' | ')}`,
    HardCap: '£0 / no paid credits',
    StopRule: 'Stop when source_request_pack, intelligence queue, and handoff actions are current.',
    OutputFolder: 'content_population_exports/',
    Verification: 'bun run source:pack && bun run intelligence:queue'
  },
  {
    Stage: '1',
    Provider: 'DataForSEO first',
    Scope: 'Small paid pilot for structured Amazon reviews/source data. Do not run full catalogue.',
    Targets: [...new Set([...pilotSkus, ...topMissing.slice(0, 5), ...collagenSkus])].join(' | '),
    HardCap: 'Max 8-12 SKU/ASIN tasks before review; stop after first invoice/credit sample.',
    StopRule: 'Stop if output lacks review body/rating/date/source URL or if per-SKU cost is too high.',
    OutputFolder: 'data/sources/amazon/',
    Verification: 'bun run validate:sources && bun run intel:refresh && bun run intelligence:queue'
  },
  {
    Stage: '2',
    Provider: 'Apify selective',
    Scope: 'Only fill gaps: competitor PDP pages, Amazon product detail pages, or Reddit/social if DataForSEO cannot provide enough detail.',
    Targets: (competitorConfig.competitors || []).map((row: any) => row.brand).join(' | '),
    HardCap: 'One actor, 1-3 targets, then inspect raw JSON before scaling.',
    StopRule: 'Stop if actor output is shallow, duplicated, blocked, or does not include source URLs/evidence fields.',
    OutputFolder: 'data/sources/competitor-pdps/ or data/sources/reddit/',
    Verification: 'bun run validate:sources && bun run build:competitors'
  },
  {
    Stage: '3',
    Provider: 'Controlled expansion',
    Scope: 'Batch by commercial priority, not full catalogue: collagen/NPD, top review SKUs, launch-critical products, then remaining catalogue.',
    Targets: topMissing.join(' | '),
    HardCap: 'One priority batch at a time; rerun handoff verification between batches.',
    StopRule: 'Stop each batch when source validation passes and intelligence task count drops as expected.',
    OutputFolder: 'data/sources/* and data/intelligence/*',
    Verification: 'bun run verify:handoff'
  }
]

const targetRows = [
  ...pilotSkus.map(sku => ({ Stage: '1', Provider: 'DataForSEO', Priority: 'pilot', SKU: sku, Product: productsBySku.get(sku)?.name || sku, Reason: 'Existing pilot content; validates provider output quality against known dashboard intelligence.' })),
  ...topMissing.map((sku, index) => ({ Stage: index < 5 ? '1' : '3', Provider: index < 5 ? 'DataForSEO' : 'Controlled expansion', Priority: index < 3 ? 'P0' : 'P1', SKU: sku, Product: productsBySku.get(sku)?.name || catalog.find((row: any) => row.sku === sku)?.name || sku, Reason: `High Feefo review count: ${catalog.find((row: any) => row.sku === sku)?.total_reviews || 0}` })),
  ...collagenSkus.map(sku => ({ Stage: '1', Provider: 'DataForSEO', Priority: 'NPD', SKU: sku, Product: productsBySku.get(sku)?.name || sku, Reason: 'Collagen/NPD research priority.' })),
  ...(competitorConfig.competitors || []).map((row: any) => ({ Stage: '2', Provider: 'Apify selective', Priority: 'competitor', SKU: '', Product: row.brand, Reason: `Competitor PDP/domain: ${row.domain}` }))
]

const manifest = {
  generatedAt,
  ready: false,
  recommendation: 'Use DataForSEO first for a tightly capped structured pilot; use Apify only as a targeted gap-filler after inspecting pilot output.',
  counts: {
    stages: stageRows.length,
    pilotSkus: pilotSkus.length,
    stageOneTargets: targetRows.filter(row => row.Stage === '1').length,
    apifyTargets: targetRows.filter(row => row.Stage === '2').length,
    expansionTargets: targetRows.filter(row => row.Stage === '3').length,
    amazonFiles: listJson(join(sourcesRoot, 'amazon')).length,
    trustpilotFiles: listJson(join(sourcesRoot, 'trustpilot')).length,
    redditFiles: listJson(join(sourcesRoot, 'reddit')).length
  },
  stages: stageRows,
  targets: targetRows
}

writeFileSync(join(outRoot, 'source_acquisition_plan.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'source_acquisition_stages.csv'), csv(stageRows, ['Stage', 'Provider', 'Scope', 'Targets', 'HardCap', 'StopRule', 'OutputFolder', 'Verification']), 'utf8')
writeFileSync(join(outRoot, 'source_acquisition_targets.csv'), csv(targetRows, ['Stage', 'Provider', 'Priority', 'SKU', 'Product', 'Reason']), 'utf8')

const md = [
  '# JV staged source acquisition plan',
  '',
  `Generated: ${generatedAt}`,
  '',
  `Recommendation: ${manifest.recommendation}`,
  '',
  '## Why staged',
  '- The dashboard needs real Amazon/Trustpilot/Reddit/claims evidence, but full-catalogue scraping would waste credit before output quality is proven.',
  '- DataForSEO is the first paid lane because it should give more predictable structured review/API output.',
  '- Apify is reserved for specific gaps where a targeted actor produces better PDP/social coverage.',
  '',
  '## Stages',
  ...stageRows.map(row => [
    `### Stage ${row.Stage} - ${row.Provider}`,
    `- Scope: ${row.Scope}`,
    `- Targets: ${row.Targets}`,
    `- Hard cap: ${row.HardCap}`,
    `- Stop rule: ${row.StopRule}`,
    `- Output folder: \`${row.OutputFolder}\``,
    `- Verification: \`${row.Verification}\``,
    ''
  ].join('\n')),
  '## Current source state',
  `- Amazon source files: ${manifest.counts.amazonFiles}`,
  `- Trustpilot source files: ${manifest.counts.trustpilotFiles}`,
  `- Reddit source files: ${manifest.counts.redditFiles}`,
  '',
  '## Rule',
  'Do not scale beyond Stage 1 until raw output has been inspected and the validation/intelligence queues improve measurably.',
  ''
].join('\n')
writeFileSync(join(outRoot, 'source_acquisition_plan.md'), md, 'utf8')

console.log(`Source acquisition plan ready: ${manifest.ready}`)
console.log(`Stage 1 targets: ${manifest.counts.stageOneTargets}`)
console.log(`Apify gap-fill targets: ${manifest.counts.apifyTargets}`)