justvitamin/scripts/ingest-apify-top3-source-captures.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const rawRoot = join(root, 'data', 'sources', 'apify', 'raw')
const intelRoot = join(root, 'data', 'intelligence')
const profileRoot = join(root, 'data', 'research-profiles')
const outRoot = join(root, 'content_population_exports')
const generatedAt = new Date().toISOString()
const top3 = ['JV-TURMERIC500', 'JV-DEFF1000', 'JV-VISISOFT']
const rawFiles = [
  join(rawRoot, 'top3-comprehensive-source-scrape-latest.json'),
  join(rawRoot, 'top3-social-source-scrape-latest.json')
]
const specialistFiles = existsSync(rawRoot)
  ? readdirSync(rawRoot)
    .filter(file => /^top3-specialist-.*-latest\.json$/.test(file))
    .map(file => join(rawRoot, file))
  : []

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function writeJson(path: string, data: any) {
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
}
function compactItem(item: any) {
  return {
    sku: item.sku,
    sourceType: item.sourceType,
    label: item.label,
    competitor: item.competitor || null,
    requestedUrl: item.requestedUrl,
    loadedUrl: item.loadedUrl,
    statusCode: item.statusCode,
    title: item.title,
    metaDescription: item.metaDescription,
    h1: item.h1,
    headings: item.headings || [],
    prices: item.prices || [],
    reviewLike: item.reviewLike || [],
    links: item.links || [],
    text: item.text || '',
    textChars: item.textChars || String(item.text || '').length,
    htmlChars: item.htmlChars || 0,
    capturedAt: item.capturedAt
  }
}
function itemMatchesSku(item: any, sku: string) {
  if (item.sku === sku || item.sku === 'ALL_TOP3') return true
  if (item.sku) return false
  const text = JSON.stringify(item).toLowerCase()
  if (sku === 'JV-TURMERIC500') return /turmeric|curcumin|bioperine|black pepper/.test(text)
  if (sku === 'JV-DEFF1000') return /vitamin d|1000iu|1000 iu|effervescent|fizz|deficien/.test(text)
  if (sku === 'JV-VISISOFT') return /visisoft|lutein|zeaxanthin|bilberry|eye health|vision/.test(text)
  return false
}

const runs = rawFiles.map(path => ({ path, data: readJson(path) })).filter(row => row.data)
const specialistRuns = specialistFiles.map(path => ({ path, data: readJson(path) })).filter(row => row.data)
const specialistJobs = Array.from(new Set([
  'trustpilot',
  'reddit',
  'amazon-reviews',
  ...specialistRuns.map(run => run.data?.job).filter(Boolean)
])).sort()
const manifestRows: any[] = []
for (const sku of top3) {
  const items = runs.flatMap(run => (run.data.items || [])
    .filter((item: any) => item.sku === sku || item.sku === 'ALL_TOP3')
    .map((item: any) => ({ ...compactItem(item), rawFile: run.path, runId: run.data.run?.id })))
  const requested = runs.flatMap(run => (run.data.requestedTargets || [])
    .filter((target: any) => target.sku === sku || target.sku === 'ALL_TOP3')
    .map((target: any) => ({ ...target, rawFile: run.path, runId: run.data.run?.id })))
  const capturedLabels = new Set(items.map((item: any) => item.label))
  const missing = requested.filter((target: any) => !capturedLabels.has(target.label))
  const byType = Object.fromEntries(['jv_public_pdp', 'competitor_pdp', 'amazon_search', 'trustpilot_brand', 'reddit_category'].map(type => [
    type,
    {
      requested: requested.filter((row: any) => row.sourceType === type).length,
      captured: items.filter((row: any) => row.sourceType === type).length
    }
  ]))
  const specialistItems = specialistRuns.flatMap(run => (run.data.items || [])
    .filter((item: any) => run.data.job === 'trustpilot' || itemMatchesSku(item, sku))
    .map((item: any) => ({
      sourceType: `specialist_${run.data.job}`,
      job: run.data.job,
      actor: run.data.actor,
      rawFile: run.path,
      runId: run.data.run?.id,
      item
    })))
  const exactProductEvidenceCount = specialistItems.filter((row: any) =>
    row.item?.sku === sku &&
    row.item?.matchContext !== 'closest_effervescent_competitor' &&
    row.item?.matchContext !== 'exact_asin_absence_search' &&
    row.item?.matchContext !== 'competitor_serp_fallback' &&
    row.item?.matchContext !== 'competitor_retailer_pdp_fallback' &&
    !['specialist_healthspan-serp', 'specialist_healthspan-browser', 'specialist_healthspan-content-crawler', 'specialist_healthspan-retailer'].includes(row.sourceType)
  ).length
  const competitorContextEvidenceCount = specialistItems.filter((row: any) =>
    row.item?.matchContext === 'closest_effervescent_competitor' ||
    row.item?.matchContext === 'exact_asin_absence_search' ||
    row.item?.matchContext === 'competitor_serp_fallback' ||
    row.item?.matchContext === 'competitor_retailer_pdp_fallback' ||
    row.sourceType === 'specialist_healthspan-serp' ||
    row.sourceType === 'specialist_healthspan-browser' ||
    row.sourceType === 'specialist_healthspan-content-crawler' ||
    row.sourceType === 'specialist_healthspan-retailer'
  ).length
  const specialistCoverage = Object.fromEntries(specialistJobs.map(job => [
    job,
    {
      run: specialistRuns.some(run => run.data.job === job),
      captured: specialistItems.filter((row: any) => row.job === job).length,
      status: specialistRuns.find(run => run.data.job === job)?.data?.run?.status || 'not_run'
    }
  ]))
  const captureProfile = {
    sku,
    generatedAt,
    generatedBy: 'ingest-apify-top3-source-captures.ts',
    purpose: 'Raw Apify source captures for comprehensive PDP research profile.',
    sourceFiles: runs.map(run => ({
      path: run.path,
      runId: run.data.run?.id,
      status: run.data.run?.status,
      requestCap: run.data.requestCap,
      itemCount: run.data.itemCount,
      usageTotalUsd: run.data.run?.usageTotalUsd
    })),
    coverage: byType,
    specialistCoverage,
    evidenceContext: {
      exactProductEvidenceCount,
      competitorContextEvidenceCount,
      note: 'Exact product evidence is tied to the SKU/confirmed listing. Competitor context is useful for market/PDP learning but must not override JV product truth.'
    },
    capturedCount: items.length,
    specialistCapturedCount: specialistItems.length,
    requestedCount: requested.length,
    missingCount: missing.length,
    missing,
    items,
    specialistItems
  }
  writeJson(join(intelRoot, sku, 'apify-source-captures.json'), captureProfile)
  const profilePath = join(profileRoot, `${sku}.json`)
  const profile = readJson(profilePath)
  if (profile) {
    profile.rawSourceCaptureProfile = {
      generatedAt,
      source: `${sku}/apify-source-captures.json`,
      coverage: byType,
      capturedCount: items.length,
      specialistCapturedCount: specialistItems.length,
      missingCount: missing.length,
      specialistCoverage,
      evidenceContext: {
        exactProductEvidenceCount,
        competitorContextEvidenceCount,
        note: 'Exact product evidence is tied to the SKU/confirmed listing. Competitor context is useful for market/PDP learning but must not override JV product truth.'
      },
      missing: missing.map((row: any) => ({ sourceType: row.sourceType, label: row.label, url: row.url }))
    }
    profile.sourceCoverage = {
      ...(profile.sourceCoverage || {}),
      apifyRawCaptures: items.length,
      apifySpecialistRawCaptures: specialistItems.length,
      apifyMissingTargets: missing.length
    }
    writeJson(profilePath, profile)
    writeJson(join(intelRoot, sku, 'research-profile.json'), profile)
  }
  manifestRows.push({ sku, captured: items.length, specialistCaptured: specialistItems.length, requested: requested.length, missing: missing.length, coverage: byType, specialistCoverage, evidenceContext: { exactProductEvidenceCount, competitorContextEvidenceCount } })
}

const manifest = {
  generatedAt,
  ready: manifestRows.every(row => row.captured > 0),
  note: 'Apify capture ingest is raw-source coverage, not content approval. Reddit/Trustpilot may remain missing when source pages block crawler access.',
  rawFiles: runs.map(run => run.path),
  specialistFiles: specialistRuns.map(run => run.path),
  products: manifestRows
}
writeJson(join(outRoot, 'top3_apify_source_capture_ingest.json'), manifest)
writeFileSync(join(outRoot, 'top3_apify_source_capture_ingest.md'), [
  '# Top-3 Apify source capture ingest',
  '',
  `Generated: ${generatedAt}`,
  '',
  manifest.note,
  '',
  ...manifestRows.map(row => `- ${row.sku}: ${row.captured}/${row.requested} page captures, ${row.specialistCaptured} specialist items, ${row.missing} missing`)
].join('\n') + '\n', 'utf8')
console.log(`Ingested Apify captures for ${manifestRows.length} products`)