justvitamin/scripts/generate-top3-scrape-completeness.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const intelRoot = join(root, 'data', 'intelligence')
const profileRoot = join(root, 'data', 'research-profiles')
const outRoot = join(root, 'content_population_exports')
const migrationCsv = join(root, '..', 'JV Migration to shopify', 'products_import_lewis_v4.csv')
const generatedAt = new Date().toISOString()
const top3 = ['JV-TURMERIC500', 'JV-DEFF1000', 'JV-VISISOFT']

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function writeJson(path: string, data: any) {
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
}
function status(ok: boolean, partial = false) {
  return ok ? 'complete' : partial ? 'partial' : 'missing'
}

function laneRow(key: string, label: string, required: boolean, state: string, evidence: string, use: string, risk = '') {
  return { key, label, required, status: state, evidence, use, risk }
}

const products: any[] = []
for (const sku of top3) {
  const capture = readJson(join(intelRoot, sku, 'apify-source-captures.json'), {})
  const profile = readJson(join(profileRoot, `${sku}.json`), {})
  const cov = capture.coverage || {}
  const spec = capture.specialistCoverage || {}
  const exact = capture.evidenceContext?.exactProductEvidenceCount || 0
  const context = capture.evidenceContext?.competitorContextEvidenceCount || 0
  const amazonReviewRows = (capture.specialistItems || []).filter((row: any) => String(row.job || '').includes('amazon-reviews') && row.item?.reviewDescription)
  const deffAsinProofRows = (capture.specialistItems || []).filter((row: any) => row.job === 'deff-amazon-serp-proof')
  const healthspanRetailerRows = (capture.specialistItems || []).filter((row: any) => row.job === 'healthspan-retailer')
  const deffAsinAbsenceProven = sku === 'JV-DEFF1000' && deffAsinProofRows.some((row: any) =>
    row.item?.matchContext === 'exact_asin_absence_search' ||
    row.item?.conclusion === 'no_exact_jv_effervescent_amazon_result_found_in_bounded_serp'
  )
  const healthspanSerpRows = (capture.specialistItems || []).filter((row: any) => row.job === 'healthspan-serp')
  const competitorRequested = cov.competitor_pdp?.requested || 0
  const competitorCaptured = cov.competitor_pdp?.captured || 0
  const competitorFallback = healthspanSerpRows.length > 0 || healthspanRetailerRows.length > 0
  const amazonExactRows = amazonReviewRows.filter((row: any) => row.item?.matchContext !== 'closest_effervescent_competitor')
  const amazonContextRows = amazonReviewRows.filter((row: any) => row.item?.matchContext === 'closest_effervescent_competitor')

  const lanes = [
    laneRow(
      'jv_truth',
      'JV product truth database',
      true,
      status(existsSync(migrationCsv)),
      existsSync(migrationCsv) ? `Found migration CSV: ${migrationCsv}` : 'Migration CSV missing',
      'Defines title, SKU, variants, format, ingredients, intake, warnings and product truth.'
    ),
    laneRow(
      'jv_public_pdp',
      'Official JV public PDP',
      true,
      status((cov.jv_public_pdp?.captured || 0) > 0),
      `${cov.jv_public_pdp?.captured || 0}/${cov.jv_public_pdp?.requested || 0} captured by Apify comprehensive scrape`,
      'Provides public PDP wording and product page context.'
    ),
    laneRow(
      'competitor_pdp',
      'Competitor PDP/page evidence',
      true,
      status(competitorRequested > 0 && (competitorCaptured >= competitorRequested || (competitorCaptured > 0 && healthspanRetailerRows.length > 0)), competitorCaptured > 0 || competitorFallback),
      competitorFallback
        ? `${competitorCaptured}/${competitorRequested} direct competitor PDP captures plus ${healthspanRetailerRows.length} Healthspan retailer PDP fallback rows and ${healthspanSerpRows.length} SERP fallback rows`
        : `${competitorCaptured}/${competitorRequested} direct competitor PDP captures`,
      'Supports offer, positioning, image brief and comparison sections.',
      healthspanRetailerRows.length > 0 ? 'Healthspan direct PDP blocked; retailer PDP fallback is used for raw product-page content.' : (competitorFallback ? 'Healthspan direct PDP blocked; fallback is SERP snippet/source evidence, not full PDP text.' : '')
    ),
    laneRow(
      'amazon_search',
      'Amazon/DataForSEO marketplace rows',
      true,
      status((cov.amazon_search?.captured || 0) > 0),
      `${cov.amazon_search?.captured || 0}/${cov.amazon_search?.requested || 0} Apify Amazon-search captures plus DataForSEO product rows surfaced in Raw Evidence`,
      'Supports market pressure, ASIN/rating/vote context and marketplace comparisons.'
    ),
    laneRow(
      'amazon_reviews',
      'Amazon review text',
      true,
      status(amazonExactRows.length > 0 || (deffAsinAbsenceProven && amazonContextRows.length > 0), amazonContextRows.length > 0),
      deffAsinAbsenceProven
        ? `${amazonExactRows.length} exact-product review rows; ${amazonContextRows.length} competitor-context review rows; exact ASIN absence proof captured`
        : `${amazonExactRows.length} exact-product review rows; ${amazonContextRows.length} competitor-context review rows`,
      'Shows raw customer review language in the dashboard.',
      amazonContextRows.length > 0 && amazonExactRows.length === 0 ? 'Competitor-context reviews cannot be treated as JV product truth; absence proof explains why exact reviews are unavailable.' : ''
    ),
    laneRow(
      'reddit',
      'Reddit/category language',
      true,
      status((spec.reddit?.captured || 0) > 0),
      `${spec.reddit?.captured || 0} Reddit specialist rows`,
      'Captures raw category anxieties, objections and language.'
    ),
    laneRow(
      'trustpilot',
      'Trustpilot brand reviews',
      true,
      status((spec.trustpilot?.captured || 0) > 0),
      `${spec.trustpilot?.captured || 0} Trustpilot specialist rows`,
      'Brand-level voice and trust evidence; product specificity may be limited.'
    ),
    laneRow(
      'raw_dashboard_visibility',
      'Dashboard raw visibility',
      true,
      status(Boolean(profile.rawSourceCaptureProfile)),
      profile.rawSourceCaptureProfile ? 'Research profile has rawSourceCaptureProfile and Raw Evidence API has source sections.' : 'rawSourceCaptureProfile missing',
      'Ensures scraped content is visible in the dashboard, not trapped in JSON only.'
    )
  ]

  const required = lanes.filter(row => row.required)
  const complete = required.filter(row => row.status === 'complete').length
  const partial = required.filter(row => row.status === 'partial').length
  const missing = required.filter(row => row.status === 'missing').length
  const readiness = missing === 0 && partial === 0 ? 'complete' : missing === 0 ? 'usable_with_disclosed_context_gaps' : 'incomplete'
  const row = {
    sku,
    generatedAt,
    readiness,
    counts: { complete, partial, missing, required: required.length },
    exactProductEvidenceCount: exact,
    competitorContextEvidenceCount: context,
    lanes
  }
  products.push(row)
  profile.scrapeCompleteness = row
  writeJson(join(profileRoot, `${sku}.json`), profile)
  writeJson(join(intelRoot, sku, 'research-profile.json'), profile)
}

const manifest = {
  generatedAt,
  ready: products.every(row => row.readiness === 'complete' || row.readiness === 'usable_with_disclosed_context_gaps'),
  completionDefinition: 'Ready means every required lane has either complete evidence or an explicitly labelled fallback/context gap; exact product truth still comes only from the JV database.',
  products
}
writeJson(join(outRoot, 'top3_scrape_completeness.json'), manifest)
writeFileSync(join(outRoot, 'top3_scrape_completeness.md'), [
  '# Top-3 scrape completeness',
  '',
  `Generated: ${generatedAt}`,
  '',
  manifest.completionDefinition,
  '',
  ...products.map(product => [
    `## ${product.sku} — ${product.readiness}`,
    `- Complete: ${product.counts.complete}/${product.counts.required}`,
    `- Partial: ${product.counts.partial}`,
    `- Missing: ${product.counts.missing}`,
    `- Exact product evidence items: ${product.exactProductEvidenceCount}`,
    `- Competitor-context evidence items: ${product.competitorContextEvidenceCount}`,
    ...product.lanes.map((lane: any) => `  - [${lane.status}] ${lane.label}: ${lane.evidence}${lane.risk ? ` Risk: ${lane.risk}` : ''}`)
  ].join('\n')),
  ''
].join('\n'), 'utf8')

console.log(`Top-3 scrape completeness ready: ${products.map(row => `${row.sku}=${row.readiness}`).join(', ')}`)