justvitamin/scripts/generate-scraping-only-source-model.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const migrationCsv = join(root, '..', 'JV Migration to shopify', 'products_import_lewis_v4.csv')
const sourceRoot = join(root, 'data', 'sources')
const outRoot = join(root, 'content_population_exports')
const modelPath = join(sourceRoot, 'scraping-only-source-model.json')
const reportPath = join(outRoot, 'scraping_only_source_model.md')
const reportJsonPath = join(outRoot, 'scraping_only_source_model.json')

const intendedTop3 = [
  { intent: 'Turmeric with BioPerine', currentSku: 'JV-TURMERIC500', canonicalSku: 'JV-TURMERIC500' },
  { intent: 'Vitamin D Effervescent 1000iu', currentSku: 'JV-D1000', canonicalSku: 'JV-DEFF1000' },
  { intent: 'Visisoft Original 10mg', currentSku: 'JV-VISISOFT', canonicalSku: 'JV-VISISOFT' }
]

function writeJson(path: string, data: any) {
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
}

function parseCsv(text: string) {
  const rows: string[][] = []
  let row: string[] = []
  let cell = ''
  let quoted = false
  for (let i = 0; i < text.length; i++) {
    const char = text[i]
    const next = text[i + 1]
    if (char === '"') {
      if (quoted && next === '"') {
        cell += '"'
        i++
      } else {
        quoted = !quoted
      }
    } else if (char === ',' && !quoted) {
      row.push(cell)
      cell = ''
    } else if ((char === '\n' || char === '\r') && !quoted) {
      if (char === '\r' && next === '\n') i++
      row.push(cell)
      if (row.some(value => value !== '')) rows.push(row)
      row = []
      cell = ''
    } else {
      cell += char
    }
  }
  row.push(cell)
  if (row.some(value => value !== '')) rows.push(row)
  const headers = rows.shift() || []
  return rows.map(values => Object.fromEntries(headers.map((header, index) => [header, values[index] || ''])))
}

function clip(value: string, limit = 700) {
  value = String(value || '').replace(/_x000D_/g, '\n')
  return value.length > limit ? `${value.slice(0, limit)}…` : value
}

function truthForSku(rows: any[], sku: string) {
  const variants = rows.filter(row => row['Variant SKU'] === sku || String(row['Variant SKU'] || '').startsWith(`${sku}-`))
  const master = variants.find(row => row.Title || row['Body HTML']) || variants[0]
  if (!master) return null
  return {
    sku,
    handle: master.Handle,
    title: master.Title,
    oldUrl: master['Metafield: temp.url [single_line_text_field]'],
    format: master['Metafield: filter.format [single_line_text_field]'],
    strength: master['Metafield: custom.strength [single_line_text_field]'] || master['Metafield: filter.strength [single_line_text_field]'],
    strapline: master['Metafield: pdp.strapline [single_line_text_field]'],
    recommendedIntake: master['Metafield: pdp.recommended_intake [single_line_text_field]'],
    ingredients: master['Metafield: pdp.ingredients [rich_text_field]'],
    uspList: master['Metafield: usp.list [list.single_line_text_field]'],
    bodyHtmlPreview: clip(master['Body HTML']),
    variantSkus: variants.map(row => row['Variant SKU']).filter(Boolean),
    variantCount: variants.length
  }
}

function sourceStatus(sku: string) {
  const files = {
    jvDatabase: existsSync(migrationCsv),
    amazonDataforseo: existsSync(join(sourceRoot, 'dataforseo', 'products_raw')) &&
      readFileSync(join(sourceRoot, 'dataforseo', 'products_raw', 'latest-products-task-summary.json'), 'utf8') !== undefined,
    publicEvidence: existsSync(join(sourceRoot, 'top3-public-evidence', `${sku}.json`)),
    redditRaw: existsSync(join(sourceRoot, 'reddit', 'supplements-vitamin-d-turmeric-public-snippets-2026-05-19.json')),
    trustpilotRaw: existsSync(join(sourceRoot, 'trustpilot', 'just-vitamins-trustpilot-public-snippet-2026-05-19.json')),
    apifyCompetitor: existsSync(join(sourceRoot, 'apify', 'raw', 'top3-market-competitor-capped-apify.json'))
  }
  return files
}

const rows = parseCsv(readFileSync(migrationCsv, 'utf8').replace(/^\uFEFF/, ''))
const products = intendedTop3.map(target => {
  const currentTruth = truthForSku(rows, target.currentSku)
  const canonicalTruth = truthForSku(rows, target.canonicalSku)
  return {
    ...target,
    status: target.currentSku === target.canonicalSku ? 'canonical_ok' : 'current_sku_mismatch_use_canonical',
    currentTruth,
    canonicalTruth,
    scrapeOnlySources: {
      productTruth: {
        role: 'ground truth only; not scraped from internet',
        source: migrationCsv,
        status: canonicalTruth ? 'available' : 'missing'
      },
      amazon: {
        role: 'marketplace listing/review evidence',
        provider: 'DataForSEO Amazon Products/Reviews or capped Apify/Amazon actor',
        status: sourceStatus(target.canonicalSku).amazonDataforseo ? 'partial_raw_available' : 'missing'
      },
      competitors: {
        role: 'competitor PDP page copy, offer, proof, price and objections',
        provider: 'Apify capped crawler or direct fetch where allowed',
        status: sourceStatus(target.canonicalSku).apifyCompetitor ? 'partial_raw_available' : 'missing'
      },
      trustpilot: {
        role: 'brand-level review voice and trust blockers',
        provider: 'Trustpilot raw public snippets / approved scraper',
        status: sourceStatus(target.canonicalSku).trustpilotRaw ? 'brand_raw_available' : 'missing'
      },
      reddit: {
        role: 'category language, anxieties, comparison mental models',
        provider: 'Reddit public snippets/API-safe capture',
        status: sourceStatus(target.canonicalSku).redditRaw ? 'partial_category_raw_available' : 'missing'
      }
    }
  }
})

const model = {
  generatedAt: new Date().toISOString(),
  mode: 'scraping_only_first',
  principle: 'No PDP content generation should run until raw source capture exists and the JV migration database has supplied product truth.',
  sourceRoles: [
    'JV database = product truth: title, SKU, variant, format, ingredients, directions, warnings, old URL, price, barcode, stock.',
    'Amazon = marketplace listing/review evidence, not product truth.',
    'Competitors = positioning and offer comparison, not product truth.',
    'Trustpilot = brand trust/review voice, mostly brand-level unless product-specific rows are captured.',
    'Reddit = category language and objections, not product claims.'
  ],
  products
}

writeJson(modelPath, model)
writeJson(reportJsonPath, model)
mkdirSync(dirname(reportPath), { recursive: true })
writeFileSync(reportPath, [
  '# Scraping-only source model',
  '',
  `Generated: ${model.generatedAt}`,
  '',
  '## Principle',
  model.principle,
  '',
  '## Source roles',
  ...model.sourceRoles.map(row => `- ${row}`),
  '',
  '## Top-3 canonical scrape targets',
  ...products.map(product => [
    '',
    `### ${product.intent}`,
    `- Current dashboard SKU: ${product.currentSku}`,
    `- Canonical scrape SKU: ${product.canonicalSku}`,
    `- Status: ${product.status}`,
    `- JV DB title: ${product.canonicalTruth?.title || 'MISSING'}`,
    `- JV DB variant SKUs: ${(product.canonicalTruth?.variantSkus || []).join(', ') || 'MISSING'}`,
    `- Product truth source: ${product.scrapeOnlySources.productTruth.status}`,
    `- Amazon: ${product.scrapeOnlySources.amazon.status}`,
    `- Competitors: ${product.scrapeOnlySources.competitors.status}`,
    `- Trustpilot: ${product.scrapeOnlySources.trustpilot.status}`,
    `- Reddit: ${product.scrapeOnlySources.reddit.status}`
  ].join('\n')),
  '',
  '## Critical correction',
  '- Vitamin D Effervescent 1000iu is `JV-DEFF1000` in the JV migration database. `JV-D1000` is High Strength Vitamin D3 1000iu tablets. Scraping and downstream research must use the canonical SKU for the actual product being researched.',
  ''
].join('\n'), 'utf8')

console.log(`Wrote ${modelPath}`)
console.log(`Wrote ${reportPath}`)