justvitamin/scripts/summarize-collagen-starter-scrape.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const rawRoot = join(root, 'data', 'sources', 'apify', 'raw')
const profilePath = join(root, 'data', 'research-profiles', 'COLLAGEN-NPD.json')
const flavorPath = join(root, 'data', 'intelligence', 'COLLAGEN-NPD', 'flavor-intelligence.json')
const summaryPath = join(root, 'data', 'intelligence', 'COLLAGEN-NPD', 'scrape-tranche-summary.json')
const manifestPath = join(root, 'content_population_exports', 'collagen_apify_starter_scrape.json')

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function writeJson(path: string, data: any) {
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
}
function asText(item: any) {
  return [
    item.reviewTitle,
    item.reviewDescription,
    item.title,
    item.text,
    item.body,
    item.content,
    item.reviewText,
    item.url,
    item.competitorLabel
  ].filter(Boolean).join(' ')
}
const groups = [
  ['amazon', 'top3-specialist-collagen-amazon-reviews-latest.json'],
  ['reddit', 'top3-specialist-collagen-reddit-latest.json'],
  ['trustpilot_feel', 'top3-specialist-collagen-trustpilot-feel-latest.json'],
  ['trustpilot_ancient_and_brave', 'top3-specialist-collagen-trustpilot-ancient-and-brave-latest.json']
]
const keywords: Record<string, RegExp> = {
  taste: /taste|tasty|flavour|flavor|palatable|nice|delicious|drink/i,
  fishy_smell_aftertaste: /fishy|smell|odou?r|after\s?taste|marine|scent/i,
  mixability: /mix|dissolv|clump|lump|smooth|stir|blend/i,
  chalkiness_texture: /chalk|grain|gritty|powdery|texture/i,
  sweetness: /sweet|sugar|sickly|artificial/i,
  retention_subscription: /subscribe|subscription|cancel|repeat|again|reorder|month|daily|routine|habit|deliver/i,
  value_packaging: /expensive|price|money|value|packaging|container|tub|last|serving|empty|half/i,
  unflavoured: /unflavou?red|coffee|smoothie|tea|oats/i,
  vanilla: /vanilla/i,
  chocolate: /chocolate|cocoa/i,
  berry: /berry|berries|strawberry|raspberry|mixed berry|watermelon|peach/i
}

const sourceSummaries: any[] = []
const allItems: any[] = []
for (const [key, file] of groups) {
  const raw = readJson(join(rawRoot, file), {})
  const items = Array.isArray(raw.items) ? raw.items : []
  allItems.push(...items.map((item: any) => ({ ...item, sourceGroup: key })))
  const counts: Record<string, number> = {}
  for (const [label, pattern] of Object.entries(keywords)) {
    counts[label] = items.filter((item: any) => pattern.test(asText(item))).length
  }
  sourceSummaries.push({
    key,
    file,
    status: raw.run?.status || raw.status || 'unknown',
    itemCount: items.length,
    usageTotalUsd: raw.run?.usageTotalUsd || 0,
    counts
  })
}
const totalItems = allItems.length
const evidenceSnippets = allItems
  .map((item: any) => ({
    source: item.sourceGroup,
    competitor: item.competitorLabel || item.matchedAsin || item.subreddit || null,
    rating: item.ratingScore || item.rating || null,
    title: item.reviewTitle || item.title || '',
    text: (item.reviewDescription || item.text || item.body || item.reviewText || '').slice(0, 420),
    url: item.reviewUrl || item.url || item.link || null
  }))
  .filter((item: any) => `${item.title} ${item.text}`.trim())
  .slice(0, 40)
const aggregateCounts: Record<string, number> = {}
for (const [label, pattern] of Object.entries(keywords)) aggregateCounts[label] = allItems.filter(item => pattern.test(asText(item))).length
const manifest = readJson(manifestPath, {})
const cumulativeSpendUsd = readdirSync(rawRoot)
  .filter(file => /^top3-specialist-collagen-.*-\d{4}-\d{2}-\d{2}T.*\.json$/.test(file))
  .reduce((sum, file) => {
    const raw = readJson(join(rawRoot, file), {})
    return sum + Number(raw.run?.usageTotalUsd || 0)
  }, 0)
const summary = {
  sku: 'COLLAGEN-NPD',
  generatedAt: new Date().toISOString(),
  scrapeManifest: manifestPath,
  totalItems,
  latestVisibleUsageUsd: sourceSummaries.reduce((sum, row) => sum + Number(row.usageTotalUsd || 0), 0),
  cumulativeSpendUsd,
  configuredMaxUsd: manifest.configuredMaxUsd,
  totalMaxUsd: manifest.totalMaxUsd,
  sourceSummaries,
  aggregateCounts,
  earlyRead: {
    whatWeCanSayNow: [
      'The first tranche successfully captured Amazon and Trustpilot review text under a very small spend cap.',
      'Amazon already contains useful product-development signals around value perception, pack fill, dissolving and repeat-use willingness.',
      'Trustpilot captured enough brand-level reviews to inspect subscription/delivery/trust themes, but they need collagen-specific filtering before final decisions.',
      'The Reddit actor produced a small and noisy set, so Reddit needs a better next pass via stricter URL/thread targeting rather than broad search.'
    ],
    doNotConcludeYet: [
      'Do not choose the final flavour from this starter sample alone.',
      'Do not claim unflavoured, vanilla, chocolate or berry wins until review volume is larger and cleaner.',
      'Do not use current JV capsule evidence as proof for powder taste or mixability.'
    ]
  },
  evidenceSnippets
}
writeJson(summaryPath, summary)

const profile = readJson(profilePath, null)
if (profile) {
  profile.generatedAt = new Date().toISOString()
  profile.sourceCoverage = {
    ...(profile.sourceCoverage || {}),
    amazonReviews: sourceSummaries.find(row => row.key === 'amazon')?.itemCount || 0,
    trustpilotReviews: sourceSummaries.filter(row => row.key.startsWith('trustpilot')).reduce((sum, row) => sum + row.itemCount, 0),
    redditThreads: sourceSummaries.find(row => row.key === 'reddit')?.itemCount || 0
  }
  profile.scrapeCompleteness = {
    ...(profile.scrapeCompleteness || {}),
    readiness: 'starter_review_corpus_captured_needs_scale_and_filtering',
    exactProductEvidenceCount: 0,
    competitorContextEvidenceCount: totalItems,
    counts: { complete: 4, partial: 2, missing: 1 },
    latestScrape: {
      generatedAt: summary.generatedAt,
      totalItems,
      cumulativeSpendUsd: summary.cumulativeSpendUsd,
      latestVisibleUsageUsd: summary.latestVisibleUsageUsd,
      summaryPath: 'data/intelligence/COLLAGEN-NPD/scrape-tranche-summary.json'
    },
    lanes: (profile.scrapeCompleteness?.lanes || []).map((lane: any) => {
      if (lane.key === 'amazon-reviews') return { ...lane, status: 'partial', evidence: `${profile.sourceCoverage.amazonReviews} starter Amazon competitor review rows captured; needs more ASINs and flavour tagging.` }
      if (lane.key === 'trustpilot') return { ...lane, status: 'partial', evidence: `${profile.sourceCoverage.trustpilotReviews} starter Trustpilot rows captured across Feel and Ancient & Brave; needs collagen-specific filtering.` }
      if (lane.key === 'reddit') return { ...lane, status: 'partial', evidence: `${profile.sourceCoverage.redditThreads} Reddit rows captured, but quality is noisy; needs stricter thread targeting.` }
      return lane
    })
  }
  writeJson(profilePath, profile)
}

const flavor = readJson(flavorPath, null)
if (flavor) {
  flavor.generatedAt = new Date().toISOString()
  flavor.latestScrapeSummary = {
    totalItems,
    cumulativeSpendUsd: summary.cumulativeSpendUsd,
    latestVisibleUsageUsd: summary.latestVisibleUsageUsd,
    aggregateCounts,
    sourceSummaries: sourceSummaries.map(row => ({ key: row.key, itemCount: row.itemCount, status: row.status, usageTotalUsd: row.usageTotalUsd })),
    summaryPath: 'data/intelligence/COLLAGEN-NPD/scrape-tranche-summary.json'
  }
  writeJson(flavorPath, flavor)
}
console.log(`Collagen scrape summary written: ${totalItems} latest-visible items, cumulative usd=${summary.cumulativeSpendUsd}`)