justvitamin/scripts/run-apify-top3-source-scrape.ts

#!/usr/bin/env bun
import { mkdirSync, writeFileSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const rawRoot = join(root, 'data', 'sources', 'apify', 'raw')
const outRoot = join(root, 'content_population_exports')
const token = Bun.env.APIFY_TOKEN || Bun.env.APIFY_API_TOKEN
const generatedAt = new Date().toISOString()

const targets = [
  // JV product pages: web evidence; database remains product truth.
  { sku: 'JV-TURMERIC500', sourceType: 'jv_public_pdp', label: 'JV Turmeric public PDP', url: 'https://www.justvitamins.co.uk/Turmeric-Tablets-with-Bioperine-Black-Pepper-Extract.aspx' },
  { sku: 'JV-DEFF1000', sourceType: 'jv_public_pdp', label: 'JV Vitamin D Effervescent public PDP', url: 'https://www.justvitamins.co.uk/Vitamin-D/One-A-Day-Vitamin-D-Effervescent-Tablets.aspx' },
  { sku: 'JV-VISISOFT', sourceType: 'jv_public_pdp', label: 'JV Visisoft public PDP', url: 'https://www.justvitamins.co.uk/Vitamins-To-Aid-Vision/Visisoft-Lutein.aspx' },

  // Competitor PDPs.
  { sku: 'JV-TURMERIC500', sourceType: 'competitor_pdp', label: 'Feel Pro Turmeric', competitor: 'Feel', url: 'https://wearefeel.com/en-us/products/feel-pro-turmeric' },
  { sku: 'JV-TURMERIC500', sourceType: 'competitor_pdp', label: 'Ancient + Brave turmeric search/product context', competitor: 'Ancient + Brave', url: 'https://ancientandbrave.earth/search?q=turmeric' },
  { sku: 'JV-DEFF1000', sourceType: 'competitor_pdp', label: 'Vitabiotics Immunace Fizz', competitor: 'Vitabiotics', url: 'https://www.vitabiotics.com/collections/all-vitabiotics-products/products/immunace-fizz-effervescent-tablets' },
  { sku: 'JV-DEFF1000', sourceType: 'competitor_pdp', label: 'Healthspan Vitamin D context', competitor: 'Healthspan', url: 'https://www.healthspan.co.uk/search?q=vitamin%20d' },
  { sku: 'JV-VISISOFT', sourceType: 'competitor_pdp', label: 'Healthspan OptiVision', competitor: 'Healthspan', url: 'https://www.healthspan.co.uk/optivision/' },
  { sku: 'JV-VISISOFT', sourceType: 'competitor_pdp', label: 'MacuShield eye health context', competitor: 'MacuShield', url: 'https://www.macushield.com/products/' },

  // Amazon marketplace pages. These may be bot-protected; preserve raw blocked pages if so.
  { sku: 'JV-TURMERIC500', sourceType: 'amazon_search', label: 'Amazon UK turmeric marketplace search', url: 'https://www.amazon.co.uk/s?k=Just+Vitamins+Turmeric+BioPerine' },
  { sku: 'JV-DEFF1000', sourceType: 'amazon_search', label: 'Amazon UK Vitamin D effervescent marketplace search', url: 'https://www.amazon.co.uk/s?k=Just+Vitamins+Vitamin+D+Effervescent+1000iu' },
  { sku: 'JV-VISISOFT', sourceType: 'amazon_search', label: 'Amazon UK Visisoft marketplace search', url: 'https://www.amazon.co.uk/s?k=Just+Vitamins+Visisoft+Lutein' },

  // Trustpilot and Reddit public pages.
  { sku: 'ALL_TOP3', sourceType: 'trustpilot_brand', label: 'Trustpilot Just Vitamins brand reviews', url: 'https://www.trustpilot.com/review/www.justvitamins.co.uk' },
  { sku: 'JV-TURMERIC500', sourceType: 'reddit_category', label: 'Reddit turmeric category search', url: 'https://www.reddit.com/r/Supplements/search.json?q=turmeric%20bioperine&restrict_sr=on&sort=relevance&t=all' },
  { sku: 'JV-DEFF1000', sourceType: 'reddit_category', label: 'Reddit Vitamin D category search', url: 'https://www.reddit.com/r/VitaminD/search.json?q=1000iu%20UK&restrict_sr=on&sort=relevance&t=all' },
  { sku: 'JV-VISISOFT', sourceType: 'reddit_category', label: 'Reddit lutein zeaxanthin category search', url: 'https://www.reddit.com/r/Supplements/search.json?q=lutein%20zeaxanthin&restrict_sr=on&sort=relevance&t=all' }
]

const onlySourceTypes = (Bun.env.APIFY_ONLY_SOURCE_TYPES || '').split(',').map(v => v.trim()).filter(Boolean)
const selectedTargets = onlySourceTypes.length ? targets.filter(target => onlySourceTypes.includes(target.sourceType)) : targets
const maxRequests = Number(Bun.env.APIFY_MAX_REQUESTS || selectedTargets.length)
if (!Number.isFinite(maxRequests) || maxRequests < 1 || maxRequests > selectedTargets.length) {
  throw new Error(`APIFY_MAX_REQUESTS must be between 1 and ${selectedTargets.length}`)
}
if (!token) throw new Error('Missing APIFY_TOKEN or APIFY_API_TOKEN')

function writeJson(path: string, data: any) {
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
}

async function apify(path: string, options: RequestInit = {}) {
  const sep = path.includes('?') ? '&' : '?'
  const res = await fetch(`https://api.apify.com${path}${sep}token=${encodeURIComponent(token!)}`, {
    ...options,
    headers: {
      'Content-Type': 'application/json',
      ...(options.headers || {})
    }
  })
  const text = await res.text()
  let data: any
  try { data = text ? JSON.parse(text) : null } catch { data = text }
  if (!res.ok) throw new Error(`Apify ${res.status} ${res.statusText}: ${typeof data === 'string' ? data : JSON.stringify(data)}`)
  return data
}

const pageFunction = String.raw`
async function pageFunction(context) {
  const { request, $, body, response } = context;
  const clean = (value) => String(value || '').replace(/\s+/g, ' ').trim();
  const text = clean($('body').text()).slice(0, 250000);
  const headings = $('h1,h2,h3').map((_, el) => clean($(el).text())).get().filter(Boolean).slice(0, 300);
  const links = $('a[href]').map((_, el) => ({ text: clean($(el).text()).slice(0, 180), href: $(el).attr('href') })).get().filter(row => row.text || row.href).slice(0, 1000);
  const prices = $('[class*=price], [data-testid*=price], [aria-label*=price]').map((_, el) => clean($(el).text())).get().filter(Boolean).slice(0, 200);
  const reviewLike = $('[class*=review], [data-testid*=review], article, blockquote').map((_, el) => clean($(el).text())).get().filter(Boolean).slice(0, 1000);
  return {
    capturedAt: new Date().toISOString(),
    requestedUrl: request.url,
    loadedUrl: request.loadedUrl || request.url,
    statusCode: response?.statusCode || null,
    sku: request.userData.sku,
    sourceType: request.userData.sourceType,
    label: request.userData.label,
    competitor: request.userData.competitor || null,
    title: clean($('title').first().text()),
    metaDescription: clean($('meta[name="description"]').attr('content')),
    h1: clean($('h1').first().text()),
    headings,
    prices,
    reviewLike,
    links,
    text,
    htmlChars: body ? body.length : 0,
    textChars: text.length
  };
}`

const input = {
  startUrls: selectedTargets.slice(0, maxRequests).map(target => ({ url: target.url, userData: target })),
  maxRequestsPerCrawl: maxRequests,
  maxConcurrency: 2,
  maxRequestRetries: 0,
  requestTimeoutSecs: 45,
  pageFunction,
  proxyConfiguration: { useApifyProxy: true },
  ignoreSslErrors: true
}

const runStart = await apify('/v2/acts/apify~cheerio-scraper/runs', {
  method: 'POST',
  body: JSON.stringify(input)
})
const runId = runStart.data.id
let run = runStart.data
const started = Date.now()
while (!['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(run.status)) {
  if (Date.now() - started > 1000 * 60 * 8) throw new Error(`Timed out waiting for Apify run ${runId}`)
  await new Promise(resolve => setTimeout(resolve, 5000))
  run = (await apify(`/v2/actor-runs/${runId}`)).data
  console.log(`Apify ${runId}: ${run.status} ${run.stats?.requestsFinished || 0}/${maxRequests}`)
}

const datasetId = run.defaultDatasetId
const items = datasetId ? await apify(`/v2/datasets/${datasetId}/items?clean=true`) : []
const result = {
  capturedAt: generatedAt,
  actor: 'apify/cheerio-scraper',
  purpose: 'top3_comprehensive_source_scrape',
  requestCap: maxRequests,
  targetCount: targets.length,
  requestedTargets: selectedTargets.slice(0, maxRequests),
  run: {
    id: run.id,
    status: run.status,
    statusMessage: run.statusMessage,
    defaultDatasetId: run.defaultDatasetId,
    startedAt: run.startedAt,
    finishedAt: run.finishedAt,
    stats: run.stats,
    usageTotalUsd: run.usageTotalUsd,
    usage: run.usage
  },
  itemCount: Array.isArray(items) ? items.length : 0,
  items
}
const stamp = generatedAt.replace(/[:.]/g, '-')
const outputBase = Bun.env.APIFY_OUTPUT_BASENAME || 'top3-comprehensive-source-scrape'
const rawPath = join(rawRoot, `${outputBase}-${stamp}.json`)
const latestPath = join(rawRoot, `${outputBase}-latest.json`)
writeJson(rawPath, result)
writeJson(latestPath, result)
writeJson(join(outRoot, 'top3_apify_comprehensive_scrape.json'), {
  generatedAt,
  ready: run.status === 'SUCCEEDED',
  runId,
  datasetId,
  requestCap: maxRequests,
  targetCount: targets.length,
  itemCount: result.itemCount,
  status: run.status,
  rawPath,
  latestPath,
  coverage: selectedTargets.slice(0, maxRequests).map(target => ({
    sku: target.sku,
    sourceType: target.sourceType,
    label: target.label,
    url: target.url,
    captured: result.items.some((item: any) => item.requestedUrl === target.url || item.label === target.label)
  }))
})
console.log(`Apify run ${runId} ${run.status}: ${result.itemCount}/${maxRequests} items`)
console.log(`Wrote ${latestPath}`)