justvitamin/scripts/run-apify-collagen-starter-scrape.ts

#!/usr/bin/env bun
import { mkdirSync, writeFileSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const rawRoot = join(root, 'data', 'sources', 'apify', 'raw')
const outRoot = join(root, 'content_population_exports')
const token = Bun.env.APIFY_TOKEN || Bun.env.APIFY_API_TOKEN
const generatedAt = new Date().toISOString()
if (!token) throw new Error('Missing APIFY_TOKEN or APIFY_API_TOKEN')

const totalMaxUsd = Number(Bun.env.COLLAGEN_APIFY_TOTAL_MAX_USD || 4.5)
const runTimeoutMs = Number(Bun.env.COLLAGEN_APIFY_RUN_TIMEOUT_MS || 240000)

type Job = {
  key: string
  actor: string
  maxTotalChargeUsd: number
  input: Record<string, any>
  note: string
}

const amazonAsins = [
  { asin: 'B08W1SXHDJ', label: 'Reverse Life mixed berry collagen liquid', url: 'https://www.amazon.co.uk/dp/B08W1SXHDJ' },
  { asin: 'B0B2KQV768', label: 'JSHealth Vitamins Vitality X collagen powder', url: 'https://www.amazon.co.uk/dp/B0B2KQV768' },
  { asin: 'B07TJ99QBC', label: 'Vital Proteins marine collagen unflavoured powder', url: 'https://www.amazon.co.uk/dp/B07TJ99QBC' },
  { asin: 'B0CQCX5HF3', label: 'Applied Nutrition marine collagen strawberry raspberry powder', url: 'https://www.amazon.co.uk/dp/B0CQCX5HF3' },
  { asin: 'B0CQCVBPC3', label: 'Applied Nutrition marine collagen cherry apple powder', url: 'https://www.amazon.co.uk/dp/B0CQCVBPC3' },
  { asin: 'B07J65XBVS', label: 'Hunter & Gather bovine collagen unflavoured powder', url: 'https://www.amazon.co.uk/dp/B07J65XBVS' },
  { asin: 'B0DDQ69DJ9', label: 'Nourish Ninjas marine collagen unflavoured powder', url: 'https://www.amazon.co.uk/dp/B0DDQ69DJ9' },
  { asin: 'B0C42H2WP4', label: 'Vital Proteins collagen peptides unflavoured travel packs', url: 'https://www.amazon.co.uk/dp/B0C42H2WP4' }
]

const jobs: Job[] = [
  {
    key: 'collagen-amazon-reviews',
    actor: 'junglee~amazon-reviews-scraper',
    maxTotalChargeUsd: Number(Bun.env.COLLAGEN_APIFY_AMAZON_MAX_USD || 1.5),
    note: 'Collagen NPD starter Amazon review capture for DataForSEO-seeded UK competitor ASINs only.',
    input: {
      productUrls: amazonAsins.map(item => ({ url: item.url })),
      maxReviews: Number(Bun.env.COLLAGEN_AMAZON_MAX_REVIEWS || 25),
      includeGdprSensitive: false,
      sort: 'recent',
      filterByRatings: ['allStars'],
      reviewsUseProductVariantFilter: false,
      scrapeProductDetails: false,
      reviewsAlwaysSaveCategoryData: true,
      deduplicateRedirectedAsins: true
    }
  },
  {
    key: 'collagen-reddit',
    actor: 'trudax~reddit-scraper-lite',
    maxTotalChargeUsd: Number(Bun.env.COLLAGEN_APIFY_REDDIT_MAX_USD || 1.25),
    note: 'Collagen NPD Reddit starter capture focused on taste, mixability, smell, unflavoured-in-coffee and UK buying language.',
    input: {
      searches: [
        'collagen powder taste fishy',
        'unflavoured collagen coffee',
        'collagen powder clumpy chalky',
        'best collagen powder UK taste',
        'marine collagen aftertaste',
        'collagen powder vanilla chocolate berry'
      ],
      searchPosts: true,
      searchComments: false,
      searchCommunities: false,
      searchUsers: false,
      sort: 'relevance',
      time: 'year',
      includeNSFW: false,
      maxItems: Number(Bun.env.COLLAGEN_REDDIT_MAX_ITEMS || 80),
      maxPostCount: Number(Bun.env.COLLAGEN_REDDIT_MAX_POST_COUNT || 80),
      maxComments: 0,
      skipComments: true,
      skipCommunity: true,
      proxy: { useApifyProxy: true }
    }
  },
  {
    key: 'collagen-trustpilot-feel',
    actor: 'zen-studio~trustpilot-review-scraper',
    maxTotalChargeUsd: Number(Bun.env.COLLAGEN_APIFY_TRUSTPILOT_MAX_USD || 0.75),
    note: 'Feel Trustpilot starter sample for subscription, delivery, taste and repeat-use language.',
    input: {
      businessUrl: 'https://www.trustpilot.com/review/wearefeel.com',
      maxResults: Number(Bun.env.COLLAGEN_TRUSTPILOT_MAX_RESULTS || 50),
      sort: 'recency',
      dateRange: 'last12months',
      language: 'en',
      verifiedOnly: false,
      repliesOnly: false
    }
  },
  {
    key: 'collagen-trustpilot-ancient-and-brave',
    actor: 'zen-studio~trustpilot-review-scraper',
    maxTotalChargeUsd: Number(Bun.env.COLLAGEN_APIFY_TRUSTPILOT2_MAX_USD || 0.75),
    note: 'Ancient + Brave Trustpilot starter sample for premium ritual, subscription and taste language.',
    input: {
      businessUrl: 'https://www.trustpilot.com/review/ancientandbrave.earth',
      maxResults: Number(Bun.env.COLLAGEN_TRUSTPILOT2_MAX_RESULTS || 50),
      sort: 'recency',
      dateRange: 'last12months',
      language: 'en',
      verifiedOnly: false,
      repliesOnly: false
    }
  }
]

const selectedKeys = (Bun.env.COLLAGEN_APIFY_JOBS || jobs.map(job => job.key).join(','))
  .split(',')
  .map(value => value.trim())
  .filter(Boolean)
const selected = jobs.filter(job => selectedKeys.includes(job.key))
const configuredMax = selected.reduce((sum, job) => sum + job.maxTotalChargeUsd, 0)
if (!selected.length) throw new Error('No collagen Apify jobs selected')
if (configuredMax > totalMaxUsd) throw new Error(`Configured job caps $${configuredMax.toFixed(2)} exceed total cap $${totalMaxUsd.toFixed(2)}`)

function writeJson(filePath: string, data: any) {
  mkdirSync(dirname(filePath), { recursive: true })
  writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n', 'utf8')
}

async function apify(path: string, options: RequestInit = {}) {
  const sep = path.includes('?') ? '&' : '?'
  const res = await fetch(`https://api.apify.com${path}${sep}token=${encodeURIComponent(token!)}`, {
    ...options,
    headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
  })
  const text = await res.text()
  let data: any
  try { data = text ? JSON.parse(text) : null } catch { data = text }
  if (!res.ok) throw new Error(`Apify ${res.status} ${res.statusText}: ${typeof data === 'string' ? data : JSON.stringify(data)}`)
  return data
}

async function abortRun(runId: string) {
  try { return (await apify(`/v2/actor-runs/${runId}/abort`, { method: 'POST' })).data } catch { return null }
}

async function waitForRun(runId: string) {
  const started = Date.now()
  let run = (await apify(`/v2/actor-runs/${runId}`)).data
  while (!['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(run.status)) {
    if (Date.now() - started > runTimeoutMs) {
      console.log(`${runId}: timeout reached, aborting to preserve cap`)
      await abortRun(runId)
      run = (await apify(`/v2/actor-runs/${runId}`)).data
      break
    }
    await new Promise(resolve => setTimeout(resolve, 5000))
    run = (await apify(`/v2/actor-runs/${runId}`)).data
    console.log(`${runId}: ${run.status} usd=${run.usageTotalUsd || 0}`)
  }
  return run
}

function annotateItems(job: Job, items: any[]) {
  if (job.key === 'collagen-amazon-reviews') {
    return items.map(item => {
      const raw = JSON.stringify(item)
      const match = amazonAsins.find(row => raw.includes(row.asin) || raw.includes(row.url))
      return { ...item, sku: 'COLLAGEN-NPD', collagenSource: 'amazon_review', matchedAsin: match?.asin || item.productAsin || item.asin, competitorLabel: match?.label || null }
    })
  }
  if (job.key.includes('trustpilot')) {
    const competitorLabel = job.key.includes('feel') ? 'Feel' : 'Ancient & Brave'
    return items.map(item => ({ ...item, sku: 'COLLAGEN-NPD', collagenSource: 'trustpilot_review', competitorLabel }))
  }
  if (job.key === 'collagen-reddit') {
    return items.map(item => ({ ...item, sku: 'COLLAGEN-NPD', collagenSource: 'reddit_thread' }))
  }
  return items.map(item => ({ ...item, sku: 'COLLAGEN-NPD' }))
}

const results = []
for (const job of selected) {
  console.log(`Starting ${job.key} via ${job.actor} cap=$${job.maxTotalChargeUsd}`)
  const start = await apify(`/v2/acts/${job.actor}/runs?maxTotalChargeUsd=${job.maxTotalChargeUsd}`, {
    method: 'POST',
    body: JSON.stringify(job.input)
  })
  const run = await waitForRun(start.data.id)
  const rawItems = run.defaultDatasetId ? await apify(`/v2/datasets/${run.defaultDatasetId}/items?clean=true`) : []
  const items = Array.isArray(rawItems) ? annotateItems(job, rawItems) : rawItems
  const result = {
    capturedAt: generatedAt,
    sku: 'COLLAGEN-NPD',
    job: job.key,
    actor: job.actor,
    note: job.note,
    maxTotalChargeUsd: job.maxTotalChargeUsd,
    input: job.input,
    run: {
      id: run.id,
      status: run.status,
      statusMessage: run.statusMessage,
      defaultDatasetId: run.defaultDatasetId,
      startedAt: run.startedAt,
      finishedAt: run.finishedAt,
      usageTotalUsd: run.usageTotalUsd,
      usage: run.usage,
      stats: run.stats,
      chargedEventCounts: run.chargedEventCounts
    },
    itemCount: Array.isArray(items) ? items.length : 0,
    items
  }
  const stamp = generatedAt.replace(/[:.]/g, '-')
  const rawPath = join(rawRoot, `top3-specialist-${job.key}-${stamp}.json`)
  const latestPath = join(rawRoot, `top3-specialist-${job.key}-latest.json`)
  writeJson(rawPath, result)
  writeJson(latestPath, result)
  results.push({ ...result, rawPath, latestPath, items: undefined })
  console.log(`${job.key}: ${run.status}, items=${result.itemCount}, usd=${run.usageTotalUsd || 0}`)
}

const totalUsd = results.reduce((sum, row: any) => sum + Number(row.run.usageTotalUsd || 0), 0)
const manifest = {
  generatedAt,
  sku: 'COLLAGEN-NPD',
  configuredMaxUsd: configuredMax,
  totalMaxUsd,
  totalUsageUsd: totalUsd,
  ready: results.every((row: any) => ['SUCCEEDED', 'ABORTED'].includes(row.run.status)) && totalUsd <= totalMaxUsd,
  jobs: results.map((row: any) => ({
    job: row.job,
    actor: row.actor,
    status: row.run.status,
    itemCount: row.itemCount,
    usageTotalUsd: row.run.usageTotalUsd,
    maxTotalChargeUsd: row.maxTotalChargeUsd,
    rawPath: row.rawPath,
    latestPath: row.latestPath
  }))
}
writeJson(join(outRoot, 'collagen_apify_starter_scrape.json'), manifest)
console.log(`Collagen starter Apify scrape complete. usage=$${totalUsd.toFixed(4)} configuredCap=$${configuredMax.toFixed(2)}`)