justvitamin/scripts/run-apify-top3-specialist-scrapes.ts

#!/usr/bin/env bun
import { mkdirSync, writeFileSync } from 'fs'
import { dirname, join } from 'path'

const root = process.cwd()
const rawRoot = join(root, 'data', 'sources', 'apify', 'raw')
const outRoot = join(root, 'content_population_exports')
const token = Bun.env.APIFY_TOKEN || Bun.env.APIFY_API_TOKEN
const generatedAt = new Date().toISOString()
if (!token) throw new Error('Missing APIFY_TOKEN or APIFY_API_TOKEN')
const outputSuffix = Bun.env.APIFY_SPECIALIST_OUTPUT_SUFFIX ? `-${Bun.env.APIFY_SPECIALIST_OUTPUT_SUFFIX.replace(/[^a-z0-9_-]/gi, '-')}` : ''
const requestedScale = Math.min(1000, Math.max(100, Number(Bun.env.TOP3_SCRAPE_ITEMS_PER_SOURCE || 300)))

type Job = {
  key: string
  actor: string
  maxTotalChargeUsd: number
  input: Record<string, any>
  note: string
}

const allJobs: Job[] = [
  {
    key: 'trustpilot',
    actor: 'zen-studio~trustpilot-review-scraper',
    maxTotalChargeUsd: Number(Bun.env.APIFY_TRUSTPILOT_MAX_USD || 2.5),
    note: `Brand-level Just Vitamins Trustpilot review capture capped to ${requestedScale} reviews.`,
    input: {
      businessUrl: 'https://www.trustpilot.com/review/www.justvitamins.co.uk',
      maxResults: Number(Bun.env.APIFY_TRUSTPILOT_MAX_RESULTS || requestedScale),
      sort: 'recency',
      dateRange: 'last12months',
      language: 'en',
      verifiedOnly: false,
      repliesOnly: false
    }
  },
  {
    key: 'reddit',
    actor: 'trudax~reddit-scraper-lite',
    maxTotalChargeUsd: Number(Bun.env.APIFY_REDDIT_MAX_USD || 2.5),
    note: `Category-language Reddit search capture capped to ${requestedScale} stored items across top-3 category searches and comments disabled by default.`,
    input: {
      searches: [
        ...(Bun.env.APIFY_REDDIT_SEARCHES
          ? Bun.env.APIFY_REDDIT_SEARCHES.split('|').map(item => item.trim()).filter(Boolean)
          : ['turmeric bioperine', 'vitamin d 1000iu UK', 'lutein zeaxanthin'])
      ],
      searchPosts: true,
      searchComments: false,
      searchCommunities: false,
      searchUsers: false,
      sort: 'relevance',
      time: 'all',
      includeNSFW: false,
      maxItems: Number(Bun.env.APIFY_REDDIT_MAX_ITEMS || requestedScale),
      maxPostCount: Number(Bun.env.APIFY_REDDIT_MAX_POST_COUNT || requestedScale),
      maxComments: Number(Bun.env.APIFY_REDDIT_MAX_COMMENTS || 0),
      skipComments: true,
      skipCommunity: true,
      proxy: { useApifyProxy: true }
    }
  },
  {
    key: 'amazon-reviews',
    actor: 'web_wanderer~amazon-reviews-extractor',
    maxTotalChargeUsd: Number(Bun.env.APIFY_AMAZON_MAX_USD || 5),
    note: 'Amazon review text capture for exact confirmed ASINs only; increase products/ASIN map before broad paid runs.',
    input: {
      personal_data: false,
      products: [
        'B009756A58',
        'B009756BHA'
      ],
      limit: Number(Bun.env.APIFY_AMAZON_PAGES_PER_PRODUCT || 10),
      sort: 'recent',
      rating: 'all',
      all_stars: false,
      avp_reviews: false,
      include_variants: true,
      region: 'amazon.co.uk',
      language: 'en'
    }
  }
]

const requestedKeys = (Bun.env.APIFY_SPECIALIST_JOBS || allJobs.map(j => j.key).join(','))
  .split(',')
  .map(v => v.trim())
  .filter(Boolean)
const jobs = allJobs.filter(job => requestedKeys.includes(job.key))
if (!jobs.length) throw new Error(`No jobs selected. Known jobs: ${allJobs.map(j => j.key).join(', ')}`)

function writeJson(path: string, data: any) {
  mkdirSync(dirname(path), { recursive: true })
  writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
}
async function apify(path: string, options: RequestInit = {}) {
  const sep = path.includes('?') ? '&' : '?'
  const res = await fetch(`https://api.apify.com${path}${sep}token=${encodeURIComponent(token!)}`, {
    ...options,
    headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
  })
  const text = await res.text()
  let data: any
  try { data = text ? JSON.parse(text) : null } catch { data = text }
  if (!res.ok) throw new Error(`Apify ${res.status} ${res.statusText}: ${typeof data === 'string' ? data : JSON.stringify(data)}`)
  return data
}
async function waitForRun(runId: string, timeoutMs = 1000 * 60 * 10) {
  const started = Date.now()
  let run = (await apify(`/v2/actor-runs/${runId}`)).data
  while (!['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(run.status)) {
    if (Date.now() - started > timeoutMs) throw new Error(`Timed out waiting for ${runId}`)
    await new Promise(resolve => setTimeout(resolve, 5000))
    run = (await apify(`/v2/actor-runs/${runId}`)).data
    console.log(`${runId}: ${run.status} usd=${run.usageTotalUsd || 0}`)
  }
  return run
}

const results = []
for (const job of jobs) {
  console.log(`Starting ${job.key} via ${job.actor} with cap $${job.maxTotalChargeUsd}`)
  const start = await apify(`/v2/acts/${job.actor}/runs?maxTotalChargeUsd=${job.maxTotalChargeUsd}`, {
    method: 'POST',
    body: JSON.stringify(job.input)
  })
  const run = await waitForRun(start.data.id)
  const datasetId = run.defaultDatasetId
  const items = datasetId ? await apify(`/v2/datasets/${datasetId}/items?clean=true`) : []
  const result = {
    capturedAt: generatedAt,
    job: job.key,
    actor: job.actor,
    note: job.note,
    maxTotalChargeUsd: job.maxTotalChargeUsd,
    input: job.input,
    run: {
      id: run.id,
      status: run.status,
      statusMessage: run.statusMessage,
      defaultDatasetId: run.defaultDatasetId,
      startedAt: run.startedAt,
      finishedAt: run.finishedAt,
      usageTotalUsd: run.usageTotalUsd,
      usage: run.usage,
      stats: run.stats,
      chargedEventCounts: run.chargedEventCounts
    },
    itemCount: Array.isArray(items) ? items.length : 0,
    items
  }
  const stamp = generatedAt.replace(/[:.]/g, '-')
  const rawPath = join(rawRoot, `top3-specialist-${job.key}${outputSuffix}-${stamp}.json`)
  const latestPath = join(rawRoot, `top3-specialist-${job.key}${outputSuffix}-latest.json`)
  writeJson(rawPath, result)
  writeJson(latestPath, result)
  results.push({ ...result, rawPath, latestPath, items: undefined })
  console.log(`${job.key}: ${run.status}, items=${result.itemCount}, usd=${run.usageTotalUsd || 0}`)
}

const manifest = {
  generatedAt,
  ready: results.every(row => row.run.status === 'SUCCEEDED'),
  jobs: results.map(row => ({
    job: row.job,
    actor: row.actor,
    status: row.run.status,
    itemCount: row.itemCount,
    usageTotalUsd: row.run.usageTotalUsd,
    maxTotalChargeUsd: row.maxTotalChargeUsd,
    rawPath: row.rawPath,
    latestPath: row.latestPath
  }))
}
writeJson(join(outRoot, 'top3_apify_specialist_scrapes.json'), manifest)
console.log(`Specialist Apify jobs completed: ${results.length}`)