justvitamin/scripts/generate-dataforseo-pilot.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
const dataforseoRoot = join(root, 'data', 'sources', 'dataforseo')
const asinMapPath = join(dataforseoRoot, 'asin-map.csv')

function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}

function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (value === undefined || value === null) value = ''
    value = Array.isArray(value) ? value.join(' | ') : String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}

function parseCsv(text: string) {
  const lines = text.split(/\r?\n/).filter(Boolean)
  if (!lines.length) return []
  const headers = lines[0].split(',').map(header => header.trim())
  return lines.slice(1).map(line => {
    const values = line.split(',')
    return Object.fromEntries(headers.map((header, index) => [header, (values[index] || '').trim()]))
  })
}

mkdirSync(outRoot, { recursive: true })
mkdirSync(dataforseoRoot, { recursive: true })

const generatedAt = new Date().toISOString()
const acquisition = readJson(join(outRoot, 'source_acquisition_plan.json'), { targets: [] })
const budget = readJson(join(root, 'data', 'sources', 'budgets', 'source-acquisition-budget.json'), {})
const reviewDepth = Math.min(1000, Math.max(100, Number(Bun.env.DATAFORSEO_REVIEWS_DEPTH || budget.dataforseoStage1?.maxReviewsPerAsin || 100)))
const stageOneTargets = (acquisition.targets || []).filter((row: any) => row.Stage === '1' && row.Provider === 'DataForSEO')
const asinRows = existsSync(asinMapPath) ? parseCsv(readFileSync(asinMapPath, 'utf8')) : []
const asinBySku = new Map(asinRows.map((row: any) => [row.SKU, row]))

const intakeRows = stageOneTargets.map((target: any) => {
  const existing: any = asinBySku.get(target.SKU) || {}
  return {
    SKU: target.SKU,
    Product: target.Product,
    Priority: target.Priority,
    Provider: 'DataForSEO',
    AmazonDomain: existing.AmazonDomain || 'amazon.co.uk',
    ASIN: existing.ASIN || '',
    ProductUrl: existing.ProductUrl || '',
    Depth: existing.Depth || String(reviewDepth),
    LocationName: existing.LocationName || 'United Kingdom',
    LanguageName: existing.LanguageName || 'English (United Kingdom)',
    Status: existing.ASIN ? 'ready_for_task_payload' : 'needs_asin'
  }
})

writeFileSync(asinMapPath, csv(intakeRows, ['SKU', 'Product', 'Priority', 'Provider', 'AmazonDomain', 'ASIN', 'ProductUrl', 'Depth', 'LocationName', 'LanguageName', 'Status']), 'utf8')

const readyRows = intakeRows.filter(row => row.ASIN)
const taskPayload = readyRows.map(row => ({
  location_name: row.LocationName,
  language_name: row.LanguageName,
  asin: row.ASIN,
  depth: Number(row.Depth) || 10,
  tag: `jv:${row.SKU}:stage1`
}))

const manifest = {
  generatedAt,
  ready: taskPayload.length > 0,
  endpoint: 'POST https://api.dataforseo.com/v3/merchant/amazon/reviews/task_post',
  docs: [
    'https://docs.dataforseo.com/v3/merchant-amazon-reviews-task_post/',
    'https://dataforseo.com/help-center/get-amazon-reviews'
  ],
  guardrails: {
    maxStageOneTargets: 12,
    recommendedDepth: reviewDepth,
    allowedDepthRange: '100-1000',
    reason: 'Top-3 PDP research needs hundreds of raw marketplace/review items; keep Stage 1 capped by ASIN count, review depth, and explicit allow flag.'
  },
  counts: {
    stageOneTargets: stageOneTargets.length,
    readyTasks: taskPayload.length,
    missingAsins: intakeRows.filter(row => !row.ASIN).length
  },
  files: {
    asinMap: 'data/sources/dataforseo/asin-map.csv',
    taskPayload: 'content_population_exports/dataforseo_amazon_reviews_task_payload.json',
    curlTemplate: 'content_population_exports/dataforseo_amazon_reviews_curl_template.sh',
    outputTemplate: 'data/sources/dataforseo/amazon-review-output-template.json'
  },
  targets: intakeRows
}

writeFileSync(join(outRoot, 'dataforseo_pilot_manifest.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'dataforseo_pilot_targets.csv'), csv(intakeRows, ['SKU', 'Product', 'Priority', 'Provider', 'AmazonDomain', 'ASIN', 'ProductUrl', 'Depth', 'LocationName', 'LanguageName', 'Status']), 'utf8')
writeFileSync(join(outRoot, 'dataforseo_amazon_reviews_task_payload.json'), JSON.stringify(taskPayload, null, 2) + '\n', 'utf8')

const curlTemplate = [
  '#!/usr/bin/env bash',
  'set -euo pipefail',
  '# Fill DATAFORSEO_LOGIN and DATAFORSEO_PASSWORD in your shell, not in this repo.',
  'cred="$(printf "%s:%s" "$DATAFORSEO_LOGIN" "$DATAFORSEO_PASSWORD" | base64)"',
  'curl --location --request POST "https://api.dataforseo.com/v3/merchant/amazon/reviews/task_post" \\',
  '  --header "Authorization: Basic ${cred}" \\',
  '  --header "Content-Type: application/json" \\',
  '  --data-binary @content_population_exports/dataforseo_amazon_reviews_task_payload.json',
  ''
].join('\n')
writeFileSync(join(outRoot, 'dataforseo_amazon_reviews_curl_template.sh'), curlTemplate, 'utf8')

writeFileSync(join(dataforseoRoot, 'amazon-review-output-template.json'), JSON.stringify({
  source: 'amazon',
  provider: 'dataforseo',
  sku: 'JV-D1000',
  asin: 'B000000000',
  productUrl: 'https://www.amazon.co.uk/dp/B000000000',
  capturedAt: generatedAt,
  taskId: '',
  cost: null,
  reviews: [
    {
      review_title: '',
      review_body: '',
      rating: 5,
      date: '',
      helpful_count: 0,
      url: ''
    }
  ]
}, null, 2) + '\n', 'utf8')

const md = [
  '# DataForSEO Stage 1 pilot packet',
  '',
  `Generated: ${generatedAt}`,
  '',
  'This packet prepares the first paid source acquisition stage without spending credits. It does not call DataForSEO.',
  '',
  '## Guardrails',
  `- Stage 1 target cap: ${manifest.guardrails.maxStageOneTargets}`,
  `- Recommended first depth: ${manifest.guardrails.recommendedDepth} reviews per ASIN`,
  '- Stop after the first payload/run and inspect raw JSON quality + cost before increasing toward 1000 reviews per ASIN.',
  '- Store credentials only in shell environment variables; do not commit credentials.',
  '',
  '## Current status',
  `- Stage 1 targets: ${manifest.counts.stageOneTargets}`,
  `- Ready task payload rows: ${manifest.counts.readyTasks}`,
  `- Missing ASINs: ${manifest.counts.missingAsins}`,
  '',
  '## Files',
  `- \`${manifest.files.asinMap}\` - fill ASIN/ProductUrl before task creation.`,
  `- \`${manifest.files.taskPayload}\` - generated DataForSEO task payload; empty until ASINs are filled.`,
  `- \`${manifest.files.curlTemplate}\` - safe curl template using env credentials.`,
  `- \`${manifest.files.outputTemplate}\` - local raw-output shape expected by source validators.`,
  '',
  '## DataForSEO notes',
  '- Amazon reviews task creation requires an ASIN plus location and language parameters.',
  '- Keep `depth` between 100 and 1000 for top-3 research runs; increase only after output quality is proven.',
  ''
].join('\n')
writeFileSync(join(outRoot, 'dataforseo_pilot_plan.md'), md, 'utf8')

console.log(`DataForSEO pilot ready: ${manifest.ready}`)
console.log(`Stage 1 targets: ${manifest.counts.stageOneTargets}`)
console.log(`Ready task payload rows: ${manifest.counts.readyTasks}`)
console.log(`Missing ASINs: ${manifest.counts.missingAsins}`)