justvitamin/scripts/run-dataforseo-reviews-live-pilot.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
const dataforseoRoot = join(root, 'data', 'sources', 'dataforseo')
const rawRoot = join(dataforseoRoot, 'raw')
const defaultEnvPath = 'C:\\Users\\uldvs\\OneDrive\\Desktop\\work\\pi-fresh-foundation\\.env'
const allowToken = 'dataforseo-stage1'
const endpoint = 'POST https://api.dataforseo.com/v3/merchant/amazon/reviews/task_post'

mkdirSync(outRoot, { recursive: true })
mkdirSync(rawRoot, { recursive: true })

function loadEnvFile(path: string) {
  if (!existsSync(path)) return false
  for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) {
    const match = line.match(/^\s*(DATAFORSEO_[A-Za-z0-9_]+)\s*=\s*(.*)\s*$/)
    if (match && !process.env[match[1]]) process.env[match[1]] = match[2].trim().replace(/^['"]|['"]$/g, '')
  }
  return true
}
function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (value === undefined || value === null) value = ''
    value = Array.isArray(value) ? value.join(' | ') : String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}
function authHeaders() {
  const login = process.env.DATAFORSEO_LOGIN
  const password = process.env.DATAFORSEO_PASSWORD
  if (!login || !password) throw new Error('Missing DATAFORSEO_LOGIN/DATAFORSEO_PASSWORD after loading env')
  return {
    Authorization: `Basic ${Buffer.from(`${login}:${password}`).toString('base64')}`,
    'Content-Type': 'application/json'
  }
}
async function sleep(ms: number) {
  await new Promise(resolve => setTimeout(resolve, ms))
}
async function dataforseoFetch(path: string, init: RequestInit = {}) {
  const response = await fetch(`https://api.dataforseo.com${path}`, { ...init, headers: { ...authHeaders(), ...(init.headers || {}) } })
  const body = await response.json()
  return { response, body }
}
function latestArchivedReviewPost() {
  const files = readdirSync(rawRoot, { withFileTypes: true })
    .filter(entry => entry.isFile() && entry.name.includes('reviews-task-post') && entry.name.endsWith('.json'))
    .map(entry => ({ name: entry.name, path: join(rawRoot, entry.name) }))
    .sort((a, b) => a.name.localeCompare(b.name)).reverse()
  for (const file of files) {
    const body = readJson(file.path, null)
    if (body?.status_code) return { file: file.path, status_code: body.status_code, status_message: body.status_message, cost: body.cost, tasks_count: body.tasks_count }
  }
  return null
}
function writeReports(manifest: any) {
  writeFileSync(join(outRoot, 'dataforseo_reviews_live_pilot.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
  writeFileSync(join(outRoot, 'dataforseo_reviews_live_pilot.csv'), csv(manifest.taskSummary || [], ['tag', 'id', 'status_code', 'status_message', 'resultCount', 'rawFile']), 'utf8')
  writeFileSync(join(outRoot, 'dataforseo_reviews_live_pilot.md'), [
    '# DataForSEO live Amazon Reviews pilot',
    '',
    `Generated: ${manifest.generatedAt}`,
    '',
    `Ready: ${manifest.ready ? 'YES' : 'NO'}`,
    '',
    manifest.blocked ? `Blocked: ${manifest.blocker}` : `Status: ${manifest.status || 'recorded'}`,
    '',
    '## Counts',
    `- Payload rows: ${manifest.counts?.payloadRows || 0}`,
    `- Tasks posted: ${manifest.counts?.tasksPosted || 0}`,
    `- Tasks completed: ${manifest.counts?.tasksCompleted || 0}`,
    `- Reviews returned: ${manifest.counts?.reviewsReturned || 0}`,
    '',
    '## Guardrails',
    `- Endpoint: ${endpoint}`,
    `- Required allow flag: \`JV_SOURCE_ALLOW_PAID=${allowToken}\``,
    '- Local task preflight must be ready before this script posts review tasks.',
    '- Raw successful task_get JSON belongs in `data/sources/dataforseo/raw/`, then `bun run dataforseo:ingest` normalizes it.',
    manifest.latestArchivedReviewPost ? '' : '',
    manifest.latestArchivedReviewPost ? '## Latest archived provider failure' : '',
    manifest.latestArchivedReviewPost ? `- ${manifest.latestArchivedReviewPost.status_code}: ${manifest.latestArchivedReviewPost.status_message}` : '',
    manifest.latestArchivedReviewPost ? `- File: ${manifest.latestArchivedReviewPost.file}` : '',
    ''
  ].filter(line => line !== null).join('\n'), 'utf8')
}

const generatedAt = new Date().toISOString()
const envFile = process.env.DATAFORSEO_ENV_FILE || defaultEnvPath
const envLoaded = loadEnvFile(envFile)
const preflight = readJson(join(outRoot, 'dataforseo_task_preflight.json'), {})
const payload = readJson(join(outRoot, 'dataforseo_amazon_reviews_task_payload.json'), [])
const latestFailure = latestArchivedReviewPost()
const explicitAllow = process.env.JV_SOURCE_ALLOW_PAID === allowToken
const preflightReady = preflight.ready === true

const base = {
  generatedAt,
  ready: false,
  blocked: true,
  endpoint,
  credentialSource: envFile,
  credentialPrinted: false,
  credentialEnvLoaded: envLoaded,
  allowToken: `JV_SOURCE_ALLOW_PAID=${allowToken}`,
  counts: { payloadRows: Array.isArray(payload) ? payload.length : 0, tasksPosted: 0, tasksCompleted: 0, reviewsReturned: 0 },
  files: {
    payload: 'content_population_exports/dataforseo_amazon_reviews_task_payload.json',
    preflight: 'content_population_exports/dataforseo_task_preflight.json',
    manifest: 'content_population_exports/dataforseo_reviews_live_pilot.json',
    report: 'content_population_exports/dataforseo_reviews_live_pilot.md',
    rawDir: 'data/sources/dataforseo/raw'
  },
  latestArchivedReviewPost: latestFailure
}

if (!explicitAllow) {
  const manifest = { ...base, blocker: `Set JV_SOURCE_ALLOW_PAID=${allowToken} only for the approved paid pilot run.` }
  writeReports(manifest)
  console.log('DataForSEO reviews pilot blocked: explicit allow flag missing')
  console.log(manifest.blocker)
  process.exit(0)
}
if (!preflightReady) {
  const manifest = { ...base, blocker: `Task preflight is not ready (${preflight.counts?.passing || 0}/${preflight.counts?.checks || 0} checks passing). Do not post review tasks.`, preflightBlockers: preflight.blockers || [] }
  writeReports(manifest)
  console.log('DataForSEO reviews pilot blocked: preflight not ready')
  console.log(manifest.blocker)
  process.exit(0)
}
if (!Array.isArray(payload) || payload.length === 0) {
  const manifest = { ...base, blocker: 'Review task payload is empty.' }
  writeReports(manifest)
  console.log('DataForSEO reviews pilot blocked: empty payload')
  process.exit(0)
}

const account = await dataforseoFetch('/v3/appendix/user_data')
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)
const post = await dataforseoFetch('/v3/merchant/amazon/reviews/task_post', { method: 'POST', body: JSON.stringify(payload) })
const postPath = join(rawRoot, `reviews-task-post-live-${stamp}.json`)
writeFileSync(postPath, JSON.stringify(post.body, null, 2) + '\n', 'utf8')
const ids = (post.body.tasks || []).filter((task: any) => task.id).map((task: any) => ({ id: task.id, tag: task.data?.tag, asin: task.data?.asin, status_code: task.status_code, status_message: task.status_message }))
writeFileSync(join(rawRoot, 'latest-reviews-task-ids-live.json'), JSON.stringify(ids, null, 2) + '\n', 'utf8')

const maxAttempts = Number(process.env.DATAFORSEO_REVIEWS_POLL_ATTEMPTS || 12)
const pollSeconds = Number(process.env.DATAFORSEO_REVIEWS_POLL_SECONDS || 10)
let taskSummary: any[] = []
for (let attempt = 1; attempt <= maxAttempts && ids.length; attempt += 1) {
  taskSummary = []
  for (const task of ids) {
    const get = await dataforseoFetch(`/v3/merchant/amazon/reviews/task_get/advanced/${task.id}`)
    const file = join(rawRoot, `reviews-task-get-${String(task.tag).replace(/:/g, '-')}-${task.id}.json`)
    writeFileSync(file, JSON.stringify(get.body, null, 2) + '\n', 'utf8')
    const firstTask = get.body.tasks?.[0]
    const items = firstTask?.result?.[0]?.items || []
    taskSummary.push({ tag: task.tag, id: task.id, status_code: firstTask?.status_code, status_message: firstTask?.status_message, resultCount: items.length, rawFile: file })
  }
  if (taskSummary.every(row => row.resultCount > 0 || row.status_code === 20000)) break
  await sleep(pollSeconds * 1000)
}
const reviewsReturned = taskSummary.reduce((sum, row) => sum + Number(row.resultCount || 0), 0)
const manifest = {
  ...base,
  blocked: ids.length === 0,
  ready: ids.length > 0 && reviewsReturned > 0,
  status: ids.length > 0 ? 'posted_and_polled' : 'provider_returned_no_task_ids',
  blocker: ids.length === 0 ? `${post.body.status_code}: ${post.body.status_message}` : (reviewsReturned > 0 ? '' : 'Tasks posted but no review items returned yet.'),
  accountConnection: { status_code: account.body.status_code, status_message: account.body.status_message, httpStatus: account.response.status },
  post: { status_code: post.body.status_code, status_message: post.body.status_message, cost: post.body.cost, tasks_count: post.body.tasks_count, file: postPath },
  counts: { payloadRows: payload.length, tasksPosted: ids.length, tasksCompleted: taskSummary.filter(row => row.status_code === 20000).length, reviewsReturned },
  taskIds: ids,
  taskSummary
}
writeReports(manifest)
console.log(`DataForSEO reviews pilot ready: ${manifest.ready}`)
console.log(`Post status: ${post.body.status_code} ${post.body.status_message}`)
console.log(`Tasks posted: ${manifest.counts.tasksPosted}`)
console.log(`Reviews returned: ${manifest.counts.reviewsReturned}`)