#!/usr/bin/env bun import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'fs' import { join } from 'path' const root = process.cwd() const outRoot = join(root, 'content_population_exports') const dataforseoRoot = join(root, 'data', 'sources', 'dataforseo') const rawRoot = join(dataforseoRoot, 'raw') const defaultEnvPath = 'C:\\Users\\uldvs\\OneDrive\\Desktop\\work\\pi-fresh-foundation\\.env' const allowToken = 'dataforseo-stage1' const endpoint = 'POST https://api.dataforseo.com/v3/merchant/amazon/reviews/task_post' mkdirSync(outRoot, { recursive: true }) mkdirSync(rawRoot, { recursive: true }) function loadEnvFile(path: string) { if (!existsSync(path)) return false for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) { const match = line.match(/^\s*(DATAFORSEO_[A-Za-z0-9_]+)\s*=\s*(.*)\s*$/) if (match && !process.env[match[1]]) process.env[match[1]] = match[2].trim().replace(/^['"]|['"]$/g, '') } return true } function readJson(path: string, fallback: any = null) { if (!existsSync(path)) return fallback return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, '')) } function csv(rows: any[], headers: string[]) { const esc = (value: any) => { if (value === undefined || value === null) value = '' value = Array.isArray(value) ? value.join(' | ') : String(value) return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value } return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n' } function authHeaders() { const login = process.env.DATAFORSEO_LOGIN const password = process.env.DATAFORSEO_PASSWORD if (!login || !password) throw new Error('Missing DATAFORSEO_LOGIN/DATAFORSEO_PASSWORD after loading env') return { Authorization: `Basic ${Buffer.from(`${login}:${password}`).toString('base64')}`, 'Content-Type': 'application/json' } } async function sleep(ms: number) { await new Promise(resolve => setTimeout(resolve, ms)) } async function dataforseoFetch(path: string, init: RequestInit = {}) { const response = await fetch(`https://api.dataforseo.com${path}`, { ...init, headers: { ...authHeaders(), ...(init.headers || {}) } }) const body = await response.json() return { response, body } } function latestArchivedReviewPost() { const files = readdirSync(rawRoot, { withFileTypes: true }) .filter(entry => entry.isFile() && entry.name.includes('reviews-task-post') && entry.name.endsWith('.json')) .map(entry => ({ name: entry.name, path: join(rawRoot, entry.name) })) .sort((a, b) => a.name.localeCompare(b.name)).reverse() for (const file of files) { const body = readJson(file.path, null) if (body?.status_code) return { file: file.path, status_code: body.status_code, status_message: body.status_message, cost: body.cost, tasks_count: body.tasks_count } } return null } function writeReports(manifest: any) { writeFileSync(join(outRoot, 'dataforseo_reviews_live_pilot.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8') writeFileSync(join(outRoot, 'dataforseo_reviews_live_pilot.csv'), csv(manifest.taskSummary || [], ['tag', 'id', 'status_code', 'status_message', 'resultCount', 'rawFile']), 'utf8') writeFileSync(join(outRoot, 'dataforseo_reviews_live_pilot.md'), [ '# DataForSEO live Amazon Reviews pilot', '', `Generated: ${manifest.generatedAt}`, '', `Ready: ${manifest.ready ? 'YES' : 'NO'}`, '', manifest.blocked ? `Blocked: ${manifest.blocker}` : `Status: ${manifest.status || 'recorded'}`, '', '## Counts', `- Payload rows: ${manifest.counts?.payloadRows || 0}`, `- Tasks posted: ${manifest.counts?.tasksPosted || 0}`, `- Tasks completed: ${manifest.counts?.tasksCompleted || 0}`, `- Reviews returned: ${manifest.counts?.reviewsReturned || 0}`, '', '## Guardrails', `- Endpoint: ${endpoint}`, `- Required allow flag: \`JV_SOURCE_ALLOW_PAID=${allowToken}\``, '- Local task preflight must be ready before this script posts review tasks.', '- Raw successful task_get JSON belongs in `data/sources/dataforseo/raw/`, then `bun run dataforseo:ingest` normalizes it.', manifest.latestArchivedReviewPost ? '' : '', manifest.latestArchivedReviewPost ? '## Latest archived provider failure' : '', manifest.latestArchivedReviewPost ? `- ${manifest.latestArchivedReviewPost.status_code}: ${manifest.latestArchivedReviewPost.status_message}` : '', manifest.latestArchivedReviewPost ? `- File: ${manifest.latestArchivedReviewPost.file}` : '', '' ].filter(line => line !== null).join('\n'), 'utf8') } const generatedAt = new Date().toISOString() const envFile = process.env.DATAFORSEO_ENV_FILE || defaultEnvPath const envLoaded = loadEnvFile(envFile) const preflight = readJson(join(outRoot, 'dataforseo_task_preflight.json'), {}) const payload = readJson(join(outRoot, 'dataforseo_amazon_reviews_task_payload.json'), []) const latestFailure = latestArchivedReviewPost() const explicitAllow = process.env.JV_SOURCE_ALLOW_PAID === allowToken const preflightReady = preflight.ready === true const base = { generatedAt, ready: false, blocked: true, endpoint, credentialSource: envFile, credentialPrinted: false, credentialEnvLoaded: envLoaded, allowToken: `JV_SOURCE_ALLOW_PAID=${allowToken}`, counts: { payloadRows: Array.isArray(payload) ? payload.length : 0, tasksPosted: 0, tasksCompleted: 0, reviewsReturned: 0 }, files: { payload: 'content_population_exports/dataforseo_amazon_reviews_task_payload.json', preflight: 'content_population_exports/dataforseo_task_preflight.json', manifest: 'content_population_exports/dataforseo_reviews_live_pilot.json', report: 'content_population_exports/dataforseo_reviews_live_pilot.md', rawDir: 'data/sources/dataforseo/raw' }, latestArchivedReviewPost: latestFailure } if (!explicitAllow) { const manifest = { ...base, blocker: `Set JV_SOURCE_ALLOW_PAID=${allowToken} only for the approved paid pilot run.` } writeReports(manifest) console.log('DataForSEO reviews pilot blocked: explicit allow flag missing') console.log(manifest.blocker) process.exit(0) } if (!preflightReady) { const manifest = { ...base, blocker: `Task preflight is not ready (${preflight.counts?.passing || 0}/${preflight.counts?.checks || 0} checks passing). Do not post review tasks.`, preflightBlockers: preflight.blockers || [] } writeReports(manifest) console.log('DataForSEO reviews pilot blocked: preflight not ready') console.log(manifest.blocker) process.exit(0) } if (!Array.isArray(payload) || payload.length === 0) { const manifest = { ...base, blocker: 'Review task payload is empty.' } writeReports(manifest) console.log('DataForSEO reviews pilot blocked: empty payload') process.exit(0) } const account = await dataforseoFetch('/v3/appendix/user_data') const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19) const post = await dataforseoFetch('/v3/merchant/amazon/reviews/task_post', { method: 'POST', body: JSON.stringify(payload) }) const postPath = join(rawRoot, `reviews-task-post-live-${stamp}.json`) writeFileSync(postPath, JSON.stringify(post.body, null, 2) + '\n', 'utf8') const ids = (post.body.tasks || []).filter((task: any) => task.id).map((task: any) => ({ id: task.id, tag: task.data?.tag, asin: task.data?.asin, status_code: task.status_code, status_message: task.status_message })) writeFileSync(join(rawRoot, 'latest-reviews-task-ids-live.json'), JSON.stringify(ids, null, 2) + '\n', 'utf8') const maxAttempts = Number(process.env.DATAFORSEO_REVIEWS_POLL_ATTEMPTS || 12) const pollSeconds = Number(process.env.DATAFORSEO_REVIEWS_POLL_SECONDS || 10) let taskSummary: any[] = [] for (let attempt = 1; attempt <= maxAttempts && ids.length; attempt += 1) { taskSummary = [] for (const task of ids) { const get = await dataforseoFetch(`/v3/merchant/amazon/reviews/task_get/advanced/${task.id}`) const file = join(rawRoot, `reviews-task-get-${String(task.tag).replace(/:/g, '-')}-${task.id}.json`) writeFileSync(file, JSON.stringify(get.body, null, 2) + '\n', 'utf8') const firstTask = get.body.tasks?.[0] const items = firstTask?.result?.[0]?.items || [] taskSummary.push({ tag: task.tag, id: task.id, status_code: firstTask?.status_code, status_message: firstTask?.status_message, resultCount: items.length, rawFile: file }) } if (taskSummary.every(row => row.resultCount > 0 || row.status_code === 20000)) break await sleep(pollSeconds * 1000) } const reviewsReturned = taskSummary.reduce((sum, row) => sum + Number(row.resultCount || 0), 0) const manifest = { ...base, blocked: ids.length === 0, ready: ids.length > 0 && reviewsReturned > 0, status: ids.length > 0 ? 'posted_and_polled' : 'provider_returned_no_task_ids', blocker: ids.length === 0 ? `${post.body.status_code}: ${post.body.status_message}` : (reviewsReturned > 0 ? '' : 'Tasks posted but no review items returned yet.'), accountConnection: { status_code: account.body.status_code, status_message: account.body.status_message, httpStatus: account.response.status }, post: { status_code: post.body.status_code, status_message: post.body.status_message, cost: post.body.cost, tasks_count: post.body.tasks_count, file: postPath }, counts: { payloadRows: payload.length, tasksPosted: ids.length, tasksCompleted: taskSummary.filter(row => row.status_code === 20000).length, reviewsReturned }, taskIds: ids, taskSummary } writeReports(manifest) console.log(`DataForSEO reviews pilot ready: ${manifest.ready}`) console.log(`Post status: ${post.body.status_code} ${post.body.status_message}`) console.log(`Tasks posted: ${manifest.counts.tasksPosted}`) console.log(`Reviews returned: ${manifest.counts.reviewsReturned}`)