justvitamin/scripts/run-dataforseo-products-live-search.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const outRoot = join(root, 'content_population_exports')
const productsRawRoot = join(root, 'data', 'sources', 'dataforseo', 'products_raw')
const defaultEnvPath = 'C:\\Users\\uldvs\\OneDrive\\Desktop\\work\\pi-fresh-foundation\\.env'
const allowToken = 'dataforseo-product-search'

mkdirSync(outRoot, { recursive: true })
mkdirSync(productsRawRoot, { recursive: true })

function loadEnvFile(path: string) {
  if (!existsSync(path)) return false
  for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) {
    const match = line.match(/^\s*(DATAFORSEO_[A-Za-z0-9_]+)\s*=\s*(.*)\s*$/)
    if (match) process.env[match[1]] = match[2].trim().replace(/^['"]|['"]$/g, '')
  }
  return true
}
function readJson(path: string, fallback: any = null) {
  if (!existsSync(path)) return fallback
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (value === undefined || value === null) value = ''
    value = Array.isArray(value) ? value.join(' | ') : String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}
function parseCsv(text: string) {
  const rows: string[][] = []
  let row: string[] = []
  let cell = ''
  let quoted = false
  for (let i = 0; i < text.length; i += 1) {
    const ch = text[i]
    const next = text[i + 1]
    if (quoted) {
      if (ch === '"' && next === '"') { cell += '"'; i += 1 }
      else if (ch === '"') quoted = false
      else cell += ch
    } else if (ch === '"') quoted = true
    else if (ch === ',') { row.push(cell); cell = '' }
    else if (ch === '\n') { row.push(cell); rows.push(row); row = []; cell = '' }
    else if (ch !== '\r') cell += ch
  }
  if (cell || row.length) { row.push(cell); rows.push(row) }
  const headers = rows.shift()?.map(header => header.trim()) || []
  return rows.filter(values => values.some(value => value.trim())).map(values => Object.fromEntries(headers.map((header, index) => [header, (values[index] || '').trim()])))
}
function authHeaders() {
  const login = process.env.DATAFORSEO_LOGIN
  const password = process.env.DATAFORSEO_PASSWORD
  if (!login || !password) throw new Error('Missing DATAFORSEO_LOGIN/DATAFORSEO_PASSWORD after loading env')
  return {
    Authorization: `Basic ${Buffer.from(`${login}:${password}`).toString('base64')}`,
    'Content-Type': 'application/json'
  }
}
function targetRows() {
  const existing = readJson(join(outRoot, 'dataforseo_pilot_manifest.json'), {})
  if (Array.isArray(existing.targets) && existing.targets.length) return existing.targets
  const csvPath = join(outRoot, 'dataforseo_pilot_targets.csv')
  if (existsSync(csvPath)) return parseCsv(readFileSync(csvPath, 'utf8'))
  return []
}
function taskPayload(targets: any[]) {
  return targets.map(target => ({
    location_name: target.LocationName || 'United Kingdom',
    language_name: target.LanguageName || 'English (United Kingdom)',
    se_domain: target.AmazonDomain || 'amazon.co.uk',
    keyword: `Just Vitamins ${target.Product || target.SKU}`.trim(),
    depth: Number(process.env.DATAFORSEO_PRODUCT_SEARCH_DEPTH || 20),
    department: 'Health & Household',
    tag: `jv:${target.SKU}:asin-search-live`
  }))
}
async function sleep(ms: number) {
  await new Promise(resolve => setTimeout(resolve, ms))
}
async function dataforseoFetch(path: string, init: RequestInit = {}) {
  const response = await fetch(`https://api.dataforseo.com${path}`, { ...init, headers: { ...authHeaders(), ...(init.headers || {}) } })
  const body = await response.json()
  return { response, body }
}

const generatedAt = new Date().toISOString()
const envFile = process.env.DATAFORSEO_ENV_FILE || defaultEnvPath
const envLoaded = loadEnvFile(envFile)
const explicitAllow = process.env.JV_SOURCE_ALLOW_PAID === allowToken
const targets = targetRows()
const manifestBase = {
  generatedAt,
  ready: false,
  credentialSource: envFile,
  credentialPrinted: false,
  endpoint: 'POST https://api.dataforseo.com/v3/merchant/amazon/products/task_post',
  allowToken: `JV_SOURCE_ALLOW_PAID=${allowToken}`,
  counts: { targets: targets.length, tasksPosted: 0, tasksCompleted: 0, resultItems: 0 },
  files: {
    manifest: 'content_population_exports/dataforseo_products_live_run.json',
    report: 'content_population_exports/dataforseo_products_live_run.md',
    rawDir: 'data/sources/dataforseo/products_raw',
    latestIds: 'data/sources/dataforseo/products_raw/latest-products-task-ids-live.json'
  }
}

if (!explicitAllow) {
  const manifest = {
    ...manifestBase,
    blocked: true,
    blocker: `Set JV_SOURCE_ALLOW_PAID=${allowToken} to intentionally spend the small live Amazon Products search task credits.`,
    envLoaded,
    counts: manifestBase.counts
  }
  writeFileSync(join(outRoot, 'dataforseo_products_live_run.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
  writeFileSync(join(outRoot, 'dataforseo_products_live_run.md'), [
    '# DataForSEO live Amazon Products search run',
    '',
    `Generated: ${generatedAt}`,
    '',
    'Ready: NO',
    '',
    `Blocked: ${manifest.blocker}`,
    '',
    `Targets available: ${targets.length}`,
    ''
  ].join('\n'), 'utf8')
  console.log('DataForSEO live product search blocked: explicit allow flag missing')
  console.log(manifest.blocker)
  process.exit(0)
}

const account = await dataforseoFetch('/v3/appendix/user_data')
const payload = taskPayload(targets)
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)
const post = await dataforseoFetch('/v3/merchant/amazon/products/task_post', { method: 'POST', body: JSON.stringify(payload) })
const postPath = join(productsRawRoot, `products-task-post-live-${stamp}.json`)
writeFileSync(postPath, JSON.stringify(post.body, null, 2) + '\n', 'utf8')
const ids = (post.body.tasks || []).filter((task: any) => task.id).map((task: any) => ({ id: task.id, tag: task.data?.tag, status_code: task.status_code, status_message: task.status_message }))
writeFileSync(join(productsRawRoot, 'latest-products-task-ids-live.json'), JSON.stringify(ids, null, 2) + '\n', 'utf8')

const maxAttempts = Number(process.env.DATAFORSEO_PRODUCT_SEARCH_POLL_ATTEMPTS || 12)
const pollSeconds = Number(process.env.DATAFORSEO_PRODUCT_SEARCH_POLL_SECONDS || 10)
let finalSummary: any[] = []
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
  finalSummary = []
  for (const task of ids) {
    const get = await dataforseoFetch(`/v3/merchant/amazon/products/task_get/advanced/${task.id}`)
    const file = join(productsRawRoot, `products-task-get-${String(task.tag).replace(/:/g, '-')}-${task.id}.json`)
    writeFileSync(file, JSON.stringify(get.body, null, 2) + '\n', 'utf8')
    const firstTask = get.body.tasks?.[0]
    const items = firstTask?.result?.[0]?.items || []
    finalSummary.push({ tag: task.tag, id: task.id, status_code: firstTask?.status_code, status_message: firstTask?.status_message, resultCount: items.length })
  }
  if (finalSummary.every(row => row.resultCount > 0 || row.status_code === 20000)) break
  await sleep(pollSeconds * 1000)
}

const resultItems = finalSummary.reduce((sum, row) => sum + Number(row.resultCount || 0), 0)
const manifest = {
  ...manifestBase,
  ready: ids.length > 0 && resultItems > 0,
  envLoaded,
  accountConnection: { status_code: account.body.status_code, status_message: account.body.status_message, httpStatus: account.response.status },
  post: { status_code: post.body.status_code, status_message: post.body.status_message, file: postPath },
  counts: { targets: targets.length, tasksPosted: ids.length, tasksCompleted: finalSummary.filter(row => row.status_code === 20000).length, resultItems },
  taskIds: ids,
  taskSummary: finalSummary,
  nextCommand: 'bun run dataforseo:live-asin-evidence'
}
writeFileSync(join(outRoot, 'dataforseo_products_live_run.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'dataforseo_products_live_run.csv'), csv(finalSummary, ['tag', 'id', 'status_code', 'status_message', 'resultCount']), 'utf8')
writeFileSync(join(outRoot, 'dataforseo_products_live_run.md'), [
  '# DataForSEO live Amazon Products search run',
  '',
  `Generated: ${generatedAt}`,
  '',
  `Ready: ${manifest.ready ? 'YES' : 'NO'}`,
  '',
  `Account: ${manifest.accountConnection.status_code} ${manifest.accountConnection.status_message}`,
  `Posted tasks: ${manifest.counts.tasksPosted}`,
  `Completed tasks: ${manifest.counts.tasksCompleted}`,
  `Result items: ${manifest.counts.resultItems}`,
  '',
  '## Next step',
  `- Run \`${manifest.nextCommand}\` to rebuild ASIN map/payload/evidence from the raw product results.`,
  ''
].join('\n'), 'utf8')

console.log(`DataForSEO live product search ready: ${manifest.ready}`)
console.log(`Posted tasks: ${manifest.counts.tasksPosted}`)
console.log(`Completed tasks: ${manifest.counts.tasksCompleted}`)
console.log(`Result items: ${manifest.counts.resultItems}`)