#!/usr/bin/env bun import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs' import { join } from 'path' const root = process.cwd() const outRoot = join(root, 'content_population_exports') const templateRoot = join(root, 'data', 'sources', '_templates') mkdirSync(outRoot, { recursive: true }) mkdirSync(templateRoot, { recursive: true }) function readJson(path: string, fallback: any = null) { if (!existsSync(path)) return fallback return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, '')) } function parseCsv(text: string) { const rows: string[][] = [] let row: string[] = [] let cell = '' let quoted = false for (let i = 0; i < text.length; i++) { const ch = text[i] const next = text[i + 1] if (quoted) { if (ch === '"' && next === '"') { cell += '"' i++ } else if (ch === '"') { quoted = false } else { cell += ch } } else if (ch === '"') { quoted = true } else if (ch === ',') { row.push(cell) cell = '' } else if (ch === '\n') { row.push(cell.replace(/\r$/, '')) rows.push(row) row = [] cell = '' } else { cell += ch } } if (cell || row.length) { row.push(cell.replace(/\r$/, '')) rows.push(row) } const headers = rows.shift() || [] return rows .filter(values => values.some(value => value !== '')) .map(values => Object.fromEntries(headers.map((header, index) => [header, values[index] || '']))) } function readCsv(path: string) { if (!existsSync(path)) return [] return parseCsv(readFileSync(path, 'utf8').replace(/^\uFEFF/, '')) } function csv(rows: any[], headers: string[]) { const esc = (value: any) => { if (value === undefined || value === null) value = '' if (Array.isArray(value)) value = value.join(' | ') value = String(value) return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value } return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n' } const generatedAt = new Date().toISOString() const asinMap = readCsv(join(root, 'data', 'sources', 'dataforseo', 'asin-map.csv')) const payload = readJson(join(outRoot, 'dataforseo_amazon_reviews_task_payload.json'), []) const supportGate = readJson(join(outRoot, 'dataforseo_support_response_gate.json'), {}) const payloadTags = new Set((Array.isArray(payload) ? payload : []).map((row: any) => row.tag)) const readyRows = asinMap.filter((row: any) => row.ASIN && row.Status === 'ready_for_task_payload') const queue = readyRows.map((row: any) => { const productUrl = row.ProductUrl || `https://www.amazon.co.uk/dp/${row.ASIN}` const reviewUrl = `https://www.amazon.co.uk/product-reviews/${row.ASIN}/?sortBy=recent` return { SKU: row.SKU, Product: row.Product, ASIN: row.ASIN, ProductUrl: productUrl, ReviewUrl: reviewUrl, DepthTarget: row.Depth || '10', SourceDropPath: `data/sources/amazon/${row.SKU.toLowerCase()}-${row.ASIN.toLowerCase()}-manual-amazon-reviews.json`, CaptureStatus: 'pending_manual_capture', Reason: supportGate.ready === true ? 'provider path may be available; manual fallback remains optional' : 'DataForSEO Reviews endpoint not cleared; manual capture can unblock Amazon source drops without provider spend' } }) const template = { source: 'amazon', brand: 'Just Vitamins', sku: '', asin: '', productUrl: '', capturedAt: generatedAt, captureMethod: 'manual_browser_capture_from_amazon_uk_reviews_page', reviewer: '', reviews: [ { review_title: '', review_body: '', rating: 5, date: '', helpful_count: 0, url: '' } ] } writeFileSync(join(templateRoot, 'amazon-manual-review-capture-template.json'), JSON.stringify(template, null, 2) + '\n', 'utf8') const manifest = { generatedAt, ready: queue.length > 0, recommendation: supportGate.ready === true ? 'Provider support has a recorded retry path, but this manual capture packet remains available as a no-spend fallback.' : 'Use this no-spend manual capture queue if DataForSEO Reviews remains blocked; save completed JSON files into data/sources/amazon/ and run bun run validate:sources.', counts: { targets: queue.length, payloadRows: Array.isArray(payload) ? payload.length : 0, payloadRowsRepresented: queue.filter(row => payloadTags.has(`jv:${row.SKU}:stage1`)).length, supportResponsePassing: supportGate.counts?.passing || 0, supportResponseChecks: supportGate.counts?.checks || 0 }, files: { queueMd: 'content_population_exports/manual_amazon_review_capture.md', queueCsv: 'content_population_exports/manual_amazon_review_capture.csv', queueJson: 'content_population_exports/manual_amazon_review_capture.json', sourceDropTemplate: 'data/sources/_templates/amazon-manual-review-capture-template.json', sourceDropDir: 'data/sources/amazon' }, queue, sourceDropTemplate: template } writeFileSync(join(outRoot, 'manual_amazon_review_capture.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8') writeFileSync(join(outRoot, 'manual_amazon_review_capture.csv'), csv(queue, ['SKU', 'Product', 'ASIN', 'ProductUrl', 'ReviewUrl', 'DepthTarget', 'SourceDropPath', 'CaptureStatus', 'Reason']), 'utf8') writeFileSync(join(outRoot, 'manual_amazon_review_capture.md'), [ '# Manual Amazon review capture fallback', '', `Generated: ${generatedAt}`, '', manifest.recommendation, '', '## Guardrails', '- This does not call Amazon, DataForSEO, Apify, or Shopify.', '- Capture only the listed Stage 1 ASINs; do not broaden to full catalogue scraping.', '- Save completed source drops under `data/sources/amazon/` using the listed `SourceDropPath`.', '- Each review needs text, rating, date when visible, and URL/permalink when available.', '- After adding source drops, run `bun run validate:sources` before generating intelligence.', '', '## Queue', ...queue.map(row => [ `### ${row.SKU} — ${row.Product}`, `- ASIN: ${row.ASIN}`, `- Product: ${row.ProductUrl}`, `- Reviews: ${row.ReviewUrl}`, `- Depth target: ${row.DepthTarget}`, `- Save as: \`${row.SourceDropPath}\``, '' ].join('\n')), '## Source-drop template', `Template written to \`${manifest.files.sourceDropTemplate}\`.`, '' ].join('\n'), 'utf8') console.log(`Manual Amazon review capture ready: ${manifest.ready}`) console.log(`Targets: ${manifest.counts.targets}`)