Omair Saleh
056c47581f
Ships the second dashboard surface — a Pattern Library + Preview Theatre — that presents the 4-section PDP pilot batch back to Umar, compliance, and the board in an editorial format. Adds the full data layer that drives it: 5 source-backed per-SKU drafts at QA 100/100, 15 competitor PDP semantic extracts, PubMed evidence packs, EFSA claims library extension, JV brand voice guide, hand-curated product FAQs, and the Matrixify-ready CSV exports for Lewis. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
103 lines
4.3 KiB
TypeScript
103 lines
4.3 KiB
TypeScript
#!/usr/bin/env bun
|
|
import { mkdirSync, writeFileSync } from 'fs'
|
|
import { dirname, join } from 'path'
|
|
|
|
const root = process.cwd()
|
|
const rawRoot = join(root, 'data', 'sources', 'apify', 'raw')
|
|
const token = Bun.env.APIFY_TOKEN || Bun.env.APIFY_API_TOKEN
|
|
const generatedAt = new Date().toISOString()
|
|
if (!token) throw new Error('Missing APIFY_TOKEN or APIFY_API_TOKEN')
|
|
|
|
function writeJson(path: string, data: any) {
|
|
mkdirSync(dirname(path), { recursive: true })
|
|
writeFileSync(path, JSON.stringify(data, null, 2) + '\n', 'utf8')
|
|
}
|
|
async function apify(path: string, options: RequestInit = {}) {
|
|
const sep = path.includes('?') ? '&' : '?'
|
|
const res = await fetch(`https://api.apify.com${path}${sep}token=${encodeURIComponent(token!)}`, {
|
|
...options,
|
|
headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
|
|
})
|
|
const text = await res.text()
|
|
let data: any
|
|
try { data = text ? JSON.parse(text) : null } catch { data = text }
|
|
if (!res.ok) throw new Error(`Apify ${res.status} ${res.statusText}: ${typeof data === 'string' ? data : JSON.stringify(data)}`)
|
|
return data
|
|
}
|
|
async function waitForRun(runId: string, timeoutMs = 1000 * 60 * 7) {
|
|
const started = Date.now()
|
|
let run = (await apify(`/v2/actor-runs/${runId}`)).data
|
|
while (!['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(run.status)) {
|
|
if (Date.now() - started > timeoutMs) throw new Error(`Timed out waiting for ${runId}`)
|
|
await new Promise(resolve => setTimeout(resolve, 5000))
|
|
run = (await apify(`/v2/actor-runs/${runId}`)).data
|
|
console.log(`${runId}: ${run.status} usd=${run.usageTotalUsd || 0}`)
|
|
}
|
|
return run
|
|
}
|
|
|
|
const pageFunction = String.raw`
|
|
async function pageFunction(context) {
|
|
const { page, request } = context;
|
|
await page.waitForTimeout(5000);
|
|
const data = await page.evaluate(() => {
|
|
const clean = (value) => String(value || '').replace(/\s+/g, ' ').trim();
|
|
const text = clean(document.body?.innerText || '').slice(0, 70000);
|
|
const headings = Array.from(document.querySelectorAll('h1,h2,h3')).map(el => clean(el.textContent)).filter(Boolean).slice(0, 80);
|
|
const prices = Array.from(document.querySelectorAll('[class*=price], [data-testid*=price], [aria-label*=price]')).map(el => clean(el.textContent)).filter(Boolean).slice(0, 50);
|
|
const links = Array.from(document.querySelectorAll('a[href]')).map(el => ({ text: clean(el.textContent).slice(0, 160), href: el.href })).filter(row => row.text || row.href).slice(0, 100);
|
|
return { title: document.title, h1: clean(document.querySelector('h1')?.textContent || ''), headings, prices, links, text, textChars: text.length, htmlChars: document.documentElement.outerHTML.length };
|
|
});
|
|
return {
|
|
capturedAt: new Date().toISOString(),
|
|
sku: 'JV-VISISOFT',
|
|
sourceType: 'competitor_pdp_browser',
|
|
label: 'Healthspan OptiVision browser capture',
|
|
competitor: 'Healthspan',
|
|
requestedUrl: request.url,
|
|
loadedUrl: request.loadedUrl || request.url,
|
|
...data
|
|
};
|
|
}`
|
|
|
|
const input = {
|
|
startUrls: [{ url: 'https://www.healthspan.co.uk/optivision/' }],
|
|
maxRequestsPerCrawl: 1,
|
|
maxConcurrency: 1,
|
|
maxRequestRetries: 0,
|
|
pageLoadTimeoutSecs: 45,
|
|
requestHandlerTimeoutSecs: 60,
|
|
pageFunction,
|
|
proxyConfiguration: { useApifyProxy: true },
|
|
ignoreSslErrors: true
|
|
}
|
|
|
|
const start = await apify('/v2/acts/apify~web-scraper/runs?maxTotalChargeUsd=0.08', {
|
|
method: 'POST',
|
|
body: JSON.stringify(input)
|
|
})
|
|
const run = await waitForRun(start.data.id)
|
|
const items = run.defaultDatasetId ? await apify(`/v2/datasets/${run.defaultDatasetId}/items?clean=true`) : []
|
|
const result = {
|
|
capturedAt: generatedAt,
|
|
job: 'healthspan-browser',
|
|
actor: 'apify/web-scraper',
|
|
run: {
|
|
id: run.id,
|
|
status: run.status,
|
|
statusMessage: run.statusMessage,
|
|
defaultDatasetId: run.defaultDatasetId,
|
|
startedAt: run.startedAt,
|
|
finishedAt: run.finishedAt,
|
|
usageTotalUsd: run.usageTotalUsd,
|
|
usage: run.usage,
|
|
stats: run.stats
|
|
},
|
|
itemCount: Array.isArray(items) ? items.length : 0,
|
|
items
|
|
}
|
|
const stamp = generatedAt.replace(/[:.]/g, '-')
|
|
writeJson(join(rawRoot, `top3-specialist-healthspan-browser-${stamp}.json`), result)
|
|
writeJson(join(rawRoot, 'top3-specialist-healthspan-browser-latest.json'), result)
|
|
console.log(`Healthspan browser capture ${run.status}: ${result.itemCount} items, usd=${run.usageTotalUsd || 0}`)
|