#!/usr/bin/env bun
import { sourcePath, loadCompetitorConfig, slug, stripHtml, writeJson, now } from './lib/intel-utils'

const outDir = sourcePath('competitor-pdps')
const timeoutMs = Number(process.env.FETCH_TIMEOUT_MS || 15000)
const competitors = loadCompetitorConfig()

async function fetchUrl(url: string) {
  const controller = new AbortController()
  const timer = setTimeout(() => controller.abort(), timeoutMs)
  try {
    const res = await fetch(url, { signal: controller.signal, headers: { 'user-agent': 'JV content intelligence research bot (manual migration workspace)' } })
    const html = await res.text()
    return { ok: res.ok, status: res.status, url: res.url, bytes: html.length, text: stripHtml(html).slice(0, 12000) }
  } catch (error: any) {
    return { ok: false, status: 0, url, bytes: 0, text: '', error: error?.message || String(error) }
  } finally {
    clearTimeout(timer)
  }
}

for (const competitor of competitors) {
  const pages = []
  for (const url of competitor.urls || []) pages.push(await fetchUrl(url))
  const out = { brand: competitor.brand, domain: competitor.domain, generatedAt: now(), sourceType: 'competitor_pdp_or_homepage_probe', pages }
  const path = `${outDir}/${slug(competitor.brand)}.json`
  writeJson(path, out)
  console.log(`${competitor.brand}: ${pages.filter(p => p.ok).length}/${pages.length} fetched`)
}