justvitamin/scripts/validate-source-drops.ts

#!/usr/bin/env bun
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'
import { join } from 'path'

const root = process.cwd()
const sourcesRoot = join(root, 'data', 'sources')
const templateRoot = join(sourcesRoot, '_templates')
const outRoot = join(root, 'content_population_exports')
mkdirSync(templateRoot, { recursive: true })
mkdirSync(outRoot, { recursive: true })

type SourceSpec = {
  key: string
  dir: string
  required: string[]
  optional: string[]
  minFiles: number
  template: any
  note: string
}

const specs: SourceSpec[] = [
  {
    key: 'amazon',
    dir: join(sourcesRoot, 'amazon'),
    minFiles: 1,
    required: ['source', 'reviews'],
    optional: ['brand', 'sku', 'asin', 'productUrl', 'capturedAt'],
    note: 'JV and competitor Amazon reviews. Each review should include review_body/comment, rating, date, and URL when available.',
    template: {
      source: 'amazon', brand: 'Just Vitamins or competitor brand', sku: 'JV-D1000 or blank for competitor-only export', asin: '', productUrl: '', capturedAt: new Date().toISOString(),
      reviews: [{ review_title: '', review_body: '', rating: 5, date: '', helpful_count: 0, url: '' }]
    }
  },
  {
    key: 'trustpilot',
    dir: join(sourcesRoot, 'trustpilot'),
    minFiles: 1,
    required: ['source', 'brand', 'reviews'],
    optional: ['domain', 'capturedAt'],
    note: 'Trustpilot/brand-level review exports for JV or competitor brands.',
    template: {
      source: 'trustpilot', brand: 'Brand name', domain: '', capturedAt: new Date().toISOString(),
      reviews: [{ review_title: '', review_body: '', rating: 5, date: '', url: '' }]
    }
  },
  {
    key: 'reddit',
    dir: join(sourcesRoot, 'reddit'),
    minFiles: 1,
    required: ['source', 'query', 'items'],
    optional: ['capturedAt'],
    note: 'Reddit/social listening exports for collagen and supplement category research.',
    template: {
      source: 'reddit', query: 'collagen flavour chalky sweetness mixability', capturedAt: new Date().toISOString(),
      items: [{ subreddit: '', thread_url: '', post_title: '', comment_body: '', upvotes: 0, date: '' }]
    }
  }
]

function readJson(path: string) {
  return JSON.parse(readFileSync(path, 'utf8').replace(/^\uFEFF/, ''))
}
function listJson(dir: string) {
  if (!existsSync(dir)) return []
  return readdirSync(dir, { withFileTypes: true }).filter(entry => entry.isFile() && entry.name.endsWith('.json')).map(entry => join(dir, entry.name))
}
function csv(rows: any[], headers: string[]) {
  const esc = (value: any) => {
    if (Array.isArray(value)) value = value.join(' | ')
    if (value === undefined || value === null) value = ''
    value = String(value)
    return /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value
  }
  return [headers.join(','), ...rows.map(row => headers.map(header => esc(row[header])).join(','))].join('\n') + '\n'
}
function validateItems(kind: string, data: any) {
  const issues: string[] = []
  const rows = kind === 'reddit' ? data.items : data.reviews
  if (!Array.isArray(rows) || rows.length === 0) issues.push(kind === 'reddit' ? 'items must be a non-empty array' : 'reviews must be a non-empty array')
  for (const [index, row] of (Array.isArray(rows) ? rows : []).entries()) {
    const text = kind === 'reddit' ? row.comment_body || row.post_title : row.review_body || row.comment || row.review_title
    if (!text || String(text).trim().length < 5) issues.push(`row ${index}: missing review/comment text`)
    if (kind !== 'reddit' && (row.rating === undefined || row.rating === null || row.rating === '')) issues.push(`row ${index}: missing rating`)
  }
  return issues
}

const rows: any[] = []
const details: any[] = []
for (const spec of specs) {
  mkdirSync(spec.dir, { recursive: true })
  writeFileSync(join(templateRoot, `${spec.key}-source-template.json`), JSON.stringify(spec.template, null, 2) + '\n', 'utf8')
  const files = listJson(spec.dir).filter(file => !file.includes(`${spec.key}-status.json`))
  if (files.length < spec.minFiles) {
    rows.push({ Source: spec.key, File: '', Status: 'missing', Issue: `Need at least ${spec.minFiles} JSON source drop(s) in ${spec.dir}`, Note: spec.note })
  }
  for (const file of files) {
    try {
      const data = readJson(file)
      const missing = spec.required.filter(field => data[field] === undefined || data[field] === null || data[field] === '' || (Array.isArray(data[field]) && data[field].length === 0))
      const itemIssues = validateItems(spec.key, data)
      const issues = [...missing.map(field => `missing required field ${field}`), ...itemIssues]
      rows.push({ Source: spec.key, File: file, Status: issues.length ? 'invalid' : 'valid', Issue: issues.join('; '), Note: spec.note })
      details.push({ source: spec.key, file, status: issues.length ? 'invalid' : 'valid', issues })
    } catch (error: any) {
      rows.push({ Source: spec.key, File: file, Status: 'invalid_json', Issue: error.message, Note: spec.note })
      details.push({ source: spec.key, file, status: 'invalid_json', issues: [error.message] })
    }
  }
}
const ready = rows.every(row => row.Status === 'valid') && specs.every(spec => listJson(spec.dir).length >= spec.minFiles)
const manifest = {
  generatedAt: new Date().toISOString(),
  ready,
  templateRoot,
  requiredSources: specs.map(spec => ({ key: spec.key, dir: spec.dir, minFiles: spec.minFiles, required: spec.required, optional: spec.optional, note: spec.note })),
  summary: specs.map(spec => ({ key: spec.key, fileCount: listJson(spec.dir).length, minFiles: spec.minFiles })),
  issues: rows.filter(row => row.Status !== 'valid'),
  details
}
writeFileSync(join(outRoot, 'source_drop_validation.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8')
writeFileSync(join(outRoot, 'source_drop_validation.csv'), csv(rows, ['Source', 'File', 'Status', 'Issue', 'Note']), 'utf8')
console.log(`Source drops ready: ${ready}`)
for (const item of manifest.summary) console.log(`${item.key}: ${item.fileCount}/${item.minFiles}`)
console.log(`Issues: ${manifest.issues.length}`)
if (!ready) process.exitCode = 1