v3: PostgreSQL backend — 20K rows seeded, server-side SQL filtering

- Added PostgreSQL 16 Alpine service to Docker Swarm stack
- db.py: schema for 17 tables, auto-seed from jv_data.json on first boot
- /api/dashboard/data?start=&end= — server-side SQL filtering
  All Time: 0.43s (was 4MB JSON download stuck loading)
  Filtered (12mo): 0.20s with ~90% less data transferred
- Dashboard HTML patched: calls API instead of static JSON
- Integer casting for IsNewCustomer/HasDiscount/IsFreeShipping
- Advisory lock prevents race condition during parallel worker startup
- Returning Revenue now shows correctly: £14.5M (75% of total)

app.py

@@ -1,7 +1,8 @@
 """JustVitamin × QuikCue — AI Content Engine
-Live proposal site with real Gemini-powered demos."""
+Live proposal site with real Gemini-powered demos.
+PostgreSQL-backed data dashboard."""
 
-import os, json, time, traceback
+import os, json, time, traceback, logging
 from pathlib import Path
 from flask import (Flask, render_template, jsonify, request,
                    send_from_directory, redirect)
@@ -9,6 +10,9 @@ from flask import (Flask, render_template, jsonify, request,
 from scraper import scrape_product, scrape_competitor
 from ai_engine import (generate_asset_pack, competitor_xray, pdp_surgeon,
                         optimise_pdp_copy, generate_product_images)
+from db import init_db, get_dashboard_data, get_meta
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
 
 app = Flask(__name__,
             static_folder="static",
@@ -17,6 +21,13 @@ app = Flask(__name__,
 GEN_DIR = Path(__file__).parent / "generated"
 GEN_DIR.mkdir(exist_ok=True)
 
+# ── Init DB on startup ───────────────────────────────────────
+with app.app_context():
+    try:
+        init_db()
+    except Exception as e:
+        logging.error(f"DB init failed: {e}")
+
 # ── Page routes ──────────────────────────────────────────────
 
 @app.route("/")
@@ -32,6 +43,37 @@ def dashboard():
 def dashboard_files(filename):
     return send_from_directory("static/dashboard", filename)
 
+
+# ── Dashboard API (Postgres-backed) ─────────────────────────
+
+@app.route("/api/dashboard/data")
+def api_dashboard_data():
+    """Return filtered dashboard data from PostgreSQL.
+    Query params: ?start=YYYY-MM&end=YYYY-MM
+    Replaces the 4MB static JSON with fast server-side SQL filtering."""
+    start = request.args.get("start") or None
+    end = request.args.get("end") or None
+    try:
+        t0 = time.time()
+        data = get_dashboard_data(start, end)
+        data["_query_time"] = f"{time.time()-t0:.3f}s"
+        data["_source"] = "postgresql"
+        resp = jsonify(data)
+        resp.headers['Cache-Control'] = 'public, max-age=60'
+        return resp
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/api/dashboard/meta")
+def api_dashboard_meta():
+    """Return just the meta info (fast, tiny)."""
+    try:
+        return jsonify(get_meta())
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
 @app.route("/proposal")
 def proposal():
     return send_from_directory("static/proposal", "index.html")
@@ -185,9 +227,23 @@ def api_generate_images():
 
 @app.route("/api/health")
 def health():
+    db_ok = False
+    try:
+        from db import get_conn
+        conn = get_conn()
+        cur = conn.cursor()
+        cur.execute("SELECT COUNT(*) FROM monthly")
+        count = cur.fetchone()[0]
+        cur.close()
+        conn.close()
+        db_ok = count > 0
+    except Exception:
+        pass
     return jsonify({
         "status": "ok",
         "gemini_key_set": bool(os.environ.get("GEMINI_API_KEY")),
+        "db_connected": db_ok,
+        "db_rows": count if db_ok else 0,
     })
 
 

db.py

@@ -0,0 +1,570 @@
+"""JustVitamin — PostgreSQL data layer.
+
+Creates tables, seeds from jv_data.json on first boot,
+provides fast filtered queries for the dashboard API."""
+
+import os, json, time, logging
+from pathlib import Path
+
+import psycopg2
+from psycopg2.extras import execute_values, RealDictCursor
+
+log = logging.getLogger(__name__)
+
+DB_URL = os.environ.get(
+    "DATABASE_URL",
+    "postgresql://jv:jvpass@tasks.db:5432/justvitamin"
+)
+
+# ── Connection ───────────────────────────────────────────────
+
+def get_conn():
+    return psycopg2.connect(DB_URL)
+
+
+def wait_for_db(retries=30, delay=2):
+    """Block until Postgres is ready."""
+    for i in range(retries):
+        try:
+            conn = get_conn()
+            conn.close()
+            log.info("✓ Database ready")
+            return True
+        except Exception:
+            if i < retries - 1:
+                log.info(f"Waiting for DB... ({i+1}/{retries})")
+                time.sleep(delay)
+    raise RuntimeError("Database not available after retries")
+
+
+# ── Schema ───────────────────────────────────────────────────
+
+SCHEMA = """
+-- Core monthly aggregates
+CREATE TABLE IF NOT EXISTS monthly (
+    year_month TEXT PRIMARY KEY,
+    revenue NUMERIC, orders INT, aov NUMERIC, customers INT,
+    new_customers INT, avg_shipping NUMERIC, free_ship_pct NUMERIC,
+    avg_margin NUMERIC, discount_pct NUMERIC, avg_items NUMERIC,
+    total_shipping NUMERIC, free_ship_orders INT, discount_orders INT,
+    total_items INT
+);
+
+-- Channel breakdown by month
+CREATE TABLE IF NOT EXISTS channel_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, referrer_source TEXT,
+    orders INT, revenue NUMERIC, avg_aov NUMERIC,
+    avg_items NUMERIC, new_pct NUMERIC, free_ship_pct NUMERIC,
+    discount_pct NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_channel_ym ON channel_monthly(year_month);
+
+-- New vs returning by month
+CREATE TABLE IF NOT EXISTS new_returning_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, is_new_customer TEXT,
+    orders INT, revenue NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_nr_ym ON new_returning_monthly(year_month);
+
+-- Day of week by month
+CREATE TABLE IF NOT EXISTS dow_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, day_of_week TEXT,
+    orders INT, revenue NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_dow_ym ON dow_monthly(year_month);
+
+-- Payment processor by month
+CREATE TABLE IF NOT EXISTS payment_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, payment_processor TEXT,
+    orders INT, revenue NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_pay_ym ON payment_monthly(year_month);
+
+-- Country by month
+CREATE TABLE IF NOT EXISTS country_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, customer_country TEXT,
+    orders INT, revenue NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_country_ym ON country_monthly(year_month);
+
+-- Delivery by month
+CREATE TABLE IF NOT EXISTS delivery_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, delivery_method TEXT,
+    orders INT, revenue NUMERIC, avg_shipping NUMERIC, avg_aov NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_delivery_ym ON delivery_monthly(year_month);
+
+-- AOV buckets by month
+CREATE TABLE IF NOT EXISTS aov_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, aov_bucket TEXT,
+    count INT, revenue NUMERIC, avg_shipping NUMERIC, pct_free_ship NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_aov_ym ON aov_monthly(year_month);
+
+-- Discount by month
+CREATE TABLE IF NOT EXISTS discount_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, has_discount TEXT,
+    count INT, avg_aov NUMERIC, avg_items NUMERIC,
+    avg_margin NUMERIC, avg_shipping NUMERIC, revenue NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_disc_ym ON discount_monthly(year_month);
+
+-- Discount codes by month
+CREATE TABLE IF NOT EXISTS discount_codes_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, discount_code TEXT,
+    uses INT, revenue NUMERIC, avg_aov NUMERIC, avg_discount_pct NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_dcode_ym ON discount_codes_monthly(year_month);
+
+-- Free ship comparison by month
+CREATE TABLE IF NOT EXISTS shipping_comp_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, is_free_shipping TEXT,
+    count INT, avg_aov NUMERIC, avg_items NUMERIC,
+    avg_margin NUMERIC, new_pct NUMERIC, discount_pct NUMERIC,
+    avg_shipping NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_shipcomp_ym ON shipping_comp_monthly(year_month);
+
+-- Category by month
+CREATE TABLE IF NOT EXISTS category_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, category TEXT,
+    total_revenue NUMERIC, total_revenue_ex_vat NUMERIC,
+    total_cost NUMERIC, total_qty INT, order_count INT,
+    unique_products INT, margin NUMERIC, margin_pct NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_cat_ym ON category_monthly(year_month);
+
+-- Product by month
+CREATE TABLE IF NOT EXISTS product_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, description TEXT, sku_description TEXT,
+    category TEXT, total_revenue NUMERIC, total_revenue_ex_vat NUMERIC,
+    total_cost NUMERIC, total_qty INT, order_count INT,
+    margin NUMERIC, margin_pct NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_prod_ym ON product_monthly(year_month);
+
+-- Cross-sell pairs (no month dimension)
+CREATE TABLE IF NOT EXISTS cross_sell (
+    id SERIAL PRIMARY KEY,
+    product1 TEXT, product2 TEXT, count INT
+);
+
+-- Cohort retention
+CREATE TABLE IF NOT EXISTS cohort_retention (
+    id SERIAL PRIMARY KEY,
+    cohort TEXT, size INT,
+    m0 NUMERIC, m1 NUMERIC, m2 NUMERIC, m3 NUMERIC,
+    m4 NUMERIC, m5 NUMERIC, m6 NUMERIC, m7 NUMERIC,
+    m8 NUMERIC, m9 NUMERIC, m10 NUMERIC, m11 NUMERIC,
+    m12 NUMERIC
+);
+
+-- Free shipping threshold by month
+CREATE TABLE IF NOT EXISTS threshold_monthly (
+    id SERIAL PRIMARY KEY,
+    year_month TEXT, total_orders INT, total_shipping_rev NUMERIC,
+    t15_eligible NUMERIC, t15_ship_absorbed NUMERIC, t15_near NUMERIC, t15_near_gap NUMERIC,
+    t20_eligible NUMERIC, t20_ship_absorbed NUMERIC, t20_near NUMERIC, t20_near_gap NUMERIC,
+    t25_eligible NUMERIC, t25_ship_absorbed NUMERIC, t25_near NUMERIC, t25_near_gap NUMERIC,
+    t30_eligible NUMERIC, t30_ship_absorbed NUMERIC, t30_near NUMERIC, t30_near_gap NUMERIC,
+    t35_eligible NUMERIC, t35_ship_absorbed NUMERIC, t35_near NUMERIC, t35_near_gap NUMERIC,
+    t40_eligible NUMERIC, t40_ship_absorbed NUMERIC, t40_near NUMERIC, t40_near_gap NUMERIC,
+    t50_eligible NUMERIC, t50_ship_absorbed NUMERIC, t50_near NUMERIC, t50_near_gap NUMERIC
+);
+CREATE INDEX IF NOT EXISTS idx_thresh_ym ON threshold_monthly(year_month);
+
+-- Metadata
+CREATE TABLE IF NOT EXISTS meta (
+    key TEXT PRIMARY KEY,
+    value TEXT
+);
+"""
+
+
+def init_schema():
+    conn = get_conn()
+    cur = conn.cursor()
+    try:
+        cur.execute(SCHEMA)
+        conn.commit()
+        log.info("✓ Schema created")
+    except Exception as e:
+        conn.rollback()
+        if "already exists" in str(e):
+            log.info("✓ Schema already exists")
+        else:
+            raise
+    finally:
+        cur.close()
+        conn.close()
+
+
+# ── Seed ─────────────────────────────────────────────────────
+
+def is_seeded():
+    conn = get_conn()
+    cur = conn.cursor()
+    cur.execute("SELECT COUNT(*) FROM monthly")
+    n = cur.fetchone()[0]
+    cur.close()
+    conn.close()
+    return n > 0
+
+
+def seed_from_json(path=None):
+    """Load jv_data.json into Postgres. Idempotent — skips if data exists."""
+    if is_seeded():
+        log.info("✓ Database already seeded — skipping")
+        return
+
+    if path is None:
+        path = Path(__file__).parent / "static" / "dashboard" / "jv_data.json"
+    
+    log.info(f"Seeding from {path} ...")
+    t0 = time.time()
+    
+    with open(path) as f:
+        data = json.load(f)
+
+    conn = get_conn()
+    cur = conn.cursor()
+
+    # 1. monthly
+    rows = [(r['YearMonth'], r.get('revenue'), r.get('orders'), r.get('aov'),
+             r.get('customers'), r.get('newCustomers'), r.get('avgShipping'),
+             r.get('freeShipPct'), r.get('avgMargin'), r.get('discountPct'),
+             r.get('avgItems'), r.get('totalShipping'), r.get('freeShipOrders'),
+             r.get('discountOrders'), r.get('totalItems'))
+            for r in data['monthly']]
+    execute_values(cur,
+        "INSERT INTO monthly VALUES %s ON CONFLICT DO NOTHING",
+        rows)
+
+    # 2. channelMonthly
+    rows = [(r['YearMonth'], r.get('ReferrerSource'), r.get('orders'),
+             r.get('revenue'), r.get('avgAOV'), r.get('avgItems'),
+             r.get('newPct'), r.get('freeShipPct'), r.get('discountPct'))
+            for r in data['channelMonthly']]
+    execute_values(cur,
+        "INSERT INTO channel_monthly (year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct) VALUES %s",
+        rows)
+
+    # 3. newReturningMonthly
+    rows = [(r['YearMonth'], r.get('IsNewCustomer'), r.get('orders'), r.get('revenue'))
+            for r in data['newReturningMonthly']]
+    execute_values(cur,
+        "INSERT INTO new_returning_monthly (year_month,is_new_customer,orders,revenue) VALUES %s",
+        rows)
+
+    # 4. dowMonthly
+    rows = [(r['YearMonth'], r.get('DayOfWeek'), r.get('orders'), r.get('revenue'))
+            for r in data['dowMonthly']]
+    execute_values(cur,
+        "INSERT INTO dow_monthly (year_month,day_of_week,orders,revenue) VALUES %s",
+        rows)
+
+    # 5. paymentMonthly
+    rows = [(r['YearMonth'], r.get('PaymentProcessor'), r.get('orders'), r.get('revenue'))
+            for r in data['paymentMonthly']]
+    execute_values(cur,
+        "INSERT INTO payment_monthly (year_month,payment_processor,orders,revenue) VALUES %s",
+        rows)
+
+    # 6. countryMonthly
+    rows = [(r['YearMonth'], r.get('CustomerCountry'), r.get('orders'), r.get('revenue'))
+            for r in data['countryMonthly']]
+    execute_values(cur,
+        "INSERT INTO country_monthly (year_month,customer_country,orders,revenue) VALUES %s",
+        rows)
+
+    # 7. deliveryMonthly
+    rows = [(r['YearMonth'], r.get('DeliveryMethod'), r.get('orders'),
+             r.get('revenue'), r.get('avgShipping'), r.get('avgAOV'))
+            for r in data['deliveryMonthly']]
+    execute_values(cur,
+        "INSERT INTO delivery_monthly (year_month,delivery_method,orders,revenue,avg_shipping,avg_aov) VALUES %s",
+        rows)
+
+    # 8. aovMonthly
+    rows = [(r['YearMonth'], r.get('AOVBucket'), r.get('count'),
+             r.get('revenue'), r.get('avgShipping'), r.get('pctFreeShip'))
+            for r in data['aovMonthly']]
+    execute_values(cur,
+        "INSERT INTO aov_monthly (year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship) VALUES %s",
+        rows)
+
+    # 9. discountMonthly
+    rows = [(r['YearMonth'], r.get('HasDiscount'), r.get('count'),
+             r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'),
+             r.get('avgShipping'), r.get('revenue'))
+            for r in data['discountMonthly']]
+    execute_values(cur,
+        "INSERT INTO discount_monthly (year_month,has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue) VALUES %s",
+        rows)
+
+    # 10. discountCodesMonthly
+    rows = [(r['YearMonth'], r.get('DiscountCode'), r.get('uses'),
+             r.get('revenue'), r.get('avgAOV'), r.get('avgDiscountPct'))
+            for r in data['discountCodesMonthly']]
+    execute_values(cur,
+        "INSERT INTO discount_codes_monthly (year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct) VALUES %s",
+        rows)
+
+    # 11. shippingCompMonthly
+    rows = [(r['YearMonth'], r.get('IsFreeShipping'), r.get('count'),
+             r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'),
+             r.get('newPct'), r.get('discountPct'), r.get('avgShipping'))
+            for r in data['shippingCompMonthly']]
+    execute_values(cur,
+        "INSERT INTO shipping_comp_monthly (year_month,is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping) VALUES %s",
+        rows)
+
+    # 12. categoryMonthly
+    rows = [(r['YearMonth'], r.get('Category'), r.get('totalRevenue'),
+             r.get('totalRevenueExVAT'), r.get('totalCost'), r.get('totalQty'),
+             r.get('orderCount'), r.get('uniqueProducts'),
+             r.get('margin'), r.get('marginPct'))
+            for r in data['categoryMonthly']]
+    execute_values(cur,
+        "INSERT INTO category_monthly (year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct) VALUES %s",
+        rows)
+
+    # 13. productMonthly
+    rows = [(r['YearMonth'], r.get('Description'), r.get('SKUDescription'),
+             r.get('Category'), r.get('totalRevenue'), r.get('totalRevenueExVAT'),
+             r.get('totalCost'), r.get('totalQty'), r.get('orderCount'),
+             r.get('margin'), r.get('marginPct'))
+            for r in data['productMonthly']]
+    execute_values(cur,
+        "INSERT INTO product_monthly (year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct) VALUES %s",
+        rows)
+
+    # 14. crossSell
+    rows = [(r.get('product1'), r.get('product2'), r.get('count'))
+            for r in data['crossSell']]
+    execute_values(cur,
+        "INSERT INTO cross_sell (product1,product2,count) VALUES %s",
+        rows)
+
+    # 15. cohortRetention
+    rows = [(r.get('cohort'), r.get('size'),
+             r.get('m0'), r.get('m1'), r.get('m2'), r.get('m3'),
+             r.get('m4'), r.get('m5'), r.get('m6'), r.get('m7'),
+             r.get('m8'), r.get('m9'), r.get('m10'), r.get('m11'),
+             r.get('m12'))
+            for r in data['cohortRetention']]
+    execute_values(cur,
+        "INSERT INTO cohort_retention (cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12) VALUES %s",
+        rows)
+
+    # 16. thresholdMonthly
+    rows = [(r['YearMonth'], r.get('totalOrders'), r.get('totalShippingRev'),
+             r.get('t15_eligible'), r.get('t15_shipAbsorbed'), r.get('t15_near'), r.get('t15_nearGap'),
+             r.get('t20_eligible'), r.get('t20_shipAbsorbed'), r.get('t20_near'), r.get('t20_nearGap'),
+             r.get('t25_eligible'), r.get('t25_shipAbsorbed'), r.get('t25_near'), r.get('t25_nearGap'),
+             r.get('t30_eligible'), r.get('t30_shipAbsorbed'), r.get('t30_near'), r.get('t30_nearGap'),
+             r.get('t35_eligible'), r.get('t35_shipAbsorbed'), r.get('t35_near'), r.get('t35_nearGap'),
+             r.get('t40_eligible'), r.get('t40_shipAbsorbed'), r.get('t40_near'), r.get('t40_nearGap'),
+             r.get('t50_eligible'), r.get('t50_shipAbsorbed'), r.get('t50_near'), r.get('t50_nearGap'))
+            for r in data['thresholdMonthly']]
+    execute_values(cur,
+        "INSERT INTO threshold_monthly (year_month,total_orders,total_shipping_rev,t15_eligible,t15_ship_absorbed,t15_near,t15_near_gap,t20_eligible,t20_ship_absorbed,t20_near,t20_near_gap,t25_eligible,t25_ship_absorbed,t25_near,t25_near_gap,t30_eligible,t30_ship_absorbed,t30_near,t30_near_gap,t35_eligible,t35_ship_absorbed,t35_near,t35_near_gap,t40_eligible,t40_ship_absorbed,t40_near,t40_near_gap,t50_eligible,t50_ship_absorbed,t50_near,t50_near_gap) VALUES %s",
+        rows)
+
+    # 17. meta
+    meta = data.get('meta', {})
+    for k, v in meta.items():
+        cur.execute(
+            "INSERT INTO meta (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",
+            (k, json.dumps(v) if not isinstance(v, str) else v)
+        )
+
+    conn.commit()
+    cur.close()
+    conn.close()
+
+    elapsed = time.time() - t0
+    log.info(f"✓ Seeded {sum(len(data[k]) for k in data if isinstance(data[k], list))} rows in {elapsed:.1f}s")
+
+
+# ── Queries ──────────────────────────────────────────────────
+
+def _ym_clause(start, end, col="year_month"):
+    """Build WHERE clause for date range."""
+    parts = []
+    params = []
+    if start:
+        parts.append(f"{col} >= %s")
+        params.append(start)
+    if end:
+        parts.append(f"{col} <= %s")
+        params.append(end)
+    where = " AND ".join(parts) if parts else "TRUE"
+    return where, params
+
+
+def _query(sql, params=()):
+    conn = get_conn()
+    cur = conn.cursor(cursor_factory=RealDictCursor)
+    cur.execute(sql, params)
+    rows = cur.fetchall()
+    cur.close()
+    conn.close()
+    # Convert Decimal to float for JSON serialisation
+    return [_dec_to_float(r) for r in rows]
+
+
+def _dec_to_float(d):
+    from decimal import Decimal
+    return {k: (float(v) if isinstance(v, Decimal) else v) for k, v in d.items()}
+
+
+def get_meta():
+    rows = _query("SELECT key, value FROM meta")
+    meta = {}
+    for r in rows:
+        v = r['value']
+        try:
+            v = json.loads(v)
+        except (json.JSONDecodeError, TypeError):
+            pass
+        meta[r['key']] = v
+    return meta
+
+
+def _remap(rows, mapping):
+    """Rename DB snake_case keys back to the camelCase the dashboard JS expects."""
+    return [{mapping.get(k, k): v for k, v in r.items()} for r in rows]
+
+
+# Column mappings: DB name → JS name
+_M_MONTHLY = {
+    'year_month':'YearMonth','revenue':'revenue','orders':'orders','aov':'aov',
+    'customers':'customers','new_customers':'newCustomers','avg_shipping':'avgShipping',
+    'free_ship_pct':'freeShipPct','avg_margin':'avgMargin','discount_pct':'discountPct',
+    'avg_items':'avgItems','total_shipping':'totalShipping','free_ship_orders':'freeShipOrders',
+    'discount_orders':'discountOrders','total_items':'totalItems'
+}
+_M_CHANNEL = {
+    'year_month':'YearMonth','referrer_source':'ReferrerSource','orders':'orders',
+    'revenue':'revenue','avg_aov':'avgAOV','avg_items':'avgItems',
+    'new_pct':'newPct','free_ship_pct':'freeShipPct','discount_pct':'discountPct'
+}
+_M_NR = {'year_month':'YearMonth','is_new_customer':'IsNewCustomer','orders':'orders','revenue':'revenue'}
+_M_DOW = {'year_month':'YearMonth','day_of_week':'DayOfWeek','orders':'orders','revenue':'revenue'}
+_M_PAY = {'year_month':'YearMonth','payment_processor':'PaymentProcessor','orders':'orders','revenue':'revenue'}
+_M_COUNTRY = {'year_month':'YearMonth','customer_country':'CustomerCountry','orders':'orders','revenue':'revenue'}
+_M_DELIVERY = {
+    'year_month':'YearMonth','delivery_method':'DeliveryMethod','orders':'orders',
+    'revenue':'revenue','avg_shipping':'avgShipping','avg_aov':'avgAOV'
+}
+_M_AOV = {
+    'year_month':'YearMonth','aov_bucket':'AOVBucket','count':'count',
+    'revenue':'revenue','avg_shipping':'avgShipping','pct_free_ship':'pctFreeShip'
+}
+_M_DISC = {
+    'year_month':'YearMonth','has_discount':'HasDiscount','count':'count',
+    'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin',
+    'avg_shipping':'avgShipping','revenue':'revenue'
+}
+_M_DCODE = {
+    'year_month':'YearMonth','discount_code':'DiscountCode','uses':'uses',
+    'revenue':'revenue','avg_aov':'avgAOV','avg_discount_pct':'avgDiscountPct'
+}
+_M_SHIPCOMP = {
+    'year_month':'YearMonth','is_free_shipping':'IsFreeShipping','count':'count',
+    'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin',
+    'new_pct':'newPct','discount_pct':'discountPct','avg_shipping':'avgShipping'
+}
+_M_CAT = {
+    'year_month':'YearMonth','category':'Category','total_revenue':'totalRevenue',
+    'total_revenue_ex_vat':'totalRevenueExVAT','total_cost':'totalCost',
+    'total_qty':'totalQty','order_count':'orderCount','unique_products':'uniqueProducts',
+    'margin':'margin','margin_pct':'marginPct'
+}
+_M_PROD = {
+    'year_month':'YearMonth','description':'Description','sku_description':'SKUDescription',
+    'category':'Category','total_revenue':'totalRevenue','total_revenue_ex_vat':'totalRevenueExVAT',
+    'total_cost':'totalCost','total_qty':'totalQty','order_count':'orderCount',
+    'margin':'margin','margin_pct':'marginPct'
+}
+_M_CROSS = {'product1':'product1','product2':'product2','count':'count'}
+_M_COHORT = {
+    'cohort':'cohort','size':'size',
+    'm0':'m0','m1':'m1','m2':'m2','m3':'m3','m4':'m4','m5':'m5',
+    'm6':'m6','m7':'m7','m8':'m8','m9':'m9','m10':'m10','m11':'m11','m12':'m12'
+}
+_M_THRESH = {
+    'year_month':'YearMonth','total_orders':'totalOrders','total_shipping_rev':'totalShippingRev',
+    't15_eligible':'t15_eligible','t15_ship_absorbed':'t15_shipAbsorbed','t15_near':'t15_near','t15_near_gap':'t15_nearGap',
+    't20_eligible':'t20_eligible','t20_ship_absorbed':'t20_shipAbsorbed','t20_near':'t20_near','t20_near_gap':'t20_nearGap',
+    't25_eligible':'t25_eligible','t25_ship_absorbed':'t25_shipAbsorbed','t25_near':'t25_near','t25_near_gap':'t25_nearGap',
+    't30_eligible':'t30_eligible','t30_ship_absorbed':'t30_shipAbsorbed','t30_near':'t30_near','t30_near_gap':'t30_nearGap',
+    't35_eligible':'t35_eligible','t35_ship_absorbed':'t35_shipAbsorbed','t35_near':'t35_near','t35_near_gap':'t35_nearGap',
+    't40_eligible':'t40_eligible','t40_ship_absorbed':'t40_shipAbsorbed','t40_near':'t40_near','t40_near_gap':'t40_nearGap',
+    't50_eligible':'t50_eligible','t50_ship_absorbed':'t50_shipAbsorbed','t50_near':'t50_near','t50_near_gap':'t50_nearGap',
+}
+
+
+def get_dashboard_data(start=None, end=None):
+    """Return the full dashboard payload, filtered by date range.
+    Returns the same JSON shape the frontend RAW variable expects."""
+    w, p = _ym_clause(start, end)
+
+    result = {
+        'monthly':              _remap(_query(f"SELECT * FROM monthly WHERE {w} ORDER BY year_month", p), _M_MONTHLY),
+        'channelMonthly':       _remap(_query(f"SELECT year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct FROM channel_monthly WHERE {w} ORDER BY year_month", p), _M_CHANNEL),
+        'newReturningMonthly':  _remap(_query(f"SELECT year_month,CAST(is_new_customer AS INTEGER) as is_new_customer,orders,revenue FROM new_returning_monthly WHERE {w} ORDER BY year_month", p), _M_NR),
+        'dowMonthly':           _remap(_query(f"SELECT year_month,day_of_week,orders,revenue FROM dow_monthly WHERE {w} ORDER BY year_month", p), _M_DOW),
+        'paymentMonthly':       _remap(_query(f"SELECT year_month,payment_processor,orders,revenue FROM payment_monthly WHERE {w} ORDER BY year_month", p), _M_PAY),
+        'countryMonthly':       _remap(_query(f"SELECT year_month,customer_country,orders,revenue FROM country_monthly WHERE {w} ORDER BY year_month", p), _M_COUNTRY),
+        'deliveryMonthly':      _remap(_query(f"SELECT year_month,delivery_method,orders,revenue,avg_shipping,avg_aov FROM delivery_monthly WHERE {w} ORDER BY year_month", p), _M_DELIVERY),
+        'aovMonthly':           _remap(_query(f"SELECT year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship FROM aov_monthly WHERE {w} ORDER BY year_month", p), _M_AOV),
+        'discountMonthly':      _remap(_query(f"SELECT year_month,CAST(has_discount AS INTEGER) as has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue FROM discount_monthly WHERE {w} ORDER BY year_month", p), _M_DISC),
+        'discountCodesMonthly': _remap(_query(f"SELECT year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct FROM discount_codes_monthly WHERE {w} ORDER BY year_month", p), _M_DCODE),
+        'shippingCompMonthly':  _remap(_query(f"SELECT year_month,CAST(is_free_shipping AS INTEGER) as is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping FROM shipping_comp_monthly WHERE {w} ORDER BY year_month", p), _M_SHIPCOMP),
+        'categoryMonthly':      _remap(_query(f"SELECT year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct FROM category_monthly WHERE {w} ORDER BY year_month", p), _M_CAT),
+        'productMonthly':       _remap(_query(f"SELECT year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct FROM product_monthly WHERE {w} ORDER BY year_month", p), _M_PROD),
+        # Non-date-filtered
+        'crossSell':            _remap(_query("SELECT product1,product2,count FROM cross_sell ORDER BY count DESC"), _M_CROSS),
+        'cohortRetention':      _remap(_query("SELECT cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12 FROM cohort_retention ORDER BY cohort"), _M_COHORT),
+        'thresholdMonthly':     _remap(_query(f"SELECT * FROM threshold_monthly WHERE {w} ORDER BY year_month", p), _M_THRESH),
+        'meta': get_meta(),
+    }
+
+    # Strip 'id' key from all rows (serial PKs not needed by frontend)
+    for k, v in result.items():
+        if isinstance(v, list):
+            result[k] = [{kk: vv for kk, vv in row.items() if kk != 'id'} for row in v]
+
+    return result
+
+
+# ── Init ─────────────────────────────────────────────────────
+
+def init_db():
+    """Wait for DB, create schema, seed if needed. Uses advisory lock to prevent races."""
+    wait_for_db()
+    conn = get_conn()
+    cur = conn.cursor()
+    try:
+        # Advisory lock so only one worker seeds
+        cur.execute("SELECT pg_advisory_lock(42)")
+        init_schema()
+        seed_from_json()
+    finally:
+        cur.execute("SELECT pg_advisory_unlock(42)")
+        cur.close()
+        conn.close()

docker-compose.yml

@@ -6,10 +6,14 @@ services:
     build: .
     environment:
       - GEMINI_API_KEY=AIzaSyCHnesXLjPw-UgeZaQotut66bgjFdvy12E
+      - DATABASE_URL=postgresql://jv:jvpass@tasks.db:5432/justvitamin
     volumes:
       - jv-generated:/app/generated
     networks:
       - dokploy-network
+      - jv-internal
+    depends_on:
+      - db
     deploy:
       labels:
         - "traefik.enable=true"
@@ -21,9 +25,30 @@ services:
       restart_policy:
         condition: on-failure
 
+  db:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=jv
+      - POSTGRES_PASSWORD=jvpass
+      - POSTGRES_DB=justvitamin
+    volumes:
+      - jv-pgdata:/var/lib/postgresql/data
+    networks:
+      - jv-internal
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+      resources:
+        limits:
+          memory: 512M
+
 volumes:
   jv-generated:
+  jv-pgdata:
 
 networks:
   dokploy-network:
     external: true
+  jv-internal:
+    driver: overlay

requirements.txt

@@ -3,3 +3,4 @@ gunicorn==23.0.0
 requests==2.32.3
 beautifulsoup4==4.13.3
 google-genai==1.5.0
+psycopg2-binary==2.9.10

static/dashboard/index.html

@@ -114,6 +114,7 @@ tr:hover td{background:rgba(99,102,241,.04)}
 <button class="preset-btn" onclick="setPreset('all')" id="presetAll">All Time</button>
 </div>
 <div class="filter-info" id="filterInfo"></div>
+<div id="dbBadge" style="margin-left:8px"></div>
 </div>
 
 <div id="loadingMsg" class="loading"><span class="spinner"></span>Loading data…</div>
@@ -277,7 +278,8 @@ function applyFilter(){
   FSTART = s || null;
   FEND = e || null;
   document.querySelectorAll('.preset-btn').forEach(b=>b.classList.remove('active'));
-  buildAll();
+  // Re-fetch from PostgreSQL with server-side filtering
+  loadData(FSTART, FEND);
 }
 
 function resetFilter(){
@@ -286,7 +288,8 @@ function resetFilter(){
   document.getElementById('endDate').value = '';
   document.querySelectorAll('.preset-btn').forEach(b=>b.classList.remove('active'));
   document.getElementById('presetAll').classList.add('active');
-  buildAll();
+  // Re-fetch full dataset from PostgreSQL
+  loadData();
 }
 
 function setPreset(key){
@@ -314,7 +317,8 @@ function setPreset(key){
   }
   document.getElementById('startDate').value = FSTART || '';
   document.getElementById('endDate').value = FEND || '';
-  buildAll();
+  // Re-fetch from PostgreSQL with server-side filtering
+  loadData(FSTART, FEND);
 }
 
 // ═══════════════════════════════════════════════════════════
@@ -869,25 +873,41 @@ function buildCustomers(fm){
 // ═══════════════════════════════════════════════════════════
 // LOAD DATA
 // ═══════════════════════════════════════════════════════════
-async function loadData(){
+async function loadData(start, end){
   try {
-    const res = await fetch('jv_data.json');
+    // Build API URL with optional date filters — data served from PostgreSQL
+    let url = '/api/dashboard/data';
+    const params = [];
+    if(start) params.push('start=' + encodeURIComponent(start));
+    if(end) params.push('end=' + encodeURIComponent(end));
+    if(params.length) url += '?' + params.join('&');
+
+    const t0 = performance.now();
+    const res = await fetch(url);
     if(!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
     RAW = await res.json();
+    const elapsed = ((performance.now()-t0)/1000).toFixed(2);
 
     // Sort monthly data
     RAW.monthly.sort((a,b) => a.YearMonth.localeCompare(b.YearMonth));
 
-    // Set date picker min/max
+    // Set date picker min/max from meta (full range)
-    const allMonths = RAW.monthly.map(r=>r.YearMonth);
+    if(RAW.meta && RAW.meta.dateMin){
-    document.getElementById('startDate').min = allMonths[0];
+      document.getElementById('startDate').min = RAW.meta.dateMin;
-    document.getElementById('startDate').max = allMonths[allMonths.length-1];
+      document.getElementById('startDate').max = RAW.meta.dateMax;
-    document.getElementById('endDate').min = allMonths[0];
+      document.getElementById('endDate').min = RAW.meta.dateMin;
-    document.getElementById('endDate').max = allMonths[allMonths.length-1];
+      document.getElementById('endDate').max = RAW.meta.dateMax;
+    }
 
     document.getElementById('loadingMsg').style.display = 'none';
     document.getElementById('mainContent').style.display = 'block';
-    document.getElementById('presetAll').classList.add('active');
+    if(!start && !end) document.getElementById('presetAll').classList.add('active');
+
+    // Show data source badge
+    const src = RAW._source || 'json';
+    const qt = RAW._query_time || elapsed+'s';
+    const badge = document.getElementById('dbBadge');
+    if(badge) badge.innerHTML = `<span style="font-size:10px;color:var(--green);background:rgba(16,185,129,.1);padding:2px 8px;border-radius:4px;font-family:monospace">⚡ PostgreSQL · ${qt} · ${RAW.monthly.length} months</span>`;
 
     buildAll();
   } catch(err) {
@@ -895,9 +915,8 @@ async function loadData(){
       <div style="color:var(--red);font-size:16px;margin-bottom:12px">⚠ Failed to load data</div>
       <div style="color:var(--muted);font-size:13px">${err.message}</div>
       <div style="color:var(--muted);font-size:12px;margin-top:12px">
-        Make sure <code>jv_data.json</code> is in the same folder as this HTML file.<br>
+        Data is served from PostgreSQL via <code>/api/dashboard/data</code>.<br>
-        Run <code>python preprocess_jv_data.py</code> first to generate it.<br>
+        Check that the database service is running.
-        Then serve via a local server: <code>python -m http.server 8000</code>
       </div>
     `;
   }