From 3f2b6b6188d2bf971a9f7a66a67f07fc127414ed Mon Sep 17 00:00:00 2001 From: Omair Saleh Date: Mon, 2 Mar 2026 20:24:15 +0800 Subject: [PATCH] =?UTF-8?q?v3:=20PostgreSQL=20backend=20=E2=80=94=2020K=20?= =?UTF-8?q?rows=20seeded,=20server-side=20SQL=20filtering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added PostgreSQL 16 Alpine service to Docker Swarm stack - db.py: schema for 17 tables, auto-seed from jv_data.json on first boot - /api/dashboard/data?start=&end= — server-side SQL filtering All Time: 0.43s (was 4MB JSON download stuck loading) Filtered (12mo): 0.20s with ~90% less data transferred - Dashboard HTML patched: calls API instead of static JSON - Integer casting for IsNewCustomer/HasDiscount/IsFreeShipping - Advisory lock prevents race condition during parallel worker startup - Returning Revenue now shows correctly: £14.5M (75% of total) --- app.py | 60 +++- db.py | 570 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 25 ++ requirements.txt | 1 + static/dashboard/index.html | 49 +++- 5 files changed, 688 insertions(+), 17 deletions(-) create mode 100644 db.py diff --git a/app.py b/app.py index c819d76..9d66504 100644 --- a/app.py +++ b/app.py @@ -1,7 +1,8 @@ """JustVitamin × QuikCue — AI Content Engine -Live proposal site with real Gemini-powered demos.""" +Live proposal site with real Gemini-powered demos. +PostgreSQL-backed data dashboard.""" -import os, json, time, traceback +import os, json, time, traceback, logging from pathlib import Path from flask import (Flask, render_template, jsonify, request, send_from_directory, redirect) @@ -9,6 +10,9 @@ from flask import (Flask, render_template, jsonify, request, from scraper import scrape_product, scrape_competitor from ai_engine import (generate_asset_pack, competitor_xray, pdp_surgeon, optimise_pdp_copy, generate_product_images) +from db import init_db, get_dashboard_data, get_meta + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") app = Flask(__name__, static_folder="static", @@ -17,6 +21,13 @@ app = Flask(__name__, GEN_DIR = Path(__file__).parent / "generated" GEN_DIR.mkdir(exist_ok=True) +# ── Init DB on startup ─────────────────────────────────────── +with app.app_context(): + try: + init_db() + except Exception as e: + logging.error(f"DB init failed: {e}") + # ── Page routes ────────────────────────────────────────────── @app.route("/") @@ -32,6 +43,37 @@ def dashboard(): def dashboard_files(filename): return send_from_directory("static/dashboard", filename) + +# ── Dashboard API (Postgres-backed) ───────────────────────── + +@app.route("/api/dashboard/data") +def api_dashboard_data(): + """Return filtered dashboard data from PostgreSQL. + Query params: ?start=YYYY-MM&end=YYYY-MM + Replaces the 4MB static JSON with fast server-side SQL filtering.""" + start = request.args.get("start") or None + end = request.args.get("end") or None + try: + t0 = time.time() + data = get_dashboard_data(start, end) + data["_query_time"] = f"{time.time()-t0:.3f}s" + data["_source"] = "postgresql" + resp = jsonify(data) + resp.headers['Cache-Control'] = 'public, max-age=60' + return resp + except Exception as e: + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/dashboard/meta") +def api_dashboard_meta(): + """Return just the meta info (fast, tiny).""" + try: + return jsonify(get_meta()) + except Exception as e: + return jsonify({"error": str(e)}), 500 + @app.route("/proposal") def proposal(): return send_from_directory("static/proposal", "index.html") @@ -185,9 +227,23 @@ def api_generate_images(): @app.route("/api/health") def health(): + db_ok = False + try: + from db import get_conn + conn = get_conn() + cur = conn.cursor() + cur.execute("SELECT COUNT(*) FROM monthly") + count = cur.fetchone()[0] + cur.close() + conn.close() + db_ok = count > 0 + except Exception: + pass return jsonify({ "status": "ok", "gemini_key_set": bool(os.environ.get("GEMINI_API_KEY")), + "db_connected": db_ok, + "db_rows": count if db_ok else 0, }) diff --git a/db.py b/db.py new file mode 100644 index 0000000..d4f28de --- /dev/null +++ b/db.py @@ -0,0 +1,570 @@ +"""JustVitamin — PostgreSQL data layer. + +Creates tables, seeds from jv_data.json on first boot, +provides fast filtered queries for the dashboard API.""" + +import os, json, time, logging +from pathlib import Path + +import psycopg2 +from psycopg2.extras import execute_values, RealDictCursor + +log = logging.getLogger(__name__) + +DB_URL = os.environ.get( + "DATABASE_URL", + "postgresql://jv:jvpass@tasks.db:5432/justvitamin" +) + +# ── Connection ─────────────────────────────────────────────── + +def get_conn(): + return psycopg2.connect(DB_URL) + + +def wait_for_db(retries=30, delay=2): + """Block until Postgres is ready.""" + for i in range(retries): + try: + conn = get_conn() + conn.close() + log.info("✓ Database ready") + return True + except Exception: + if i < retries - 1: + log.info(f"Waiting for DB... ({i+1}/{retries})") + time.sleep(delay) + raise RuntimeError("Database not available after retries") + + +# ── Schema ─────────────────────────────────────────────────── + +SCHEMA = """ +-- Core monthly aggregates +CREATE TABLE IF NOT EXISTS monthly ( + year_month TEXT PRIMARY KEY, + revenue NUMERIC, orders INT, aov NUMERIC, customers INT, + new_customers INT, avg_shipping NUMERIC, free_ship_pct NUMERIC, + avg_margin NUMERIC, discount_pct NUMERIC, avg_items NUMERIC, + total_shipping NUMERIC, free_ship_orders INT, discount_orders INT, + total_items INT +); + +-- Channel breakdown by month +CREATE TABLE IF NOT EXISTS channel_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, referrer_source TEXT, + orders INT, revenue NUMERIC, avg_aov NUMERIC, + avg_items NUMERIC, new_pct NUMERIC, free_ship_pct NUMERIC, + discount_pct NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_channel_ym ON channel_monthly(year_month); + +-- New vs returning by month +CREATE TABLE IF NOT EXISTS new_returning_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, is_new_customer TEXT, + orders INT, revenue NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_nr_ym ON new_returning_monthly(year_month); + +-- Day of week by month +CREATE TABLE IF NOT EXISTS dow_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, day_of_week TEXT, + orders INT, revenue NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_dow_ym ON dow_monthly(year_month); + +-- Payment processor by month +CREATE TABLE IF NOT EXISTS payment_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, payment_processor TEXT, + orders INT, revenue NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_pay_ym ON payment_monthly(year_month); + +-- Country by month +CREATE TABLE IF NOT EXISTS country_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, customer_country TEXT, + orders INT, revenue NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_country_ym ON country_monthly(year_month); + +-- Delivery by month +CREATE TABLE IF NOT EXISTS delivery_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, delivery_method TEXT, + orders INT, revenue NUMERIC, avg_shipping NUMERIC, avg_aov NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_delivery_ym ON delivery_monthly(year_month); + +-- AOV buckets by month +CREATE TABLE IF NOT EXISTS aov_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, aov_bucket TEXT, + count INT, revenue NUMERIC, avg_shipping NUMERIC, pct_free_ship NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_aov_ym ON aov_monthly(year_month); + +-- Discount by month +CREATE TABLE IF NOT EXISTS discount_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, has_discount TEXT, + count INT, avg_aov NUMERIC, avg_items NUMERIC, + avg_margin NUMERIC, avg_shipping NUMERIC, revenue NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_disc_ym ON discount_monthly(year_month); + +-- Discount codes by month +CREATE TABLE IF NOT EXISTS discount_codes_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, discount_code TEXT, + uses INT, revenue NUMERIC, avg_aov NUMERIC, avg_discount_pct NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_dcode_ym ON discount_codes_monthly(year_month); + +-- Free ship comparison by month +CREATE TABLE IF NOT EXISTS shipping_comp_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, is_free_shipping TEXT, + count INT, avg_aov NUMERIC, avg_items NUMERIC, + avg_margin NUMERIC, new_pct NUMERIC, discount_pct NUMERIC, + avg_shipping NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_shipcomp_ym ON shipping_comp_monthly(year_month); + +-- Category by month +CREATE TABLE IF NOT EXISTS category_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, category TEXT, + total_revenue NUMERIC, total_revenue_ex_vat NUMERIC, + total_cost NUMERIC, total_qty INT, order_count INT, + unique_products INT, margin NUMERIC, margin_pct NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_cat_ym ON category_monthly(year_month); + +-- Product by month +CREATE TABLE IF NOT EXISTS product_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, description TEXT, sku_description TEXT, + category TEXT, total_revenue NUMERIC, total_revenue_ex_vat NUMERIC, + total_cost NUMERIC, total_qty INT, order_count INT, + margin NUMERIC, margin_pct NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_prod_ym ON product_monthly(year_month); + +-- Cross-sell pairs (no month dimension) +CREATE TABLE IF NOT EXISTS cross_sell ( + id SERIAL PRIMARY KEY, + product1 TEXT, product2 TEXT, count INT +); + +-- Cohort retention +CREATE TABLE IF NOT EXISTS cohort_retention ( + id SERIAL PRIMARY KEY, + cohort TEXT, size INT, + m0 NUMERIC, m1 NUMERIC, m2 NUMERIC, m3 NUMERIC, + m4 NUMERIC, m5 NUMERIC, m6 NUMERIC, m7 NUMERIC, + m8 NUMERIC, m9 NUMERIC, m10 NUMERIC, m11 NUMERIC, + m12 NUMERIC +); + +-- Free shipping threshold by month +CREATE TABLE IF NOT EXISTS threshold_monthly ( + id SERIAL PRIMARY KEY, + year_month TEXT, total_orders INT, total_shipping_rev NUMERIC, + t15_eligible NUMERIC, t15_ship_absorbed NUMERIC, t15_near NUMERIC, t15_near_gap NUMERIC, + t20_eligible NUMERIC, t20_ship_absorbed NUMERIC, t20_near NUMERIC, t20_near_gap NUMERIC, + t25_eligible NUMERIC, t25_ship_absorbed NUMERIC, t25_near NUMERIC, t25_near_gap NUMERIC, + t30_eligible NUMERIC, t30_ship_absorbed NUMERIC, t30_near NUMERIC, t30_near_gap NUMERIC, + t35_eligible NUMERIC, t35_ship_absorbed NUMERIC, t35_near NUMERIC, t35_near_gap NUMERIC, + t40_eligible NUMERIC, t40_ship_absorbed NUMERIC, t40_near NUMERIC, t40_near_gap NUMERIC, + t50_eligible NUMERIC, t50_ship_absorbed NUMERIC, t50_near NUMERIC, t50_near_gap NUMERIC +); +CREATE INDEX IF NOT EXISTS idx_thresh_ym ON threshold_monthly(year_month); + +-- Metadata +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT +); +""" + + +def init_schema(): + conn = get_conn() + cur = conn.cursor() + try: + cur.execute(SCHEMA) + conn.commit() + log.info("✓ Schema created") + except Exception as e: + conn.rollback() + if "already exists" in str(e): + log.info("✓ Schema already exists") + else: + raise + finally: + cur.close() + conn.close() + + +# ── Seed ───────────────────────────────────────────────────── + +def is_seeded(): + conn = get_conn() + cur = conn.cursor() + cur.execute("SELECT COUNT(*) FROM monthly") + n = cur.fetchone()[0] + cur.close() + conn.close() + return n > 0 + + +def seed_from_json(path=None): + """Load jv_data.json into Postgres. Idempotent — skips if data exists.""" + if is_seeded(): + log.info("✓ Database already seeded — skipping") + return + + if path is None: + path = Path(__file__).parent / "static" / "dashboard" / "jv_data.json" + + log.info(f"Seeding from {path} ...") + t0 = time.time() + + with open(path) as f: + data = json.load(f) + + conn = get_conn() + cur = conn.cursor() + + # 1. monthly + rows = [(r['YearMonth'], r.get('revenue'), r.get('orders'), r.get('aov'), + r.get('customers'), r.get('newCustomers'), r.get('avgShipping'), + r.get('freeShipPct'), r.get('avgMargin'), r.get('discountPct'), + r.get('avgItems'), r.get('totalShipping'), r.get('freeShipOrders'), + r.get('discountOrders'), r.get('totalItems')) + for r in data['monthly']] + execute_values(cur, + "INSERT INTO monthly VALUES %s ON CONFLICT DO NOTHING", + rows) + + # 2. channelMonthly + rows = [(r['YearMonth'], r.get('ReferrerSource'), r.get('orders'), + r.get('revenue'), r.get('avgAOV'), r.get('avgItems'), + r.get('newPct'), r.get('freeShipPct'), r.get('discountPct')) + for r in data['channelMonthly']] + execute_values(cur, + "INSERT INTO channel_monthly (year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct) VALUES %s", + rows) + + # 3. newReturningMonthly + rows = [(r['YearMonth'], r.get('IsNewCustomer'), r.get('orders'), r.get('revenue')) + for r in data['newReturningMonthly']] + execute_values(cur, + "INSERT INTO new_returning_monthly (year_month,is_new_customer,orders,revenue) VALUES %s", + rows) + + # 4. dowMonthly + rows = [(r['YearMonth'], r.get('DayOfWeek'), r.get('orders'), r.get('revenue')) + for r in data['dowMonthly']] + execute_values(cur, + "INSERT INTO dow_monthly (year_month,day_of_week,orders,revenue) VALUES %s", + rows) + + # 5. paymentMonthly + rows = [(r['YearMonth'], r.get('PaymentProcessor'), r.get('orders'), r.get('revenue')) + for r in data['paymentMonthly']] + execute_values(cur, + "INSERT INTO payment_monthly (year_month,payment_processor,orders,revenue) VALUES %s", + rows) + + # 6. countryMonthly + rows = [(r['YearMonth'], r.get('CustomerCountry'), r.get('orders'), r.get('revenue')) + for r in data['countryMonthly']] + execute_values(cur, + "INSERT INTO country_monthly (year_month,customer_country,orders,revenue) VALUES %s", + rows) + + # 7. deliveryMonthly + rows = [(r['YearMonth'], r.get('DeliveryMethod'), r.get('orders'), + r.get('revenue'), r.get('avgShipping'), r.get('avgAOV')) + for r in data['deliveryMonthly']] + execute_values(cur, + "INSERT INTO delivery_monthly (year_month,delivery_method,orders,revenue,avg_shipping,avg_aov) VALUES %s", + rows) + + # 8. aovMonthly + rows = [(r['YearMonth'], r.get('AOVBucket'), r.get('count'), + r.get('revenue'), r.get('avgShipping'), r.get('pctFreeShip')) + for r in data['aovMonthly']] + execute_values(cur, + "INSERT INTO aov_monthly (year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship) VALUES %s", + rows) + + # 9. discountMonthly + rows = [(r['YearMonth'], r.get('HasDiscount'), r.get('count'), + r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'), + r.get('avgShipping'), r.get('revenue')) + for r in data['discountMonthly']] + execute_values(cur, + "INSERT INTO discount_monthly (year_month,has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue) VALUES %s", + rows) + + # 10. discountCodesMonthly + rows = [(r['YearMonth'], r.get('DiscountCode'), r.get('uses'), + r.get('revenue'), r.get('avgAOV'), r.get('avgDiscountPct')) + for r in data['discountCodesMonthly']] + execute_values(cur, + "INSERT INTO discount_codes_monthly (year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct) VALUES %s", + rows) + + # 11. shippingCompMonthly + rows = [(r['YearMonth'], r.get('IsFreeShipping'), r.get('count'), + r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'), + r.get('newPct'), r.get('discountPct'), r.get('avgShipping')) + for r in data['shippingCompMonthly']] + execute_values(cur, + "INSERT INTO shipping_comp_monthly (year_month,is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping) VALUES %s", + rows) + + # 12. categoryMonthly + rows = [(r['YearMonth'], r.get('Category'), r.get('totalRevenue'), + r.get('totalRevenueExVAT'), r.get('totalCost'), r.get('totalQty'), + r.get('orderCount'), r.get('uniqueProducts'), + r.get('margin'), r.get('marginPct')) + for r in data['categoryMonthly']] + execute_values(cur, + "INSERT INTO category_monthly (year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct) VALUES %s", + rows) + + # 13. productMonthly + rows = [(r['YearMonth'], r.get('Description'), r.get('SKUDescription'), + r.get('Category'), r.get('totalRevenue'), r.get('totalRevenueExVAT'), + r.get('totalCost'), r.get('totalQty'), r.get('orderCount'), + r.get('margin'), r.get('marginPct')) + for r in data['productMonthly']] + execute_values(cur, + "INSERT INTO product_monthly (year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct) VALUES %s", + rows) + + # 14. crossSell + rows = [(r.get('product1'), r.get('product2'), r.get('count')) + for r in data['crossSell']] + execute_values(cur, + "INSERT INTO cross_sell (product1,product2,count) VALUES %s", + rows) + + # 15. cohortRetention + rows = [(r.get('cohort'), r.get('size'), + r.get('m0'), r.get('m1'), r.get('m2'), r.get('m3'), + r.get('m4'), r.get('m5'), r.get('m6'), r.get('m7'), + r.get('m8'), r.get('m9'), r.get('m10'), r.get('m11'), + r.get('m12')) + for r in data['cohortRetention']] + execute_values(cur, + "INSERT INTO cohort_retention (cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12) VALUES %s", + rows) + + # 16. thresholdMonthly + rows = [(r['YearMonth'], r.get('totalOrders'), r.get('totalShippingRev'), + r.get('t15_eligible'), r.get('t15_shipAbsorbed'), r.get('t15_near'), r.get('t15_nearGap'), + r.get('t20_eligible'), r.get('t20_shipAbsorbed'), r.get('t20_near'), r.get('t20_nearGap'), + r.get('t25_eligible'), r.get('t25_shipAbsorbed'), r.get('t25_near'), r.get('t25_nearGap'), + r.get('t30_eligible'), r.get('t30_shipAbsorbed'), r.get('t30_near'), r.get('t30_nearGap'), + r.get('t35_eligible'), r.get('t35_shipAbsorbed'), r.get('t35_near'), r.get('t35_nearGap'), + r.get('t40_eligible'), r.get('t40_shipAbsorbed'), r.get('t40_near'), r.get('t40_nearGap'), + r.get('t50_eligible'), r.get('t50_shipAbsorbed'), r.get('t50_near'), r.get('t50_nearGap')) + for r in data['thresholdMonthly']] + execute_values(cur, + "INSERT INTO threshold_monthly (year_month,total_orders,total_shipping_rev,t15_eligible,t15_ship_absorbed,t15_near,t15_near_gap,t20_eligible,t20_ship_absorbed,t20_near,t20_near_gap,t25_eligible,t25_ship_absorbed,t25_near,t25_near_gap,t30_eligible,t30_ship_absorbed,t30_near,t30_near_gap,t35_eligible,t35_ship_absorbed,t35_near,t35_near_gap,t40_eligible,t40_ship_absorbed,t40_near,t40_near_gap,t50_eligible,t50_ship_absorbed,t50_near,t50_near_gap) VALUES %s", + rows) + + # 17. meta + meta = data.get('meta', {}) + for k, v in meta.items(): + cur.execute( + "INSERT INTO meta (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + (k, json.dumps(v) if not isinstance(v, str) else v) + ) + + conn.commit() + cur.close() + conn.close() + + elapsed = time.time() - t0 + log.info(f"✓ Seeded {sum(len(data[k]) for k in data if isinstance(data[k], list))} rows in {elapsed:.1f}s") + + +# ── Queries ────────────────────────────────────────────────── + +def _ym_clause(start, end, col="year_month"): + """Build WHERE clause for date range.""" + parts = [] + params = [] + if start: + parts.append(f"{col} >= %s") + params.append(start) + if end: + parts.append(f"{col} <= %s") + params.append(end) + where = " AND ".join(parts) if parts else "TRUE" + return where, params + + +def _query(sql, params=()): + conn = get_conn() + cur = conn.cursor(cursor_factory=RealDictCursor) + cur.execute(sql, params) + rows = cur.fetchall() + cur.close() + conn.close() + # Convert Decimal to float for JSON serialisation + return [_dec_to_float(r) for r in rows] + + +def _dec_to_float(d): + from decimal import Decimal + return {k: (float(v) if isinstance(v, Decimal) else v) for k, v in d.items()} + + +def get_meta(): + rows = _query("SELECT key, value FROM meta") + meta = {} + for r in rows: + v = r['value'] + try: + v = json.loads(v) + except (json.JSONDecodeError, TypeError): + pass + meta[r['key']] = v + return meta + + +def _remap(rows, mapping): + """Rename DB snake_case keys back to the camelCase the dashboard JS expects.""" + return [{mapping.get(k, k): v for k, v in r.items()} for r in rows] + + +# Column mappings: DB name → JS name +_M_MONTHLY = { + 'year_month':'YearMonth','revenue':'revenue','orders':'orders','aov':'aov', + 'customers':'customers','new_customers':'newCustomers','avg_shipping':'avgShipping', + 'free_ship_pct':'freeShipPct','avg_margin':'avgMargin','discount_pct':'discountPct', + 'avg_items':'avgItems','total_shipping':'totalShipping','free_ship_orders':'freeShipOrders', + 'discount_orders':'discountOrders','total_items':'totalItems' +} +_M_CHANNEL = { + 'year_month':'YearMonth','referrer_source':'ReferrerSource','orders':'orders', + 'revenue':'revenue','avg_aov':'avgAOV','avg_items':'avgItems', + 'new_pct':'newPct','free_ship_pct':'freeShipPct','discount_pct':'discountPct' +} +_M_NR = {'year_month':'YearMonth','is_new_customer':'IsNewCustomer','orders':'orders','revenue':'revenue'} +_M_DOW = {'year_month':'YearMonth','day_of_week':'DayOfWeek','orders':'orders','revenue':'revenue'} +_M_PAY = {'year_month':'YearMonth','payment_processor':'PaymentProcessor','orders':'orders','revenue':'revenue'} +_M_COUNTRY = {'year_month':'YearMonth','customer_country':'CustomerCountry','orders':'orders','revenue':'revenue'} +_M_DELIVERY = { + 'year_month':'YearMonth','delivery_method':'DeliveryMethod','orders':'orders', + 'revenue':'revenue','avg_shipping':'avgShipping','avg_aov':'avgAOV' +} +_M_AOV = { + 'year_month':'YearMonth','aov_bucket':'AOVBucket','count':'count', + 'revenue':'revenue','avg_shipping':'avgShipping','pct_free_ship':'pctFreeShip' +} +_M_DISC = { + 'year_month':'YearMonth','has_discount':'HasDiscount','count':'count', + 'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin', + 'avg_shipping':'avgShipping','revenue':'revenue' +} +_M_DCODE = { + 'year_month':'YearMonth','discount_code':'DiscountCode','uses':'uses', + 'revenue':'revenue','avg_aov':'avgAOV','avg_discount_pct':'avgDiscountPct' +} +_M_SHIPCOMP = { + 'year_month':'YearMonth','is_free_shipping':'IsFreeShipping','count':'count', + 'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin', + 'new_pct':'newPct','discount_pct':'discountPct','avg_shipping':'avgShipping' +} +_M_CAT = { + 'year_month':'YearMonth','category':'Category','total_revenue':'totalRevenue', + 'total_revenue_ex_vat':'totalRevenueExVAT','total_cost':'totalCost', + 'total_qty':'totalQty','order_count':'orderCount','unique_products':'uniqueProducts', + 'margin':'margin','margin_pct':'marginPct' +} +_M_PROD = { + 'year_month':'YearMonth','description':'Description','sku_description':'SKUDescription', + 'category':'Category','total_revenue':'totalRevenue','total_revenue_ex_vat':'totalRevenueExVAT', + 'total_cost':'totalCost','total_qty':'totalQty','order_count':'orderCount', + 'margin':'margin','margin_pct':'marginPct' +} +_M_CROSS = {'product1':'product1','product2':'product2','count':'count'} +_M_COHORT = { + 'cohort':'cohort','size':'size', + 'm0':'m0','m1':'m1','m2':'m2','m3':'m3','m4':'m4','m5':'m5', + 'm6':'m6','m7':'m7','m8':'m8','m9':'m9','m10':'m10','m11':'m11','m12':'m12' +} +_M_THRESH = { + 'year_month':'YearMonth','total_orders':'totalOrders','total_shipping_rev':'totalShippingRev', + 't15_eligible':'t15_eligible','t15_ship_absorbed':'t15_shipAbsorbed','t15_near':'t15_near','t15_near_gap':'t15_nearGap', + 't20_eligible':'t20_eligible','t20_ship_absorbed':'t20_shipAbsorbed','t20_near':'t20_near','t20_near_gap':'t20_nearGap', + 't25_eligible':'t25_eligible','t25_ship_absorbed':'t25_shipAbsorbed','t25_near':'t25_near','t25_near_gap':'t25_nearGap', + 't30_eligible':'t30_eligible','t30_ship_absorbed':'t30_shipAbsorbed','t30_near':'t30_near','t30_near_gap':'t30_nearGap', + 't35_eligible':'t35_eligible','t35_ship_absorbed':'t35_shipAbsorbed','t35_near':'t35_near','t35_near_gap':'t35_nearGap', + 't40_eligible':'t40_eligible','t40_ship_absorbed':'t40_shipAbsorbed','t40_near':'t40_near','t40_near_gap':'t40_nearGap', + 't50_eligible':'t50_eligible','t50_ship_absorbed':'t50_shipAbsorbed','t50_near':'t50_near','t50_near_gap':'t50_nearGap', +} + + +def get_dashboard_data(start=None, end=None): + """Return the full dashboard payload, filtered by date range. + Returns the same JSON shape the frontend RAW variable expects.""" + w, p = _ym_clause(start, end) + + result = { + 'monthly': _remap(_query(f"SELECT * FROM monthly WHERE {w} ORDER BY year_month", p), _M_MONTHLY), + 'channelMonthly': _remap(_query(f"SELECT year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct FROM channel_monthly WHERE {w} ORDER BY year_month", p), _M_CHANNEL), + 'newReturningMonthly': _remap(_query(f"SELECT year_month,CAST(is_new_customer AS INTEGER) as is_new_customer,orders,revenue FROM new_returning_monthly WHERE {w} ORDER BY year_month", p), _M_NR), + 'dowMonthly': _remap(_query(f"SELECT year_month,day_of_week,orders,revenue FROM dow_monthly WHERE {w} ORDER BY year_month", p), _M_DOW), + 'paymentMonthly': _remap(_query(f"SELECT year_month,payment_processor,orders,revenue FROM payment_monthly WHERE {w} ORDER BY year_month", p), _M_PAY), + 'countryMonthly': _remap(_query(f"SELECT year_month,customer_country,orders,revenue FROM country_monthly WHERE {w} ORDER BY year_month", p), _M_COUNTRY), + 'deliveryMonthly': _remap(_query(f"SELECT year_month,delivery_method,orders,revenue,avg_shipping,avg_aov FROM delivery_monthly WHERE {w} ORDER BY year_month", p), _M_DELIVERY), + 'aovMonthly': _remap(_query(f"SELECT year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship FROM aov_monthly WHERE {w} ORDER BY year_month", p), _M_AOV), + 'discountMonthly': _remap(_query(f"SELECT year_month,CAST(has_discount AS INTEGER) as has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue FROM discount_monthly WHERE {w} ORDER BY year_month", p), _M_DISC), + 'discountCodesMonthly': _remap(_query(f"SELECT year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct FROM discount_codes_monthly WHERE {w} ORDER BY year_month", p), _M_DCODE), + 'shippingCompMonthly': _remap(_query(f"SELECT year_month,CAST(is_free_shipping AS INTEGER) as is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping FROM shipping_comp_monthly WHERE {w} ORDER BY year_month", p), _M_SHIPCOMP), + 'categoryMonthly': _remap(_query(f"SELECT year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct FROM category_monthly WHERE {w} ORDER BY year_month", p), _M_CAT), + 'productMonthly': _remap(_query(f"SELECT year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct FROM product_monthly WHERE {w} ORDER BY year_month", p), _M_PROD), + # Non-date-filtered + 'crossSell': _remap(_query("SELECT product1,product2,count FROM cross_sell ORDER BY count DESC"), _M_CROSS), + 'cohortRetention': _remap(_query("SELECT cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12 FROM cohort_retention ORDER BY cohort"), _M_COHORT), + 'thresholdMonthly': _remap(_query(f"SELECT * FROM threshold_monthly WHERE {w} ORDER BY year_month", p), _M_THRESH), + 'meta': get_meta(), + } + + # Strip 'id' key from all rows (serial PKs not needed by frontend) + for k, v in result.items(): + if isinstance(v, list): + result[k] = [{kk: vv for kk, vv in row.items() if kk != 'id'} for row in v] + + return result + + +# ── Init ───────────────────────────────────────────────────── + +def init_db(): + """Wait for DB, create schema, seed if needed. Uses advisory lock to prevent races.""" + wait_for_db() + conn = get_conn() + cur = conn.cursor() + try: + # Advisory lock so only one worker seeds + cur.execute("SELECT pg_advisory_lock(42)") + init_schema() + seed_from_json() + finally: + cur.execute("SELECT pg_advisory_unlock(42)") + cur.close() + conn.close() diff --git a/docker-compose.yml b/docker-compose.yml index 4516ca4..2590645 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,10 +6,14 @@ services: build: . environment: - GEMINI_API_KEY=AIzaSyCHnesXLjPw-UgeZaQotut66bgjFdvy12E + - DATABASE_URL=postgresql://jv:jvpass@tasks.db:5432/justvitamin volumes: - jv-generated:/app/generated networks: - dokploy-network + - jv-internal + depends_on: + - db deploy: labels: - "traefik.enable=true" @@ -21,9 +25,30 @@ services: restart_policy: condition: on-failure + db: + image: postgres:16-alpine + environment: + - POSTGRES_USER=jv + - POSTGRES_PASSWORD=jvpass + - POSTGRES_DB=justvitamin + volumes: + - jv-pgdata:/var/lib/postgresql/data + networks: + - jv-internal + deploy: + replicas: 1 + restart_policy: + condition: on-failure + resources: + limits: + memory: 512M + volumes: jv-generated: + jv-pgdata: networks: dokploy-network: external: true + jv-internal: + driver: overlay diff --git a/requirements.txt b/requirements.txt index 4fb19d4..781cfef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ gunicorn==23.0.0 requests==2.32.3 beautifulsoup4==4.13.3 google-genai==1.5.0 +psycopg2-binary==2.9.10 diff --git a/static/dashboard/index.html b/static/dashboard/index.html index 1b84387..87d7362 100644 --- a/static/dashboard/index.html +++ b/static/dashboard/index.html @@ -114,6 +114,7 @@ tr:hover td{background:rgba(99,102,241,.04)}
+
Loading data…
@@ -277,7 +278,8 @@ function applyFilter(){ FSTART = s || null; FEND = e || null; document.querySelectorAll('.preset-btn').forEach(b=>b.classList.remove('active')); - buildAll(); + // Re-fetch from PostgreSQL with server-side filtering + loadData(FSTART, FEND); } function resetFilter(){ @@ -286,7 +288,8 @@ function resetFilter(){ document.getElementById('endDate').value = ''; document.querySelectorAll('.preset-btn').forEach(b=>b.classList.remove('active')); document.getElementById('presetAll').classList.add('active'); - buildAll(); + // Re-fetch full dataset from PostgreSQL + loadData(); } function setPreset(key){ @@ -314,7 +317,8 @@ function setPreset(key){ } document.getElementById('startDate').value = FSTART || ''; document.getElementById('endDate').value = FEND || ''; - buildAll(); + // Re-fetch from PostgreSQL with server-side filtering + loadData(FSTART, FEND); } // ═══════════════════════════════════════════════════════════ @@ -869,25 +873,41 @@ function buildCustomers(fm){ // ═══════════════════════════════════════════════════════════ // LOAD DATA // ═══════════════════════════════════════════════════════════ -async function loadData(){ +async function loadData(start, end){ try { - const res = await fetch('jv_data.json'); + // Build API URL with optional date filters — data served from PostgreSQL + let url = '/api/dashboard/data'; + const params = []; + if(start) params.push('start=' + encodeURIComponent(start)); + if(end) params.push('end=' + encodeURIComponent(end)); + if(params.length) url += '?' + params.join('&'); + + const t0 = performance.now(); + const res = await fetch(url); if(!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`); RAW = await res.json(); + const elapsed = ((performance.now()-t0)/1000).toFixed(2); // Sort monthly data RAW.monthly.sort((a,b) => a.YearMonth.localeCompare(b.YearMonth)); - // Set date picker min/max - const allMonths = RAW.monthly.map(r=>r.YearMonth); - document.getElementById('startDate').min = allMonths[0]; - document.getElementById('startDate').max = allMonths[allMonths.length-1]; - document.getElementById('endDate').min = allMonths[0]; - document.getElementById('endDate').max = allMonths[allMonths.length-1]; + // Set date picker min/max from meta (full range) + if(RAW.meta && RAW.meta.dateMin){ + document.getElementById('startDate').min = RAW.meta.dateMin; + document.getElementById('startDate').max = RAW.meta.dateMax; + document.getElementById('endDate').min = RAW.meta.dateMin; + document.getElementById('endDate').max = RAW.meta.dateMax; + } document.getElementById('loadingMsg').style.display = 'none'; document.getElementById('mainContent').style.display = 'block'; - document.getElementById('presetAll').classList.add('active'); + if(!start && !end) document.getElementById('presetAll').classList.add('active'); + + // Show data source badge + const src = RAW._source || 'json'; + const qt = RAW._query_time || elapsed+'s'; + const badge = document.getElementById('dbBadge'); + if(badge) badge.innerHTML = `⚡ PostgreSQL · ${qt} · ${RAW.monthly.length} months`; buildAll(); } catch(err) { @@ -895,9 +915,8 @@ async function loadData(){
⚠ Failed to load data
${err.message}
- Make sure jv_data.json is in the same folder as this HTML file.
- Run python preprocess_jv_data.py first to generate it.
- Then serve via a local server: python -m http.server 8000 + Data is served from PostgreSQL via /api/dashboard/data.
+ Check that the database service is running.
`; }