v3: PostgreSQL backend — 20K rows seeded, server-side SQL filtering
- Added PostgreSQL 16 Alpine service to Docker Swarm stack - db.py: schema for 17 tables, auto-seed from jv_data.json on first boot - /api/dashboard/data?start=&end= — server-side SQL filtering All Time: 0.43s (was 4MB JSON download stuck loading) Filtered (12mo): 0.20s with ~90% less data transferred - Dashboard HTML patched: calls API instead of static JSON - Integer casting for IsNewCustomer/HasDiscount/IsFreeShipping - Advisory lock prevents race condition during parallel worker startup - Returning Revenue now shows correctly: £14.5M (75% of total)
This commit is contained in:
570
db.py
Normal file
570
db.py
Normal file
@@ -0,0 +1,570 @@
|
||||
"""JustVitamin — PostgreSQL data layer.
|
||||
|
||||
Creates tables, seeds from jv_data.json on first boot,
|
||||
provides fast filtered queries for the dashboard API."""
|
||||
|
||||
import os, json, time, logging
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg2
|
||||
from psycopg2.extras import execute_values, RealDictCursor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB_URL = os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql://jv:jvpass@tasks.db:5432/justvitamin"
|
||||
)
|
||||
|
||||
# ── Connection ───────────────────────────────────────────────
|
||||
|
||||
def get_conn():
|
||||
return psycopg2.connect(DB_URL)
|
||||
|
||||
|
||||
def wait_for_db(retries=30, delay=2):
|
||||
"""Block until Postgres is ready."""
|
||||
for i in range(retries):
|
||||
try:
|
||||
conn = get_conn()
|
||||
conn.close()
|
||||
log.info("✓ Database ready")
|
||||
return True
|
||||
except Exception:
|
||||
if i < retries - 1:
|
||||
log.info(f"Waiting for DB... ({i+1}/{retries})")
|
||||
time.sleep(delay)
|
||||
raise RuntimeError("Database not available after retries")
|
||||
|
||||
|
||||
# ── Schema ───────────────────────────────────────────────────
|
||||
|
||||
SCHEMA = """
|
||||
-- Core monthly aggregates
|
||||
CREATE TABLE IF NOT EXISTS monthly (
|
||||
year_month TEXT PRIMARY KEY,
|
||||
revenue NUMERIC, orders INT, aov NUMERIC, customers INT,
|
||||
new_customers INT, avg_shipping NUMERIC, free_ship_pct NUMERIC,
|
||||
avg_margin NUMERIC, discount_pct NUMERIC, avg_items NUMERIC,
|
||||
total_shipping NUMERIC, free_ship_orders INT, discount_orders INT,
|
||||
total_items INT
|
||||
);
|
||||
|
||||
-- Channel breakdown by month
|
||||
CREATE TABLE IF NOT EXISTS channel_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, referrer_source TEXT,
|
||||
orders INT, revenue NUMERIC, avg_aov NUMERIC,
|
||||
avg_items NUMERIC, new_pct NUMERIC, free_ship_pct NUMERIC,
|
||||
discount_pct NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_channel_ym ON channel_monthly(year_month);
|
||||
|
||||
-- New vs returning by month
|
||||
CREATE TABLE IF NOT EXISTS new_returning_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, is_new_customer TEXT,
|
||||
orders INT, revenue NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_nr_ym ON new_returning_monthly(year_month);
|
||||
|
||||
-- Day of week by month
|
||||
CREATE TABLE IF NOT EXISTS dow_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, day_of_week TEXT,
|
||||
orders INT, revenue NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dow_ym ON dow_monthly(year_month);
|
||||
|
||||
-- Payment processor by month
|
||||
CREATE TABLE IF NOT EXISTS payment_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, payment_processor TEXT,
|
||||
orders INT, revenue NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pay_ym ON payment_monthly(year_month);
|
||||
|
||||
-- Country by month
|
||||
CREATE TABLE IF NOT EXISTS country_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, customer_country TEXT,
|
||||
orders INT, revenue NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_country_ym ON country_monthly(year_month);
|
||||
|
||||
-- Delivery by month
|
||||
CREATE TABLE IF NOT EXISTS delivery_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, delivery_method TEXT,
|
||||
orders INT, revenue NUMERIC, avg_shipping NUMERIC, avg_aov NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_delivery_ym ON delivery_monthly(year_month);
|
||||
|
||||
-- AOV buckets by month
|
||||
CREATE TABLE IF NOT EXISTS aov_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, aov_bucket TEXT,
|
||||
count INT, revenue NUMERIC, avg_shipping NUMERIC, pct_free_ship NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_aov_ym ON aov_monthly(year_month);
|
||||
|
||||
-- Discount by month
|
||||
CREATE TABLE IF NOT EXISTS discount_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, has_discount TEXT,
|
||||
count INT, avg_aov NUMERIC, avg_items NUMERIC,
|
||||
avg_margin NUMERIC, avg_shipping NUMERIC, revenue NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_disc_ym ON discount_monthly(year_month);
|
||||
|
||||
-- Discount codes by month
|
||||
CREATE TABLE IF NOT EXISTS discount_codes_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, discount_code TEXT,
|
||||
uses INT, revenue NUMERIC, avg_aov NUMERIC, avg_discount_pct NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dcode_ym ON discount_codes_monthly(year_month);
|
||||
|
||||
-- Free ship comparison by month
|
||||
CREATE TABLE IF NOT EXISTS shipping_comp_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, is_free_shipping TEXT,
|
||||
count INT, avg_aov NUMERIC, avg_items NUMERIC,
|
||||
avg_margin NUMERIC, new_pct NUMERIC, discount_pct NUMERIC,
|
||||
avg_shipping NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_shipcomp_ym ON shipping_comp_monthly(year_month);
|
||||
|
||||
-- Category by month
|
||||
CREATE TABLE IF NOT EXISTS category_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, category TEXT,
|
||||
total_revenue NUMERIC, total_revenue_ex_vat NUMERIC,
|
||||
total_cost NUMERIC, total_qty INT, order_count INT,
|
||||
unique_products INT, margin NUMERIC, margin_pct NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cat_ym ON category_monthly(year_month);
|
||||
|
||||
-- Product by month
|
||||
CREATE TABLE IF NOT EXISTS product_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, description TEXT, sku_description TEXT,
|
||||
category TEXT, total_revenue NUMERIC, total_revenue_ex_vat NUMERIC,
|
||||
total_cost NUMERIC, total_qty INT, order_count INT,
|
||||
margin NUMERIC, margin_pct NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_prod_ym ON product_monthly(year_month);
|
||||
|
||||
-- Cross-sell pairs (no month dimension)
|
||||
CREATE TABLE IF NOT EXISTS cross_sell (
|
||||
id SERIAL PRIMARY KEY,
|
||||
product1 TEXT, product2 TEXT, count INT
|
||||
);
|
||||
|
||||
-- Cohort retention
|
||||
CREATE TABLE IF NOT EXISTS cohort_retention (
|
||||
id SERIAL PRIMARY KEY,
|
||||
cohort TEXT, size INT,
|
||||
m0 NUMERIC, m1 NUMERIC, m2 NUMERIC, m3 NUMERIC,
|
||||
m4 NUMERIC, m5 NUMERIC, m6 NUMERIC, m7 NUMERIC,
|
||||
m8 NUMERIC, m9 NUMERIC, m10 NUMERIC, m11 NUMERIC,
|
||||
m12 NUMERIC
|
||||
);
|
||||
|
||||
-- Free shipping threshold by month
|
||||
CREATE TABLE IF NOT EXISTS threshold_monthly (
|
||||
id SERIAL PRIMARY KEY,
|
||||
year_month TEXT, total_orders INT, total_shipping_rev NUMERIC,
|
||||
t15_eligible NUMERIC, t15_ship_absorbed NUMERIC, t15_near NUMERIC, t15_near_gap NUMERIC,
|
||||
t20_eligible NUMERIC, t20_ship_absorbed NUMERIC, t20_near NUMERIC, t20_near_gap NUMERIC,
|
||||
t25_eligible NUMERIC, t25_ship_absorbed NUMERIC, t25_near NUMERIC, t25_near_gap NUMERIC,
|
||||
t30_eligible NUMERIC, t30_ship_absorbed NUMERIC, t30_near NUMERIC, t30_near_gap NUMERIC,
|
||||
t35_eligible NUMERIC, t35_ship_absorbed NUMERIC, t35_near NUMERIC, t35_near_gap NUMERIC,
|
||||
t40_eligible NUMERIC, t40_ship_absorbed NUMERIC, t40_near NUMERIC, t40_near_gap NUMERIC,
|
||||
t50_eligible NUMERIC, t50_ship_absorbed NUMERIC, t50_near NUMERIC, t50_near_gap NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_thresh_ym ON threshold_monthly(year_month);
|
||||
|
||||
-- Metadata
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def init_schema():
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(SCHEMA)
|
||||
conn.commit()
|
||||
log.info("✓ Schema created")
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
if "already exists" in str(e):
|
||||
log.info("✓ Schema already exists")
|
||||
else:
|
||||
raise
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── Seed ─────────────────────────────────────────────────────
|
||||
|
||||
def is_seeded():
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM monthly")
|
||||
n = cur.fetchone()[0]
|
||||
cur.close()
|
||||
conn.close()
|
||||
return n > 0
|
||||
|
||||
|
||||
def seed_from_json(path=None):
|
||||
"""Load jv_data.json into Postgres. Idempotent — skips if data exists."""
|
||||
if is_seeded():
|
||||
log.info("✓ Database already seeded — skipping")
|
||||
return
|
||||
|
||||
if path is None:
|
||||
path = Path(__file__).parent / "static" / "dashboard" / "jv_data.json"
|
||||
|
||||
log.info(f"Seeding from {path} ...")
|
||||
t0 = time.time()
|
||||
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
# 1. monthly
|
||||
rows = [(r['YearMonth'], r.get('revenue'), r.get('orders'), r.get('aov'),
|
||||
r.get('customers'), r.get('newCustomers'), r.get('avgShipping'),
|
||||
r.get('freeShipPct'), r.get('avgMargin'), r.get('discountPct'),
|
||||
r.get('avgItems'), r.get('totalShipping'), r.get('freeShipOrders'),
|
||||
r.get('discountOrders'), r.get('totalItems'))
|
||||
for r in data['monthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO monthly VALUES %s ON CONFLICT DO NOTHING",
|
||||
rows)
|
||||
|
||||
# 2. channelMonthly
|
||||
rows = [(r['YearMonth'], r.get('ReferrerSource'), r.get('orders'),
|
||||
r.get('revenue'), r.get('avgAOV'), r.get('avgItems'),
|
||||
r.get('newPct'), r.get('freeShipPct'), r.get('discountPct'))
|
||||
for r in data['channelMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO channel_monthly (year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 3. newReturningMonthly
|
||||
rows = [(r['YearMonth'], r.get('IsNewCustomer'), r.get('orders'), r.get('revenue'))
|
||||
for r in data['newReturningMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO new_returning_monthly (year_month,is_new_customer,orders,revenue) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 4. dowMonthly
|
||||
rows = [(r['YearMonth'], r.get('DayOfWeek'), r.get('orders'), r.get('revenue'))
|
||||
for r in data['dowMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO dow_monthly (year_month,day_of_week,orders,revenue) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 5. paymentMonthly
|
||||
rows = [(r['YearMonth'], r.get('PaymentProcessor'), r.get('orders'), r.get('revenue'))
|
||||
for r in data['paymentMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO payment_monthly (year_month,payment_processor,orders,revenue) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 6. countryMonthly
|
||||
rows = [(r['YearMonth'], r.get('CustomerCountry'), r.get('orders'), r.get('revenue'))
|
||||
for r in data['countryMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO country_monthly (year_month,customer_country,orders,revenue) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 7. deliveryMonthly
|
||||
rows = [(r['YearMonth'], r.get('DeliveryMethod'), r.get('orders'),
|
||||
r.get('revenue'), r.get('avgShipping'), r.get('avgAOV'))
|
||||
for r in data['deliveryMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO delivery_monthly (year_month,delivery_method,orders,revenue,avg_shipping,avg_aov) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 8. aovMonthly
|
||||
rows = [(r['YearMonth'], r.get('AOVBucket'), r.get('count'),
|
||||
r.get('revenue'), r.get('avgShipping'), r.get('pctFreeShip'))
|
||||
for r in data['aovMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO aov_monthly (year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 9. discountMonthly
|
||||
rows = [(r['YearMonth'], r.get('HasDiscount'), r.get('count'),
|
||||
r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'),
|
||||
r.get('avgShipping'), r.get('revenue'))
|
||||
for r in data['discountMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO discount_monthly (year_month,has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 10. discountCodesMonthly
|
||||
rows = [(r['YearMonth'], r.get('DiscountCode'), r.get('uses'),
|
||||
r.get('revenue'), r.get('avgAOV'), r.get('avgDiscountPct'))
|
||||
for r in data['discountCodesMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO discount_codes_monthly (year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 11. shippingCompMonthly
|
||||
rows = [(r['YearMonth'], r.get('IsFreeShipping'), r.get('count'),
|
||||
r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'),
|
||||
r.get('newPct'), r.get('discountPct'), r.get('avgShipping'))
|
||||
for r in data['shippingCompMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO shipping_comp_monthly (year_month,is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 12. categoryMonthly
|
||||
rows = [(r['YearMonth'], r.get('Category'), r.get('totalRevenue'),
|
||||
r.get('totalRevenueExVAT'), r.get('totalCost'), r.get('totalQty'),
|
||||
r.get('orderCount'), r.get('uniqueProducts'),
|
||||
r.get('margin'), r.get('marginPct'))
|
||||
for r in data['categoryMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO category_monthly (year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 13. productMonthly
|
||||
rows = [(r['YearMonth'], r.get('Description'), r.get('SKUDescription'),
|
||||
r.get('Category'), r.get('totalRevenue'), r.get('totalRevenueExVAT'),
|
||||
r.get('totalCost'), r.get('totalQty'), r.get('orderCount'),
|
||||
r.get('margin'), r.get('marginPct'))
|
||||
for r in data['productMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO product_monthly (year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 14. crossSell
|
||||
rows = [(r.get('product1'), r.get('product2'), r.get('count'))
|
||||
for r in data['crossSell']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO cross_sell (product1,product2,count) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 15. cohortRetention
|
||||
rows = [(r.get('cohort'), r.get('size'),
|
||||
r.get('m0'), r.get('m1'), r.get('m2'), r.get('m3'),
|
||||
r.get('m4'), r.get('m5'), r.get('m6'), r.get('m7'),
|
||||
r.get('m8'), r.get('m9'), r.get('m10'), r.get('m11'),
|
||||
r.get('m12'))
|
||||
for r in data['cohortRetention']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO cohort_retention (cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 16. thresholdMonthly
|
||||
rows = [(r['YearMonth'], r.get('totalOrders'), r.get('totalShippingRev'),
|
||||
r.get('t15_eligible'), r.get('t15_shipAbsorbed'), r.get('t15_near'), r.get('t15_nearGap'),
|
||||
r.get('t20_eligible'), r.get('t20_shipAbsorbed'), r.get('t20_near'), r.get('t20_nearGap'),
|
||||
r.get('t25_eligible'), r.get('t25_shipAbsorbed'), r.get('t25_near'), r.get('t25_nearGap'),
|
||||
r.get('t30_eligible'), r.get('t30_shipAbsorbed'), r.get('t30_near'), r.get('t30_nearGap'),
|
||||
r.get('t35_eligible'), r.get('t35_shipAbsorbed'), r.get('t35_near'), r.get('t35_nearGap'),
|
||||
r.get('t40_eligible'), r.get('t40_shipAbsorbed'), r.get('t40_near'), r.get('t40_nearGap'),
|
||||
r.get('t50_eligible'), r.get('t50_shipAbsorbed'), r.get('t50_near'), r.get('t50_nearGap'))
|
||||
for r in data['thresholdMonthly']]
|
||||
execute_values(cur,
|
||||
"INSERT INTO threshold_monthly (year_month,total_orders,total_shipping_rev,t15_eligible,t15_ship_absorbed,t15_near,t15_near_gap,t20_eligible,t20_ship_absorbed,t20_near,t20_near_gap,t25_eligible,t25_ship_absorbed,t25_near,t25_near_gap,t30_eligible,t30_ship_absorbed,t30_near,t30_near_gap,t35_eligible,t35_ship_absorbed,t35_near,t35_near_gap,t40_eligible,t40_ship_absorbed,t40_near,t40_near_gap,t50_eligible,t50_ship_absorbed,t50_near,t50_near_gap) VALUES %s",
|
||||
rows)
|
||||
|
||||
# 17. meta
|
||||
meta = data.get('meta', {})
|
||||
for k, v in meta.items():
|
||||
cur.execute(
|
||||
"INSERT INTO meta (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",
|
||||
(k, json.dumps(v) if not isinstance(v, str) else v)
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
elapsed = time.time() - t0
|
||||
log.info(f"✓ Seeded {sum(len(data[k]) for k in data if isinstance(data[k], list))} rows in {elapsed:.1f}s")
|
||||
|
||||
|
||||
# ── Queries ──────────────────────────────────────────────────
|
||||
|
||||
def _ym_clause(start, end, col="year_month"):
|
||||
"""Build WHERE clause for date range."""
|
||||
parts = []
|
||||
params = []
|
||||
if start:
|
||||
parts.append(f"{col} >= %s")
|
||||
params.append(start)
|
||||
if end:
|
||||
parts.append(f"{col} <= %s")
|
||||
params.append(end)
|
||||
where = " AND ".join(parts) if parts else "TRUE"
|
||||
return where, params
|
||||
|
||||
|
||||
def _query(sql, params=()):
|
||||
conn = get_conn()
|
||||
cur = conn.cursor(cursor_factory=RealDictCursor)
|
||||
cur.execute(sql, params)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
conn.close()
|
||||
# Convert Decimal to float for JSON serialisation
|
||||
return [_dec_to_float(r) for r in rows]
|
||||
|
||||
|
||||
def _dec_to_float(d):
|
||||
from decimal import Decimal
|
||||
return {k: (float(v) if isinstance(v, Decimal) else v) for k, v in d.items()}
|
||||
|
||||
|
||||
def get_meta():
|
||||
rows = _query("SELECT key, value FROM meta")
|
||||
meta = {}
|
||||
for r in rows:
|
||||
v = r['value']
|
||||
try:
|
||||
v = json.loads(v)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
meta[r['key']] = v
|
||||
return meta
|
||||
|
||||
|
||||
def _remap(rows, mapping):
|
||||
"""Rename DB snake_case keys back to the camelCase the dashboard JS expects."""
|
||||
return [{mapping.get(k, k): v for k, v in r.items()} for r in rows]
|
||||
|
||||
|
||||
# Column mappings: DB name → JS name
|
||||
_M_MONTHLY = {
|
||||
'year_month':'YearMonth','revenue':'revenue','orders':'orders','aov':'aov',
|
||||
'customers':'customers','new_customers':'newCustomers','avg_shipping':'avgShipping',
|
||||
'free_ship_pct':'freeShipPct','avg_margin':'avgMargin','discount_pct':'discountPct',
|
||||
'avg_items':'avgItems','total_shipping':'totalShipping','free_ship_orders':'freeShipOrders',
|
||||
'discount_orders':'discountOrders','total_items':'totalItems'
|
||||
}
|
||||
_M_CHANNEL = {
|
||||
'year_month':'YearMonth','referrer_source':'ReferrerSource','orders':'orders',
|
||||
'revenue':'revenue','avg_aov':'avgAOV','avg_items':'avgItems',
|
||||
'new_pct':'newPct','free_ship_pct':'freeShipPct','discount_pct':'discountPct'
|
||||
}
|
||||
_M_NR = {'year_month':'YearMonth','is_new_customer':'IsNewCustomer','orders':'orders','revenue':'revenue'}
|
||||
_M_DOW = {'year_month':'YearMonth','day_of_week':'DayOfWeek','orders':'orders','revenue':'revenue'}
|
||||
_M_PAY = {'year_month':'YearMonth','payment_processor':'PaymentProcessor','orders':'orders','revenue':'revenue'}
|
||||
_M_COUNTRY = {'year_month':'YearMonth','customer_country':'CustomerCountry','orders':'orders','revenue':'revenue'}
|
||||
_M_DELIVERY = {
|
||||
'year_month':'YearMonth','delivery_method':'DeliveryMethod','orders':'orders',
|
||||
'revenue':'revenue','avg_shipping':'avgShipping','avg_aov':'avgAOV'
|
||||
}
|
||||
_M_AOV = {
|
||||
'year_month':'YearMonth','aov_bucket':'AOVBucket','count':'count',
|
||||
'revenue':'revenue','avg_shipping':'avgShipping','pct_free_ship':'pctFreeShip'
|
||||
}
|
||||
_M_DISC = {
|
||||
'year_month':'YearMonth','has_discount':'HasDiscount','count':'count',
|
||||
'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin',
|
||||
'avg_shipping':'avgShipping','revenue':'revenue'
|
||||
}
|
||||
_M_DCODE = {
|
||||
'year_month':'YearMonth','discount_code':'DiscountCode','uses':'uses',
|
||||
'revenue':'revenue','avg_aov':'avgAOV','avg_discount_pct':'avgDiscountPct'
|
||||
}
|
||||
_M_SHIPCOMP = {
|
||||
'year_month':'YearMonth','is_free_shipping':'IsFreeShipping','count':'count',
|
||||
'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin',
|
||||
'new_pct':'newPct','discount_pct':'discountPct','avg_shipping':'avgShipping'
|
||||
}
|
||||
_M_CAT = {
|
||||
'year_month':'YearMonth','category':'Category','total_revenue':'totalRevenue',
|
||||
'total_revenue_ex_vat':'totalRevenueExVAT','total_cost':'totalCost',
|
||||
'total_qty':'totalQty','order_count':'orderCount','unique_products':'uniqueProducts',
|
||||
'margin':'margin','margin_pct':'marginPct'
|
||||
}
|
||||
_M_PROD = {
|
||||
'year_month':'YearMonth','description':'Description','sku_description':'SKUDescription',
|
||||
'category':'Category','total_revenue':'totalRevenue','total_revenue_ex_vat':'totalRevenueExVAT',
|
||||
'total_cost':'totalCost','total_qty':'totalQty','order_count':'orderCount',
|
||||
'margin':'margin','margin_pct':'marginPct'
|
||||
}
|
||||
_M_CROSS = {'product1':'product1','product2':'product2','count':'count'}
|
||||
_M_COHORT = {
|
||||
'cohort':'cohort','size':'size',
|
||||
'm0':'m0','m1':'m1','m2':'m2','m3':'m3','m4':'m4','m5':'m5',
|
||||
'm6':'m6','m7':'m7','m8':'m8','m9':'m9','m10':'m10','m11':'m11','m12':'m12'
|
||||
}
|
||||
_M_THRESH = {
|
||||
'year_month':'YearMonth','total_orders':'totalOrders','total_shipping_rev':'totalShippingRev',
|
||||
't15_eligible':'t15_eligible','t15_ship_absorbed':'t15_shipAbsorbed','t15_near':'t15_near','t15_near_gap':'t15_nearGap',
|
||||
't20_eligible':'t20_eligible','t20_ship_absorbed':'t20_shipAbsorbed','t20_near':'t20_near','t20_near_gap':'t20_nearGap',
|
||||
't25_eligible':'t25_eligible','t25_ship_absorbed':'t25_shipAbsorbed','t25_near':'t25_near','t25_near_gap':'t25_nearGap',
|
||||
't30_eligible':'t30_eligible','t30_ship_absorbed':'t30_shipAbsorbed','t30_near':'t30_near','t30_near_gap':'t30_nearGap',
|
||||
't35_eligible':'t35_eligible','t35_ship_absorbed':'t35_shipAbsorbed','t35_near':'t35_near','t35_near_gap':'t35_nearGap',
|
||||
't40_eligible':'t40_eligible','t40_ship_absorbed':'t40_shipAbsorbed','t40_near':'t40_near','t40_near_gap':'t40_nearGap',
|
||||
't50_eligible':'t50_eligible','t50_ship_absorbed':'t50_shipAbsorbed','t50_near':'t50_near','t50_near_gap':'t50_nearGap',
|
||||
}
|
||||
|
||||
|
||||
def get_dashboard_data(start=None, end=None):
|
||||
"""Return the full dashboard payload, filtered by date range.
|
||||
Returns the same JSON shape the frontend RAW variable expects."""
|
||||
w, p = _ym_clause(start, end)
|
||||
|
||||
result = {
|
||||
'monthly': _remap(_query(f"SELECT * FROM monthly WHERE {w} ORDER BY year_month", p), _M_MONTHLY),
|
||||
'channelMonthly': _remap(_query(f"SELECT year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct FROM channel_monthly WHERE {w} ORDER BY year_month", p), _M_CHANNEL),
|
||||
'newReturningMonthly': _remap(_query(f"SELECT year_month,CAST(is_new_customer AS INTEGER) as is_new_customer,orders,revenue FROM new_returning_monthly WHERE {w} ORDER BY year_month", p), _M_NR),
|
||||
'dowMonthly': _remap(_query(f"SELECT year_month,day_of_week,orders,revenue FROM dow_monthly WHERE {w} ORDER BY year_month", p), _M_DOW),
|
||||
'paymentMonthly': _remap(_query(f"SELECT year_month,payment_processor,orders,revenue FROM payment_monthly WHERE {w} ORDER BY year_month", p), _M_PAY),
|
||||
'countryMonthly': _remap(_query(f"SELECT year_month,customer_country,orders,revenue FROM country_monthly WHERE {w} ORDER BY year_month", p), _M_COUNTRY),
|
||||
'deliveryMonthly': _remap(_query(f"SELECT year_month,delivery_method,orders,revenue,avg_shipping,avg_aov FROM delivery_monthly WHERE {w} ORDER BY year_month", p), _M_DELIVERY),
|
||||
'aovMonthly': _remap(_query(f"SELECT year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship FROM aov_monthly WHERE {w} ORDER BY year_month", p), _M_AOV),
|
||||
'discountMonthly': _remap(_query(f"SELECT year_month,CAST(has_discount AS INTEGER) as has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue FROM discount_monthly WHERE {w} ORDER BY year_month", p), _M_DISC),
|
||||
'discountCodesMonthly': _remap(_query(f"SELECT year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct FROM discount_codes_monthly WHERE {w} ORDER BY year_month", p), _M_DCODE),
|
||||
'shippingCompMonthly': _remap(_query(f"SELECT year_month,CAST(is_free_shipping AS INTEGER) as is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping FROM shipping_comp_monthly WHERE {w} ORDER BY year_month", p), _M_SHIPCOMP),
|
||||
'categoryMonthly': _remap(_query(f"SELECT year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct FROM category_monthly WHERE {w} ORDER BY year_month", p), _M_CAT),
|
||||
'productMonthly': _remap(_query(f"SELECT year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct FROM product_monthly WHERE {w} ORDER BY year_month", p), _M_PROD),
|
||||
# Non-date-filtered
|
||||
'crossSell': _remap(_query("SELECT product1,product2,count FROM cross_sell ORDER BY count DESC"), _M_CROSS),
|
||||
'cohortRetention': _remap(_query("SELECT cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12 FROM cohort_retention ORDER BY cohort"), _M_COHORT),
|
||||
'thresholdMonthly': _remap(_query(f"SELECT * FROM threshold_monthly WHERE {w} ORDER BY year_month", p), _M_THRESH),
|
||||
'meta': get_meta(),
|
||||
}
|
||||
|
||||
# Strip 'id' key from all rows (serial PKs not needed by frontend)
|
||||
for k, v in result.items():
|
||||
if isinstance(v, list):
|
||||
result[k] = [{kk: vv for kk, vv in row.items() if kk != 'id'} for row in v]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── Init ─────────────────────────────────────────────────────
|
||||
|
||||
def init_db():
|
||||
"""Wait for DB, create schema, seed if needed. Uses advisory lock to prevent races."""
|
||||
wait_for_db()
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
# Advisory lock so only one worker seeds
|
||||
cur.execute("SELECT pg_advisory_lock(42)")
|
||||
init_schema()
|
||||
seed_from_json()
|
||||
finally:
|
||||
cur.execute("SELECT pg_advisory_unlock(42)")
|
||||
cur.close()
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user