Omair Saleh
3f2b6b6188
- Added PostgreSQL 16 Alpine service to Docker Swarm stack - db.py: schema for 17 tables, auto-seed from jv_data.json on first boot - /api/dashboard/data?start=&end= — server-side SQL filtering All Time: 0.43s (was 4MB JSON download stuck loading) Filtered (12mo): 0.20s with ~90% less data transferred - Dashboard HTML patched: calls API instead of static JSON - Integer casting for IsNewCustomer/HasDiscount/IsFreeShipping - Advisory lock prevents race condition during parallel worker startup - Returning Revenue now shows correctly: £14.5M (75% of total)
571 lines
25 KiB
Python
571 lines
25 KiB
Python
"""JustVitamin — PostgreSQL data layer.
|
|
|
|
Creates tables, seeds from jv_data.json on first boot,
|
|
provides fast filtered queries for the dashboard API."""
|
|
|
|
import os, json, time, logging
|
|
from pathlib import Path
|
|
|
|
import psycopg2
|
|
from psycopg2.extras import execute_values, RealDictCursor
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
DB_URL = os.environ.get(
|
|
"DATABASE_URL",
|
|
"postgresql://jv:jvpass@tasks.db:5432/justvitamin"
|
|
)
|
|
|
|
# ── Connection ───────────────────────────────────────────────
|
|
|
|
def get_conn():
|
|
return psycopg2.connect(DB_URL)
|
|
|
|
|
|
def wait_for_db(retries=30, delay=2):
|
|
"""Block until Postgres is ready."""
|
|
for i in range(retries):
|
|
try:
|
|
conn = get_conn()
|
|
conn.close()
|
|
log.info("✓ Database ready")
|
|
return True
|
|
except Exception:
|
|
if i < retries - 1:
|
|
log.info(f"Waiting for DB... ({i+1}/{retries})")
|
|
time.sleep(delay)
|
|
raise RuntimeError("Database not available after retries")
|
|
|
|
|
|
# ── Schema ───────────────────────────────────────────────────
|
|
|
|
SCHEMA = """
|
|
-- Core monthly aggregates
|
|
CREATE TABLE IF NOT EXISTS monthly (
|
|
year_month TEXT PRIMARY KEY,
|
|
revenue NUMERIC, orders INT, aov NUMERIC, customers INT,
|
|
new_customers INT, avg_shipping NUMERIC, free_ship_pct NUMERIC,
|
|
avg_margin NUMERIC, discount_pct NUMERIC, avg_items NUMERIC,
|
|
total_shipping NUMERIC, free_ship_orders INT, discount_orders INT,
|
|
total_items INT
|
|
);
|
|
|
|
-- Channel breakdown by month
|
|
CREATE TABLE IF NOT EXISTS channel_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, referrer_source TEXT,
|
|
orders INT, revenue NUMERIC, avg_aov NUMERIC,
|
|
avg_items NUMERIC, new_pct NUMERIC, free_ship_pct NUMERIC,
|
|
discount_pct NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_channel_ym ON channel_monthly(year_month);
|
|
|
|
-- New vs returning by month
|
|
CREATE TABLE IF NOT EXISTS new_returning_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, is_new_customer TEXT,
|
|
orders INT, revenue NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_nr_ym ON new_returning_monthly(year_month);
|
|
|
|
-- Day of week by month
|
|
CREATE TABLE IF NOT EXISTS dow_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, day_of_week TEXT,
|
|
orders INT, revenue NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_dow_ym ON dow_monthly(year_month);
|
|
|
|
-- Payment processor by month
|
|
CREATE TABLE IF NOT EXISTS payment_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, payment_processor TEXT,
|
|
orders INT, revenue NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_pay_ym ON payment_monthly(year_month);
|
|
|
|
-- Country by month
|
|
CREATE TABLE IF NOT EXISTS country_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, customer_country TEXT,
|
|
orders INT, revenue NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_country_ym ON country_monthly(year_month);
|
|
|
|
-- Delivery by month
|
|
CREATE TABLE IF NOT EXISTS delivery_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, delivery_method TEXT,
|
|
orders INT, revenue NUMERIC, avg_shipping NUMERIC, avg_aov NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_delivery_ym ON delivery_monthly(year_month);
|
|
|
|
-- AOV buckets by month
|
|
CREATE TABLE IF NOT EXISTS aov_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, aov_bucket TEXT,
|
|
count INT, revenue NUMERIC, avg_shipping NUMERIC, pct_free_ship NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_aov_ym ON aov_monthly(year_month);
|
|
|
|
-- Discount by month
|
|
CREATE TABLE IF NOT EXISTS discount_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, has_discount TEXT,
|
|
count INT, avg_aov NUMERIC, avg_items NUMERIC,
|
|
avg_margin NUMERIC, avg_shipping NUMERIC, revenue NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_disc_ym ON discount_monthly(year_month);
|
|
|
|
-- Discount codes by month
|
|
CREATE TABLE IF NOT EXISTS discount_codes_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, discount_code TEXT,
|
|
uses INT, revenue NUMERIC, avg_aov NUMERIC, avg_discount_pct NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_dcode_ym ON discount_codes_monthly(year_month);
|
|
|
|
-- Free ship comparison by month
|
|
CREATE TABLE IF NOT EXISTS shipping_comp_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, is_free_shipping TEXT,
|
|
count INT, avg_aov NUMERIC, avg_items NUMERIC,
|
|
avg_margin NUMERIC, new_pct NUMERIC, discount_pct NUMERIC,
|
|
avg_shipping NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_shipcomp_ym ON shipping_comp_monthly(year_month);
|
|
|
|
-- Category by month
|
|
CREATE TABLE IF NOT EXISTS category_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, category TEXT,
|
|
total_revenue NUMERIC, total_revenue_ex_vat NUMERIC,
|
|
total_cost NUMERIC, total_qty INT, order_count INT,
|
|
unique_products INT, margin NUMERIC, margin_pct NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_cat_ym ON category_monthly(year_month);
|
|
|
|
-- Product by month
|
|
CREATE TABLE IF NOT EXISTS product_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, description TEXT, sku_description TEXT,
|
|
category TEXT, total_revenue NUMERIC, total_revenue_ex_vat NUMERIC,
|
|
total_cost NUMERIC, total_qty INT, order_count INT,
|
|
margin NUMERIC, margin_pct NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_prod_ym ON product_monthly(year_month);
|
|
|
|
-- Cross-sell pairs (no month dimension)
|
|
CREATE TABLE IF NOT EXISTS cross_sell (
|
|
id SERIAL PRIMARY KEY,
|
|
product1 TEXT, product2 TEXT, count INT
|
|
);
|
|
|
|
-- Cohort retention
|
|
CREATE TABLE IF NOT EXISTS cohort_retention (
|
|
id SERIAL PRIMARY KEY,
|
|
cohort TEXT, size INT,
|
|
m0 NUMERIC, m1 NUMERIC, m2 NUMERIC, m3 NUMERIC,
|
|
m4 NUMERIC, m5 NUMERIC, m6 NUMERIC, m7 NUMERIC,
|
|
m8 NUMERIC, m9 NUMERIC, m10 NUMERIC, m11 NUMERIC,
|
|
m12 NUMERIC
|
|
);
|
|
|
|
-- Free shipping threshold by month
|
|
CREATE TABLE IF NOT EXISTS threshold_monthly (
|
|
id SERIAL PRIMARY KEY,
|
|
year_month TEXT, total_orders INT, total_shipping_rev NUMERIC,
|
|
t15_eligible NUMERIC, t15_ship_absorbed NUMERIC, t15_near NUMERIC, t15_near_gap NUMERIC,
|
|
t20_eligible NUMERIC, t20_ship_absorbed NUMERIC, t20_near NUMERIC, t20_near_gap NUMERIC,
|
|
t25_eligible NUMERIC, t25_ship_absorbed NUMERIC, t25_near NUMERIC, t25_near_gap NUMERIC,
|
|
t30_eligible NUMERIC, t30_ship_absorbed NUMERIC, t30_near NUMERIC, t30_near_gap NUMERIC,
|
|
t35_eligible NUMERIC, t35_ship_absorbed NUMERIC, t35_near NUMERIC, t35_near_gap NUMERIC,
|
|
t40_eligible NUMERIC, t40_ship_absorbed NUMERIC, t40_near NUMERIC, t40_near_gap NUMERIC,
|
|
t50_eligible NUMERIC, t50_ship_absorbed NUMERIC, t50_near NUMERIC, t50_near_gap NUMERIC
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_thresh_ym ON threshold_monthly(year_month);
|
|
|
|
-- Metadata
|
|
CREATE TABLE IF NOT EXISTS meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT
|
|
);
|
|
"""
|
|
|
|
|
|
def init_schema():
|
|
conn = get_conn()
|
|
cur = conn.cursor()
|
|
try:
|
|
cur.execute(SCHEMA)
|
|
conn.commit()
|
|
log.info("✓ Schema created")
|
|
except Exception as e:
|
|
conn.rollback()
|
|
if "already exists" in str(e):
|
|
log.info("✓ Schema already exists")
|
|
else:
|
|
raise
|
|
finally:
|
|
cur.close()
|
|
conn.close()
|
|
|
|
|
|
# ── Seed ─────────────────────────────────────────────────────
|
|
|
|
def is_seeded():
|
|
conn = get_conn()
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT COUNT(*) FROM monthly")
|
|
n = cur.fetchone()[0]
|
|
cur.close()
|
|
conn.close()
|
|
return n > 0
|
|
|
|
|
|
def seed_from_json(path=None):
|
|
"""Load jv_data.json into Postgres. Idempotent — skips if data exists."""
|
|
if is_seeded():
|
|
log.info("✓ Database already seeded — skipping")
|
|
return
|
|
|
|
if path is None:
|
|
path = Path(__file__).parent / "static" / "dashboard" / "jv_data.json"
|
|
|
|
log.info(f"Seeding from {path} ...")
|
|
t0 = time.time()
|
|
|
|
with open(path) as f:
|
|
data = json.load(f)
|
|
|
|
conn = get_conn()
|
|
cur = conn.cursor()
|
|
|
|
# 1. monthly
|
|
rows = [(r['YearMonth'], r.get('revenue'), r.get('orders'), r.get('aov'),
|
|
r.get('customers'), r.get('newCustomers'), r.get('avgShipping'),
|
|
r.get('freeShipPct'), r.get('avgMargin'), r.get('discountPct'),
|
|
r.get('avgItems'), r.get('totalShipping'), r.get('freeShipOrders'),
|
|
r.get('discountOrders'), r.get('totalItems'))
|
|
for r in data['monthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO monthly VALUES %s ON CONFLICT DO NOTHING",
|
|
rows)
|
|
|
|
# 2. channelMonthly
|
|
rows = [(r['YearMonth'], r.get('ReferrerSource'), r.get('orders'),
|
|
r.get('revenue'), r.get('avgAOV'), r.get('avgItems'),
|
|
r.get('newPct'), r.get('freeShipPct'), r.get('discountPct'))
|
|
for r in data['channelMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO channel_monthly (year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct) VALUES %s",
|
|
rows)
|
|
|
|
# 3. newReturningMonthly
|
|
rows = [(r['YearMonth'], r.get('IsNewCustomer'), r.get('orders'), r.get('revenue'))
|
|
for r in data['newReturningMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO new_returning_monthly (year_month,is_new_customer,orders,revenue) VALUES %s",
|
|
rows)
|
|
|
|
# 4. dowMonthly
|
|
rows = [(r['YearMonth'], r.get('DayOfWeek'), r.get('orders'), r.get('revenue'))
|
|
for r in data['dowMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO dow_monthly (year_month,day_of_week,orders,revenue) VALUES %s",
|
|
rows)
|
|
|
|
# 5. paymentMonthly
|
|
rows = [(r['YearMonth'], r.get('PaymentProcessor'), r.get('orders'), r.get('revenue'))
|
|
for r in data['paymentMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO payment_monthly (year_month,payment_processor,orders,revenue) VALUES %s",
|
|
rows)
|
|
|
|
# 6. countryMonthly
|
|
rows = [(r['YearMonth'], r.get('CustomerCountry'), r.get('orders'), r.get('revenue'))
|
|
for r in data['countryMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO country_monthly (year_month,customer_country,orders,revenue) VALUES %s",
|
|
rows)
|
|
|
|
# 7. deliveryMonthly
|
|
rows = [(r['YearMonth'], r.get('DeliveryMethod'), r.get('orders'),
|
|
r.get('revenue'), r.get('avgShipping'), r.get('avgAOV'))
|
|
for r in data['deliveryMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO delivery_monthly (year_month,delivery_method,orders,revenue,avg_shipping,avg_aov) VALUES %s",
|
|
rows)
|
|
|
|
# 8. aovMonthly
|
|
rows = [(r['YearMonth'], r.get('AOVBucket'), r.get('count'),
|
|
r.get('revenue'), r.get('avgShipping'), r.get('pctFreeShip'))
|
|
for r in data['aovMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO aov_monthly (year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship) VALUES %s",
|
|
rows)
|
|
|
|
# 9. discountMonthly
|
|
rows = [(r['YearMonth'], r.get('HasDiscount'), r.get('count'),
|
|
r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'),
|
|
r.get('avgShipping'), r.get('revenue'))
|
|
for r in data['discountMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO discount_monthly (year_month,has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue) VALUES %s",
|
|
rows)
|
|
|
|
# 10. discountCodesMonthly
|
|
rows = [(r['YearMonth'], r.get('DiscountCode'), r.get('uses'),
|
|
r.get('revenue'), r.get('avgAOV'), r.get('avgDiscountPct'))
|
|
for r in data['discountCodesMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO discount_codes_monthly (year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct) VALUES %s",
|
|
rows)
|
|
|
|
# 11. shippingCompMonthly
|
|
rows = [(r['YearMonth'], r.get('IsFreeShipping'), r.get('count'),
|
|
r.get('avgAOV'), r.get('avgItems'), r.get('avgMargin'),
|
|
r.get('newPct'), r.get('discountPct'), r.get('avgShipping'))
|
|
for r in data['shippingCompMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO shipping_comp_monthly (year_month,is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping) VALUES %s",
|
|
rows)
|
|
|
|
# 12. categoryMonthly
|
|
rows = [(r['YearMonth'], r.get('Category'), r.get('totalRevenue'),
|
|
r.get('totalRevenueExVAT'), r.get('totalCost'), r.get('totalQty'),
|
|
r.get('orderCount'), r.get('uniqueProducts'),
|
|
r.get('margin'), r.get('marginPct'))
|
|
for r in data['categoryMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO category_monthly (year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct) VALUES %s",
|
|
rows)
|
|
|
|
# 13. productMonthly
|
|
rows = [(r['YearMonth'], r.get('Description'), r.get('SKUDescription'),
|
|
r.get('Category'), r.get('totalRevenue'), r.get('totalRevenueExVAT'),
|
|
r.get('totalCost'), r.get('totalQty'), r.get('orderCount'),
|
|
r.get('margin'), r.get('marginPct'))
|
|
for r in data['productMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO product_monthly (year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct) VALUES %s",
|
|
rows)
|
|
|
|
# 14. crossSell
|
|
rows = [(r.get('product1'), r.get('product2'), r.get('count'))
|
|
for r in data['crossSell']]
|
|
execute_values(cur,
|
|
"INSERT INTO cross_sell (product1,product2,count) VALUES %s",
|
|
rows)
|
|
|
|
# 15. cohortRetention
|
|
rows = [(r.get('cohort'), r.get('size'),
|
|
r.get('m0'), r.get('m1'), r.get('m2'), r.get('m3'),
|
|
r.get('m4'), r.get('m5'), r.get('m6'), r.get('m7'),
|
|
r.get('m8'), r.get('m9'), r.get('m10'), r.get('m11'),
|
|
r.get('m12'))
|
|
for r in data['cohortRetention']]
|
|
execute_values(cur,
|
|
"INSERT INTO cohort_retention (cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12) VALUES %s",
|
|
rows)
|
|
|
|
# 16. thresholdMonthly
|
|
rows = [(r['YearMonth'], r.get('totalOrders'), r.get('totalShippingRev'),
|
|
r.get('t15_eligible'), r.get('t15_shipAbsorbed'), r.get('t15_near'), r.get('t15_nearGap'),
|
|
r.get('t20_eligible'), r.get('t20_shipAbsorbed'), r.get('t20_near'), r.get('t20_nearGap'),
|
|
r.get('t25_eligible'), r.get('t25_shipAbsorbed'), r.get('t25_near'), r.get('t25_nearGap'),
|
|
r.get('t30_eligible'), r.get('t30_shipAbsorbed'), r.get('t30_near'), r.get('t30_nearGap'),
|
|
r.get('t35_eligible'), r.get('t35_shipAbsorbed'), r.get('t35_near'), r.get('t35_nearGap'),
|
|
r.get('t40_eligible'), r.get('t40_shipAbsorbed'), r.get('t40_near'), r.get('t40_nearGap'),
|
|
r.get('t50_eligible'), r.get('t50_shipAbsorbed'), r.get('t50_near'), r.get('t50_nearGap'))
|
|
for r in data['thresholdMonthly']]
|
|
execute_values(cur,
|
|
"INSERT INTO threshold_monthly (year_month,total_orders,total_shipping_rev,t15_eligible,t15_ship_absorbed,t15_near,t15_near_gap,t20_eligible,t20_ship_absorbed,t20_near,t20_near_gap,t25_eligible,t25_ship_absorbed,t25_near,t25_near_gap,t30_eligible,t30_ship_absorbed,t30_near,t30_near_gap,t35_eligible,t35_ship_absorbed,t35_near,t35_near_gap,t40_eligible,t40_ship_absorbed,t40_near,t40_near_gap,t50_eligible,t50_ship_absorbed,t50_near,t50_near_gap) VALUES %s",
|
|
rows)
|
|
|
|
# 17. meta
|
|
meta = data.get('meta', {})
|
|
for k, v in meta.items():
|
|
cur.execute(
|
|
"INSERT INTO meta (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value",
|
|
(k, json.dumps(v) if not isinstance(v, str) else v)
|
|
)
|
|
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
|
|
elapsed = time.time() - t0
|
|
log.info(f"✓ Seeded {sum(len(data[k]) for k in data if isinstance(data[k], list))} rows in {elapsed:.1f}s")
|
|
|
|
|
|
# ── Queries ──────────────────────────────────────────────────
|
|
|
|
def _ym_clause(start, end, col="year_month"):
|
|
"""Build WHERE clause for date range."""
|
|
parts = []
|
|
params = []
|
|
if start:
|
|
parts.append(f"{col} >= %s")
|
|
params.append(start)
|
|
if end:
|
|
parts.append(f"{col} <= %s")
|
|
params.append(end)
|
|
where = " AND ".join(parts) if parts else "TRUE"
|
|
return where, params
|
|
|
|
|
|
def _query(sql, params=()):
|
|
conn = get_conn()
|
|
cur = conn.cursor(cursor_factory=RealDictCursor)
|
|
cur.execute(sql, params)
|
|
rows = cur.fetchall()
|
|
cur.close()
|
|
conn.close()
|
|
# Convert Decimal to float for JSON serialisation
|
|
return [_dec_to_float(r) for r in rows]
|
|
|
|
|
|
def _dec_to_float(d):
|
|
from decimal import Decimal
|
|
return {k: (float(v) if isinstance(v, Decimal) else v) for k, v in d.items()}
|
|
|
|
|
|
def get_meta():
|
|
rows = _query("SELECT key, value FROM meta")
|
|
meta = {}
|
|
for r in rows:
|
|
v = r['value']
|
|
try:
|
|
v = json.loads(v)
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
meta[r['key']] = v
|
|
return meta
|
|
|
|
|
|
def _remap(rows, mapping):
|
|
"""Rename DB snake_case keys back to the camelCase the dashboard JS expects."""
|
|
return [{mapping.get(k, k): v for k, v in r.items()} for r in rows]
|
|
|
|
|
|
# Column mappings: DB name → JS name
|
|
_M_MONTHLY = {
|
|
'year_month':'YearMonth','revenue':'revenue','orders':'orders','aov':'aov',
|
|
'customers':'customers','new_customers':'newCustomers','avg_shipping':'avgShipping',
|
|
'free_ship_pct':'freeShipPct','avg_margin':'avgMargin','discount_pct':'discountPct',
|
|
'avg_items':'avgItems','total_shipping':'totalShipping','free_ship_orders':'freeShipOrders',
|
|
'discount_orders':'discountOrders','total_items':'totalItems'
|
|
}
|
|
_M_CHANNEL = {
|
|
'year_month':'YearMonth','referrer_source':'ReferrerSource','orders':'orders',
|
|
'revenue':'revenue','avg_aov':'avgAOV','avg_items':'avgItems',
|
|
'new_pct':'newPct','free_ship_pct':'freeShipPct','discount_pct':'discountPct'
|
|
}
|
|
_M_NR = {'year_month':'YearMonth','is_new_customer':'IsNewCustomer','orders':'orders','revenue':'revenue'}
|
|
_M_DOW = {'year_month':'YearMonth','day_of_week':'DayOfWeek','orders':'orders','revenue':'revenue'}
|
|
_M_PAY = {'year_month':'YearMonth','payment_processor':'PaymentProcessor','orders':'orders','revenue':'revenue'}
|
|
_M_COUNTRY = {'year_month':'YearMonth','customer_country':'CustomerCountry','orders':'orders','revenue':'revenue'}
|
|
_M_DELIVERY = {
|
|
'year_month':'YearMonth','delivery_method':'DeliveryMethod','orders':'orders',
|
|
'revenue':'revenue','avg_shipping':'avgShipping','avg_aov':'avgAOV'
|
|
}
|
|
_M_AOV = {
|
|
'year_month':'YearMonth','aov_bucket':'AOVBucket','count':'count',
|
|
'revenue':'revenue','avg_shipping':'avgShipping','pct_free_ship':'pctFreeShip'
|
|
}
|
|
_M_DISC = {
|
|
'year_month':'YearMonth','has_discount':'HasDiscount','count':'count',
|
|
'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin',
|
|
'avg_shipping':'avgShipping','revenue':'revenue'
|
|
}
|
|
_M_DCODE = {
|
|
'year_month':'YearMonth','discount_code':'DiscountCode','uses':'uses',
|
|
'revenue':'revenue','avg_aov':'avgAOV','avg_discount_pct':'avgDiscountPct'
|
|
}
|
|
_M_SHIPCOMP = {
|
|
'year_month':'YearMonth','is_free_shipping':'IsFreeShipping','count':'count',
|
|
'avg_aov':'avgAOV','avg_items':'avgItems','avg_margin':'avgMargin',
|
|
'new_pct':'newPct','discount_pct':'discountPct','avg_shipping':'avgShipping'
|
|
}
|
|
_M_CAT = {
|
|
'year_month':'YearMonth','category':'Category','total_revenue':'totalRevenue',
|
|
'total_revenue_ex_vat':'totalRevenueExVAT','total_cost':'totalCost',
|
|
'total_qty':'totalQty','order_count':'orderCount','unique_products':'uniqueProducts',
|
|
'margin':'margin','margin_pct':'marginPct'
|
|
}
|
|
_M_PROD = {
|
|
'year_month':'YearMonth','description':'Description','sku_description':'SKUDescription',
|
|
'category':'Category','total_revenue':'totalRevenue','total_revenue_ex_vat':'totalRevenueExVAT',
|
|
'total_cost':'totalCost','total_qty':'totalQty','order_count':'orderCount',
|
|
'margin':'margin','margin_pct':'marginPct'
|
|
}
|
|
_M_CROSS = {'product1':'product1','product2':'product2','count':'count'}
|
|
_M_COHORT = {
|
|
'cohort':'cohort','size':'size',
|
|
'm0':'m0','m1':'m1','m2':'m2','m3':'m3','m4':'m4','m5':'m5',
|
|
'm6':'m6','m7':'m7','m8':'m8','m9':'m9','m10':'m10','m11':'m11','m12':'m12'
|
|
}
|
|
_M_THRESH = {
|
|
'year_month':'YearMonth','total_orders':'totalOrders','total_shipping_rev':'totalShippingRev',
|
|
't15_eligible':'t15_eligible','t15_ship_absorbed':'t15_shipAbsorbed','t15_near':'t15_near','t15_near_gap':'t15_nearGap',
|
|
't20_eligible':'t20_eligible','t20_ship_absorbed':'t20_shipAbsorbed','t20_near':'t20_near','t20_near_gap':'t20_nearGap',
|
|
't25_eligible':'t25_eligible','t25_ship_absorbed':'t25_shipAbsorbed','t25_near':'t25_near','t25_near_gap':'t25_nearGap',
|
|
't30_eligible':'t30_eligible','t30_ship_absorbed':'t30_shipAbsorbed','t30_near':'t30_near','t30_near_gap':'t30_nearGap',
|
|
't35_eligible':'t35_eligible','t35_ship_absorbed':'t35_shipAbsorbed','t35_near':'t35_near','t35_near_gap':'t35_nearGap',
|
|
't40_eligible':'t40_eligible','t40_ship_absorbed':'t40_shipAbsorbed','t40_near':'t40_near','t40_near_gap':'t40_nearGap',
|
|
't50_eligible':'t50_eligible','t50_ship_absorbed':'t50_shipAbsorbed','t50_near':'t50_near','t50_near_gap':'t50_nearGap',
|
|
}
|
|
|
|
|
|
def get_dashboard_data(start=None, end=None):
|
|
"""Return the full dashboard payload, filtered by date range.
|
|
Returns the same JSON shape the frontend RAW variable expects."""
|
|
w, p = _ym_clause(start, end)
|
|
|
|
result = {
|
|
'monthly': _remap(_query(f"SELECT * FROM monthly WHERE {w} ORDER BY year_month", p), _M_MONTHLY),
|
|
'channelMonthly': _remap(_query(f"SELECT year_month,referrer_source,orders,revenue,avg_aov,avg_items,new_pct,free_ship_pct,discount_pct FROM channel_monthly WHERE {w} ORDER BY year_month", p), _M_CHANNEL),
|
|
'newReturningMonthly': _remap(_query(f"SELECT year_month,CAST(is_new_customer AS INTEGER) as is_new_customer,orders,revenue FROM new_returning_monthly WHERE {w} ORDER BY year_month", p), _M_NR),
|
|
'dowMonthly': _remap(_query(f"SELECT year_month,day_of_week,orders,revenue FROM dow_monthly WHERE {w} ORDER BY year_month", p), _M_DOW),
|
|
'paymentMonthly': _remap(_query(f"SELECT year_month,payment_processor,orders,revenue FROM payment_monthly WHERE {w} ORDER BY year_month", p), _M_PAY),
|
|
'countryMonthly': _remap(_query(f"SELECT year_month,customer_country,orders,revenue FROM country_monthly WHERE {w} ORDER BY year_month", p), _M_COUNTRY),
|
|
'deliveryMonthly': _remap(_query(f"SELECT year_month,delivery_method,orders,revenue,avg_shipping,avg_aov FROM delivery_monthly WHERE {w} ORDER BY year_month", p), _M_DELIVERY),
|
|
'aovMonthly': _remap(_query(f"SELECT year_month,aov_bucket,count,revenue,avg_shipping,pct_free_ship FROM aov_monthly WHERE {w} ORDER BY year_month", p), _M_AOV),
|
|
'discountMonthly': _remap(_query(f"SELECT year_month,CAST(has_discount AS INTEGER) as has_discount,count,avg_aov,avg_items,avg_margin,avg_shipping,revenue FROM discount_monthly WHERE {w} ORDER BY year_month", p), _M_DISC),
|
|
'discountCodesMonthly': _remap(_query(f"SELECT year_month,discount_code,uses,revenue,avg_aov,avg_discount_pct FROM discount_codes_monthly WHERE {w} ORDER BY year_month", p), _M_DCODE),
|
|
'shippingCompMonthly': _remap(_query(f"SELECT year_month,CAST(is_free_shipping AS INTEGER) as is_free_shipping,count,avg_aov,avg_items,avg_margin,new_pct,discount_pct,avg_shipping FROM shipping_comp_monthly WHERE {w} ORDER BY year_month", p), _M_SHIPCOMP),
|
|
'categoryMonthly': _remap(_query(f"SELECT year_month,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,unique_products,margin,margin_pct FROM category_monthly WHERE {w} ORDER BY year_month", p), _M_CAT),
|
|
'productMonthly': _remap(_query(f"SELECT year_month,description,sku_description,category,total_revenue,total_revenue_ex_vat,total_cost,total_qty,order_count,margin,margin_pct FROM product_monthly WHERE {w} ORDER BY year_month", p), _M_PROD),
|
|
# Non-date-filtered
|
|
'crossSell': _remap(_query("SELECT product1,product2,count FROM cross_sell ORDER BY count DESC"), _M_CROSS),
|
|
'cohortRetention': _remap(_query("SELECT cohort,size,m0,m1,m2,m3,m4,m5,m6,m7,m8,m9,m10,m11,m12 FROM cohort_retention ORDER BY cohort"), _M_COHORT),
|
|
'thresholdMonthly': _remap(_query(f"SELECT * FROM threshold_monthly WHERE {w} ORDER BY year_month", p), _M_THRESH),
|
|
'meta': get_meta(),
|
|
}
|
|
|
|
# Strip 'id' key from all rows (serial PKs not needed by frontend)
|
|
for k, v in result.items():
|
|
if isinstance(v, list):
|
|
result[k] = [{kk: vv for kk, vv in row.items() if kk != 'id'} for row in v]
|
|
|
|
return result
|
|
|
|
|
|
# ── Init ─────────────────────────────────────────────────────
|
|
|
|
def init_db():
|
|
"""Wait for DB, create schema, seed if needed. Uses advisory lock to prevent races."""
|
|
wait_for_db()
|
|
conn = get_conn()
|
|
cur = conn.cursor()
|
|
try:
|
|
# Advisory lock so only one worker seeds
|
|
cur.execute("SELECT pg_advisory_lock(42)")
|
|
init_schema()
|
|
seed_from_json()
|
|
finally:
|
|
cur.execute("SELECT pg_advisory_unlock(42)")
|
|
cur.close()
|
|
conn.close()
|