From 6ecff261f6c387e122741beea49d40f79630b052 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Thu, 14 Dec 2023 06:00:32 +0000 Subject: [PATCH 01/13] Logic fix for is_on_time_delivery --- models/marts/reporting/rep_exchange__daily_orders.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql index a626178..abbeb17 100644 --- a/models/marts/reporting/rep_exchange__daily_orders.sql +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -197,6 +197,7 @@ fct_and_dim_joins AS ( CASE -- On-time boolean is null when order is not complete WHEN account_verified_payment_datetime IS NULL THEN NULL + WHEN estimated_order_delivery_datetime_website_sla IS NULL THEN NULL WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1 ELSE 0 END AS is_on_time_delivery_website_sla, @@ -205,6 +206,7 @@ fct_and_dim_joins AS ( CASE -- On-time boolean is null when order is not complete WHEN account_verified_payment_datetime IS NULL THEN NULL + WHEN estimated_order_delivery_datetime_customer_expectation IS NULL THEN NULL WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1 ELSE 0 END AS is_on_time_delivery_customer_expectation, From 0b0af14e2a1f9c0715182b94f5ba5a69e2522b16 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 22 Dec 2023 07:36:03 +0000 Subject: [PATCH 02/13] migrated_company_list for shipping --- ..._shipping__migrated_company_marking_id.csv | 2164 +++++++++++++++++ 1 file changed, 2164 insertions(+) create mode 100644 seeds/shipping/seed_shipping__migrated_company_marking_id.csv diff --git a/seeds/shipping/seed_shipping__migrated_company_marking_id.csv b/seeds/shipping/seed_shipping__migrated_company_marking_id.csv new file mode 100644 index 0000000..b1e5a1c --- /dev/null +++ b/seeds/shipping/seed_shipping__migrated_company_marking_id.csv @@ -0,0 +1,2164 @@ +COMPANY_MARKING_ID +990AFE +915JYH +796AFT +744RTC +742KHT +682TPS +646OSS +1211GRE +1186SOP +1181LAD +1176ONG +1139POT +1076VOC +1052CTF +1032BWS +1018LBS +952CJY +848LLM +775TYY +697ECB +693TRT +1180SNA +1074 +1058TWT +1006KHK +999CLW +998EME +997ESY +996AMX +995KPY +994EDG +993TKY +992CSS +991VLX +98OTC +989ECO +988TCM +987HAN +986KEE +985AFF +984FED +983CZY +982DNA +981JNR +980LAM +979SDN +978LCL +977DZF +976WFM +975FNX +974LSQ +973SNM +972WAN +971SAF +970LBC +969MGR +968IKI +967HER +966YAN +965EYE +964SLT +963WIL +962LOW +961QTM +960APC +959LOV +958GDN +957WSK +956JEM +956GSC +955VSS +954LKE +953GMC +951EZE +950CWM +949JFN +948VAS +947UVS +946PAC +945TJS +944FTT +943GCC +942GCE +941RTE +940SSJ +939TPL +938SMH +937TPN +936UPN +935SKY +934HUB +933JKJ +932KPC +931BBL +930JCF +929CTG +928SMY +927NGM +926CLT +925BPS +924BSC +923GRB +922LWM +921QQC +920RWA +919GTM +918COT +917YCE +916SGE +914LWW +913DLT +912CMC +911ILR +910ZPE +909YLY +908LSI +907BMF +906YBE +905BTP +904RHZ +903LYS +902YYM +901KLY +900FLE +899MCS +898JSL +897MKS +896AWT +895CHH +894YNS +893JFU +892SSE +891CES +890AZH +88JSA +889NSM +888BWA +887KKS +886WBC +885WCY +884YTS +883ACM +882WJT +881GMT +880KEN +879LFE +878LKS +877YKK +876KHW +875RFM +874LCT +873KET +872SPT +871VMS +870WPC +869PYE +868HDS +867RNR +866 +865CLS +864SHD +863NMF +862QBL +861TMY +860CWS +859CMT +858ACE +857WSY +856FGS +855STE +854PNS +853EQZ +852CVS +851AWS +850MRS +84IMS +849MFN +847YHE +846ETT +845HLR +844JPT +843NMR +842GLR +841BLS +840APLUS +839BRN +838QVS +837CWK +836TST +835UMR +834QDC +833JYP +832AMY +831ESS +830FLP +82YHS +829TBP +828YPM +827NTA +826CHS +825LCK +824VNV +823MHZ +822SJN +821YLT +820FGE +819UNI +818FSA +817YHZ +816MLT +815JTT +814ALE +813AAM +812EBM +811MHA +810EDS +809LSY +808JKN +807VTG +806BST +805SYU +804SWT +803AFD +802QTS +801STT +800MGF +799LTR +798MTH +797STB +795NSF +794HNT +793AZS +792NWA +791HTO +790SSF +789JYS +788AJR +787MFY +786BKE +785ENT +784LYA +783STC +782MGE +781HMK +780TEC +779ZAF +778GLB +777SDM +776QCE +774CRS +773NDZ +772PCK +771HHL +770WYJ +770BAA +76WWS +769SMC +768LEP +767NRT +766PRO +765LFM +764FUN +763ICS +762MTF +761JOE +760SGN +759LDT +759LDT +758SRI +757JDL +756LKM +755KBK +754/SBE +753ERA +752CTK +751LAP +750MMH +749WJI +748NBS +747ERN +746OHN +745JOM +743FLA +741TFS +740SDI +739XMI +738ATO +737MCC +736ELN +735BIS +734RDS +733HSS +732EAV +731HLB +730FWS +729VTS +728EVO +727TOS +726BNV +725WMM +724ETY +723HGE +722FRL +721PCV +720PWH +719SSW +718TPP +717HAG +716SAM +715LRF +714TOP +713MKR +712PMP +711LMS +710TWC +709PSR +708MSS +707ERS +706HBC +705SCL +704TCS +703SKH +703KLJ +702LCB +701DMM +700SKD +699SPF +694STF +692EMV +691TSH +690NAN +689SBE +688HNK +687SNT +686CPM +685CLY +685CLY +684HRH +683MTY +681NNC +680GLM +679ZCK +678JSD +677DRM +676NCL +675KHF +674CKE +673HRP +672LYC +671JNN +670EMP +669CFT +668EDW +667PKM +666STA +665CJW +664DEF +663YMC +662SPE +661SME +660CKH +659IFC +658JIS +657RHD +656HHM +655HBS +654ASN +653MYC +652TSI +651DEW +650AKK +649WES +648VCM +647SGT +645ASL +644DAD +643JRY +642SOT +642LKI +641NEW +640AME +639NMQ +638IVO +637SCM +635KWA +634HCI +633FCR +632KCJ +631KNE +630ECS +629YTM +628MCR +627SKC +626CFB +625OCI +624AEH +623TMC +622BCL +621JVS +620UPI +619ITW +618PLA +617NVM +615BTA +614ELI +613FPN +612SKE +611SLE +605HOS +604SDS +603CGT +602KWI +601CCC +600LKK +54STS +53ING +51CMK +49SPK +488TCA +487BKP +483SMT +482APA +481WMC +480ALP +479SSS +478GEC +477SJC +476CIC +475RFX +474JFE +473KNY +472HSH +472HLS +471RYT +470BLA +469HBM +467PRI +467MCH +465EXC +461FPE +460GAW +459PTS +458VTS +457SCG +456ADP +455HTE +454DLN +453MIT +452CMG +451YWY +450BZG +449EPB +448FGT +447SYL +446CWC +446CWC +445MNG +445MNG +444SHB +443MLP +442TWH +441BLT +440KHS +43HAA +438PMY +434UDC +432ETM +431CTP +430PP +429WKT +428WLS +426SLA +426KOKS +420CYG +417JRB +416HKS +415MNC +414TTP +412WTX +410NMC +409AWN +404AJM +403SLR +402ZOEL +401MVL +39M2T +396SLY +396MAH +396EST +396EST/BOLTON +396EST/FNL +396EST/SEGAR +396EST/Selayang +396CLE +392TAC +388SYD +386JYT +384SMR +374SYNC +368JOSF +366EOA +361LPD +347GVU +30GVS +309GMS +278MFW +251LTS +240ATM +236GHE +200CSA +18BIT +17SUN +162MFE +153DSR +150ECE +128LUMZ +1265EHT +1264LYL +1263NIR +1262TBL +1261TYC +1260FMC +1259CRA +1258ZAI +1257CXL +1256YCH +1255LCF +1254CZQ +1253JHT +1252REN +1251VBB +1250AGS +1249TSM +1248VGS +1247MMC +1246HCC +1245JNT +1244HRF +1243FID +1242LIT +1241ASB +1240MHF +1239TJH +1238PJY +1237LWL +1236DIY +1235SQT +1234MTM +1233HTS +1232EPW +1231CYF +1230LLL +1229CJH +1228SSB +1227MHY +1226VFT +1225LSC +1224SOL +1223KCJ +1222TRG +1221HAY +1220KYC +1219HKC +1218CZL +1217CYK +1216HCL +1215TEN +1214CSP +1213DCA +1212AGR +1210TZN +1209YZC +1208JIA +1207CRE +1206ASR +1205MDF +1204GOW +1203HSY +1202EDH +1201CTL +1200KJH +1199FYL +1198HAS +1197JAY +1196VNK +1195NNA +1194FKI +1193ADR +1192LIM +1191WPY +1190EDM +1189NYY +1188JAC +1187COR +1185JAP +1184CEO +1183LLY +1182TKK +1179SBS +1178LCM +1177TDT +1175EZK +1174YWH +1173SPJ +1172YCM +1171MMD +1170STD +1169CCJ +1168CSJ +1167GKF +1166LPY +1165ROW +1164LCY +1163CHE +1162BRD +1161KPT +1160YPC +1159KLC +1158JSC +1157LYM +1156EOW +1155ATF +1154SCY +1153LGE +1152AAT +1151KAG +1150SWC +1149CEG +1148KNR +1147YCY +1146FFG +1145MYK +1144TCL +1144ASC +1143TCO +1143AVL +1142NEW +1141ADB +1140CHA +1138HWA +1137KTE +1136SSC +1135CTM +1134LEV +1133WSW +1132GJL +1131TSJ +1130EAS +1129DLM +1128ZSA +1127LMN +1126AES +1125MBW +1124CYJ +1123UEK +1122CCL +1121LAV +1120SLL +1119KIT +1118DVT +1117NVD +1116YEJ +1115NBY +1114RGS +1113KON +1112THW +1111VNL +1110YMY +1109HHG +1108CME +1107DKA +1106JTV +1105BKK +1104TCG +1103FSB +1102GTD +1101LSW +1100KTS +1099HOT +1098EET +1097H0H +1096WZW +1095SKL +1094LVL +1093SNR +1092BWH +1091SCA +1090WKL +1089KPP +1088CSM +1087MGS +1086MOE +1085TMS +1084MRC +1083TWE +1082ALC +1081HFZ +1080AMB +1079VFS +1078NST +1077WLE +1075BTT +1073WLK +1072EMS +1071LCZ +1070 +1069REX +1068LNS +1067MIG +1066CHP +1065DYJ +1064DBN +1063TTS +1062ANG +1061KYH +1060UGG +1059LDY +1057HTR +1056JMN +1055PGE +1054WWQ +1053PNA +1051JDA +1050GRP +1049LGC +1048BAC +1047AMN +1046MLV +1045CMS +1044CJH +1043KBT +1042SYB +1041VSB +1040DYL +103KST +1039BRO +1038MCD +1037KCW +1036RFD +1035JNS +1034TFA +1033PRF +1031LPE +1030HSE +1029TZB +1028NGK +1027SMM +1026SOC +1025NGD +1024CIT +1023WAC +1022SGW +1021NWH +1020VLC +1019 +1017JJB +1016PLY +1015WOD +1014KYN +1013HRB +1012YKS +1011ART +1010 +1009STJ +1008LAC +1007LCC +1005HMB +1004AST +1003NVE +1002STP +1001 +1000HYS +027WIN +1266TAN +1267KPD +1268LCS +1269LEC +1270KIR +1271GPL +1272APY +1273LTT +1274LTW +1275GES +1276LOP +1277FHG +1278MUK +1279LUI +1280ZTS +1281TTC +1282LYK +1283CZE +1284YWX +1285YWT +1286SUC +1287TKH +1288RSF +1289LSM +1290CSW +1291NSC +1292INT +1293YSV +1294GFI +1295LMW +1296YAM +1297SYZ +1298TOF +1299NEE +1300OPX +1301EXO +1302LEW +1303MUN +1304TIE +1305EMA +1306JDH +1307ECZ +1308GQK +1309YHC +1310BEU +1311NJE +1312BIG +1313SUK +1314FWJ +1315CHT +1316CHW +1317TEE +1318WOH +1319TRK +1320TEH +1321CSL +1322EBP +1323UNT +1324WYH +1325LWS +1326YHY +1327VQQ +1328SXX +1329YPY +1330SJV +1331OKH +1332CJJ +1333YTT +1334YKW +1335ADW +1336RGV +1337PFF +1338LZQ +1339LYQ +1340OTP +1341JCL +1342KIE +1343PST +1344WFF +1345TSB +1346EYK +1347THK +1348YFS +1349LST +1350KHC +1351LWT +1352DMA +1353YJC +1354SHS +1355CPH +1356JUN +1357GKM +1358YYS +1359CKL +1360KCH +1361WHH +1362TXN +1363KYY +1364SYS +1365YYY +1366YNK +1367LZS +1368ABB +1369LYV +1370LCN +1371TYT +1372ANW +1373SHH +1374BYF +1375SQY +1376MMN +1377SHC +1378YFP +1379UNA +1380EPK +1381GGE +1382PME +1383MXY +1384ITM +1385FGB +1386LML +1387WKY +1388LJC +1389TKS +1390THQ +1391LJY +1392GNS +1393ELS +1394HAU +1395HRY +1396ACH +1397TWL +1398JCC +1399WYM +1400OST +1401VAB +1402EFM +1403SCB +1404PSY +1405TKJ +1406KRL +1407LAI +1408CKY +1409TME +1410LCQ +1411AZL +1412TLW +1413TSL +1414TJY +1415JPC +1416BBE +1417TLS +1418WSX +1419CJS +1420SNL +1421LSG +1422TRS +1423ANN +1424CCP +1425TXH +1426FSK +1427HYY +1428TYQ +1429NHF +1430CLL +1431NGS +1432KKE +1433LPC +1434CMP +1435LKL +1436CSY +1437GCG +1438TPE +1439YSK +1440LCH +1441SES +1442KSY +1443LHY +1444GYT +1445OSY +1446WCC +1447CKW +1448WHN +1449HLE +1450LEG +1451LJW +1452JCI +1453MBL +1454LTH +1455LXY +1456SNY +1457TWY +1458WST +1459OOI +1460GHY +1461HWS +1462LWH +1463ESC +1464KSL +1465ASP +1466OYC +1467PPK +1468APV +1469CWY +1470LEL +1471NLS +1472CYS +1473NEX +1474CTW +1475ASV +1476CPT +1477KFH +1478LSL +1479GSY +1480CWL +1481CCK +1482VXR +1483WJY +1484TXY +1485KWC +1486HLP +1487CPY +1488LXE +1489NKS +1490WKS +1491LRB +1492SLK +1493LPP +1494CJT +1495THP +1496LJQ +1497MSF +1498LHP +1499LYG +1500NHL +1501CCS +1502CCH +1503JLK +1504MOT +1505KKM +1506SMW +1507FSE +1508CGB +1509LWJ +1510WEG +1511HCS +1512AIS +1513SYE +1514GKP +1515SXY +1516SWH +1517KBA +1518NLY +1519QHL +1520HMC +1521KYT +1522GMF +1523LSK +1524THL +1525AQA +1526NCA +1527STL +1528WGO +1529FCS +1530TMF +1531IVY +1532SWY +1533LAE +1534BNS +1535CMY +1536THK +1537KJT +1538ABS +1539ACY +1540BSS +1541CMR +1542YYT +1543EZY +1544YPA +1545WWL +1546LPS +1547SVY +1548TSS +1549BGV +1550WTL +1551YZH +1552GAC +1553KWY +1554LOS +1555HYL +1556DTC +1557CKC +1558CKN +1559GZQ +1560CSE +1561SEK +1562SVS +1563MME +1564CYM +1565CPG +1566TER +1567NSY +1568CSQ +1569EMX +1570MTI +1571NKY +1572LMF +1573SXW +1574MES +1575AWE +1576YCP +1577WJL +1578ACS +1579HLT +1580KLT +1581KYS +1582NJX +1583TRJ +1584INA +1585KPW +1586VGT +1587DMB +1588WLC +1589YYJ +1590WNT +1591MSY +1592IHK +1593LSJ +1594TKW +1595TFY +1596ANS +1597KJK +1598WCK +1599KZM +1600SLS +1601PIP +1602LPD +1603MTS +1604YYK +1605CXQ +1606YWY +1607ALM +1608OKM +1609KRM +1610CHL +1611PMW +1612EXX +1613ALA +1614ACK +1615LJN +1616ANC +1617ECR +1618ONM +1619SMS +1620HCR +1621ASO +1622OSW +1623CKT +1624MOZ +1625LSZ +1626YSF +1627ITG +1628KYX +1629GWS +1630WTE +1631PYC +1632YCL +1633WYC +1634PTP +1635YBL +1636ALY +1637LYH +1638KKT +1639LPH +1640DAL +1641YKT +1642LYT +1643JYC +1644IEN +1645TLC +1646NIK +1647HWK +1648CBH +1649BJY +1650WTH +1651LWK +1652YSW +1653WCB +1654MFZ +1655WYZ +1656HBP +1657HGR +1658MMT +1659TKC +1660GPT +1661BCS +1662PYX +1663YHL +1664KHJ +1665KMY +1666KLQ +1667PLE +1668PPG +1669YSN +1670WCT +1671KRA +1672SOO +1673EHB +1674NCE +1675DLF +1676CHI +1677HBB +1678DIO +1679TEA +1680CCX +1681YEC +1682NYH +1683JHA +1684XYS +1685HNG +1686SHO +1687SPP +1688LMC +1689TEO +1690YWZ +1691CKS +1692LAW +1693RET +1694YSL +1695GTK +1696NLW +1697TJW +1698HHT +1699JOC +1700SHT +1701OKE +1702OCY +1703OSM +1704LZX +1705JCT +1706YSD +1707KKW +1708WHT +1709DDL +1710LZW +1711CAK +1712HME +1713SFM +1714EOH +1715PAS +1716TGS +1717CHA +1718LIO +1719CTY +1720TTD +1721SRI +1722YJW +1723SWJ +1724HCY +1725NCP +1726OLS +1727SEN +1728CHE +1729SEE +1730ONG +1731UCA +1732SYC +1733HAC +1734PHO +1735CJA +1736CSK +1737LYY +1738CYH +1739WDK +1740KMK +1741PYL +1742WAG +1743ADO +1744KZQ +1745YST +1746BKC +1747TVE +1748PSL +1749TLL +1750 +1751CTE +1752TCH +1753CKK +1754XQC +1755MON +1756TBH +1757AVE +1758VKH +1759SAS +1760IBO +1761GCY +1762LEB +1763TJQ +1764HAH +1765FOL +1766OCT +1767WLF +1768WKC +1769THM +1770HKL +1771HIN +1772MAX +1773FAN +1774YWB +1775VTY +1776CSR +1777MYY +1778OCK +1779XYT +1780MCW +1781JBI +1782EHY +1783ITL +1784ANL +1785KPL +1786MIN +1787GME +1788BER +1789CER +1790KAR +1791JAL +1792PCL +1793JNO +1794NKG +1795TMK +1796JKT +1797EJH +1798TKA +1799OKN +1800SFN +1801MKO +1802GYX +1803NHG +1804OAP +1805CYL +1806TSY +1807NYR +1808NCM +1809TAK +1810SEW +1811MAY +1812VOL +1813YZT +1814YXN +1815FNY +1816ZZE +1817HJY +1818DSN +1819TTF +1820SUI +1821FFS +1822NKN +1823JVL +1824LGP +1825LSH +1826FCH +1827OPN +1828VTM +1829CSX +1830OZT +1831MFS +1832PKW +1833SNG +1834ZST +1835TLY +1836LYJ +1837SYH +1838TEI +1839HOW +1840ECK +1841CTR +1842SKW +1843HSW +1844SJS +1845JKC +1846YYC +1847JEF +1848YSY +1849BLC +1850LCP +1851NHK +1852BTY +1853CHY +1854KEC +1855OSE +1856YXY +1857CDW +1858YKL +1859EVC +1860RCC +1861GGD +1862JQI +1863LMB +1864TKG +1865VSC +1866PPY +1867EEH +1868WSL +1869ITS +1870TYW +1871SKS +1872TZH +1873SDC +1874HNS +1875SRK +1876VMM +1877HHK +1878PYK +1879LQL +1880TXZ +1881AGL +1882KAI +1883SWB +1884ZHT +1885NPM +1886VHH +1887LMP +1888KLL +1889LBH +1890JJP +1891BSM +1892JLN +1893TAS +1894BLE +1895KVC +1896SCK +1897SNH +1898SJK +1899KBE +1900KMH +1901BSV +1902WMQ +1903THY +1904AKY +1905JYW +1906CHE +1907TCF +1908TXE +1909LHX +1910KVU +1911MDJ +1912OYE +1913TMH +1914PCS +1915HIV +1916LIN +1917MUL +1918PPM +1919KOK +1920MAA +1921RUA +1922YMZ +1923KME +1924SLN +1925ELF +1926WRQ +1927HUI +1928LVV +1929YOU +1930AOR +1931GLT +1932TEI +1933FJY +1934ILE +1935LOC +1936OJO +1937LOT +1938LJS +1939JKF +1940CCY +1941BQG +1942YHO +1943LCW +1944TST +1945TTK +1946TXK +1947WZX +1948EWT +1949TCE +1950JIE +1951KML +1952LII +1953NZY +1954CSC +1955PEI +1956CWH +1957MKB +1958LEO +1959TZH +1960GGF +1961SWK +1962WTW +1963GEC +1964CYE +1965LJL +1966SHM +1967NHS +1968YLX +1969YST +1970PEE +1971TBS +1972YEE +1973RWT +1974SLC +1975TXV +1976YAT +1977LKP +1978YHA +1979WEK +1980TSW +1981TCY +1982WPC +1983FIR +1984TEP +1985GOO +1986TPY +1987LYE +1988SPY +1989NVG +1990TAI +1991CRY +1992FKH +1993ISO +1994EAY +1995WSQ +1996LWY +1997YCC +1998CYT +1999CYY +2000GMH +2001HON +2002DON +2003XNT +2004WSA +2005TCW +2006YTH +2007CEW +2008PMD +2009LYB +2010YEO +2011JAK +2012EBG +2013LMK +2014CEE +2015YSQ +2016AIR +2017YJH +2018HMJ +2019NYJ +2020WKW +2021RIC +2022SKU +2023KLS +2024CKH +2025CYG +2026CHX +2027HYW +2028WSG +2029LCY +2030PAK +2031WES +2032EYM +2033KWM +2034NYM +2035ATS +2036TKL +2037LKE +2038KKC +2039TAJ +2040LEH +2041DSP +2042COS +2043OTH +2044WLL +2045THI +2046LSP +2047WYF +2048LKW +2049SMH +2050BUS +2051SOA +2052LCO +2053TKM +2054KCC +2055CHB +2056KWL +2057CWT +2058WSY +2059CYJ +2060PFT +2061BMK +2062LIY +2063MMM +2064HMT +2065CXY +2066LHS +2067WCS +2068FZK +2069CSF +2070LCD +2071VEE +2072HYE +2073JHE +2074TMY +2075SKH +2076EHP +2077MYE +2078SHP +2079HYA +2080AMN +2081AKT +2082YEK +2083LOV +2084KOY +2085LAJ +2086WSC +2087IRR +2088THJ +2089WFC +2090GRK +2091PXY +2092FSS +2093CHJ +2094CSZ +2095GHW +2096LIH +2097CHC +2098SPS +2099BPB +2100JYL +2101WMF +2102LME +2103MZF +2104WOL +2105CLN +2106KAY +2107KHH +2108JSW +2109ALW +2110DZE +2111HSJ +2112WJH +2113CUS +2114WDY +2115KTG +2116MXW +2117HHS +2118RSS +2119ICD +2120XYL +2121PYT +2122TWS +2123MNS +2124SUL +2125WCL +2126KDT +2127SCC +2128MHM +2129OMY +2130BEO +2131BEM +2132MWE +2133JVN +2134LIC +2135SEH +2136KAP +2137LPB +2138HUE +2139ASE +2140TYJ +2141SUB +2142CWZ +2143MYW +2144XYY +2145KCZ +2146KCS +2147LXW +2148EYU +2149MSP +2150NPJ +2151YQP +2152PTJ +2153SBC +2154CCR +2155MLD +2156DHF +2157GFC +2158TYA +2159TZY +2160KGL +2161WKD +2162DNP +2163IFH +2164DRF +2165SNX +2166PSJ +2167TJZ +2168WJJ +2169WPH +2170MXA +2171YMK +2172IPS +2173JWT +2174JSH +2175CPK +2176JCS +2177BRL +2178SWS +2179SHK +2180PWS +2181KNK +2182RXJ +2183PCC +2184PWY +2185MIK +2186HSL +2187SYG +2188YZY +2189LKN +2190ESN +2191HWD +2192KAA +2193JNG +2194JST +2195PTC +2196KWE +2197NSS +2198VVM +2199GJH +2200LBB +2201SHL +2202SYV +2203HST +2204YNG +2205BFH +2206SWL +2207ICY +2208JGO +2209MNB +2210ZHC +2211YKH +2212LKC +2213MPT +2214KHY +2215AWK +2216JSU +2217YCF +2218HKY +2219TSF +2220FCY +2221SWF +2222WWG +2223LMY +2224HYH +2225HPL +2226CML +2227LVM +2228TSP +2229PHL +2230YJY +2231KKD +2232PQH +2233TTN +2234UQX +2235FRH +2236UET +2237YHM +2238CKG +2239LQS +2240CLC +2241GPS +2242KTZ +2243SKB +2244WHL +2245NKC +2246CXJ +2247HNM +2248CHQ +2249CNT +2250SCT +2251JWC +2252KTC +2253TOH +2254KSB +2255TGT +2256CKM +2257YCG +2258MYX +2259NJJ +2260SUY +2261JDC +2262CGH +2263MWR +2264EWE +2265PFK +2266VOT +2267LJT +2268LZY +2269WOY +2270LHH +2271EZB +2272KYE +2273LNG +2274YMP +2275SMG +2276YKY +2277KZY +2278HTL +2279AMT +2280JNL +2281CLF +2282XYE +2283EMI +2284TNG +2285MNM +2286LCA +2287EMD +2288JML +2289LMH +2290HNB +2291SML +2292WHI +2293NTH +2294FCM +2295DDT +2296HLV +2297WIC +2298CCM +2299SFS +2300MAE +2301PAW +2302DND +2303BDA +2304KLN +2305CPB +2306BAF +2307OPE +2308JHL +2309SLH +2310CJX +2311OET +2312ATB +2313ZEN +2314VIZ +2315WLT +2316MEF +2317KTL +2318NNB +2319CST +2320JTY +2321YWC +2322TKR +2323KCL +2324HNW +2325FRT +2326SLW +2327KVP +2328ABR +2329SYP +2330GTY +2331HGT +2332BOP +2333JEY +2334SKN +2335RWW +2336CRC +2337UTS +2338SSY +2339THN +2340WJR +2341JRM +2342MML +2343HYD +2324SOK +2345GYC +2346SCH +2347CXC +2348PCY +2349MLL +2350ODY +2351WYX +2352YLC +2353GFE +2354NHJ +2355TYK +2356KJS +2357CYX +2358LON +2359NJN +2360TYX +2361EJL +2362NSH +2363ANO +2364TEJ +2365PMN +2366/GSM +2367/NTS +2368/FAE +2369/CJL +2370/NBB +2371/AAL +2372/NIE +2373/TCC +2374/CSH +2375/AGP +2376FRW +2377SYY +2378LPK +2379CHO +2380WKJ +2381LSE +2382WRJ +2383LWC +2384LGY +2385/ABF +2386SET +2387BAB +2388YTY +2389IGE +2390SQI +2391KER +2392JYG +2393WNJ +2394LCJ +2395XGJ +2396CLZ +2397WZY +2398YEW +2399SMN +2400CEL +2401FKP +2402JOY +2403AHW +2404CET +2405MCK +2406DNE +2407PIQ +2408BMO +2409OWS +2410HAP +2411LZE +2412WHE +2413BBC +2414YIE +2415CSB +2416YOK +2417NDB +2418KSM +2419YXW +2420OOS +2421STO +2422VLB +2423ARE +2424AJS +2425OBL +2426HLY +2427YLY +2428TYH +2429KWX +2430CCW +2431SOI +2432TCZ +2433CHU +2434FJJ +2435LYF +2436YIK +2437LXJ +2438PIK +2439TSR +2440LLG +2441ENG +2442OAN +2443CGP +2444KCV +2445GMW +2446PLH +2447EIY +2448GPK +2449HBT +2450SFL +2451NEO +2452BCG +2453FZT +2454CPF +2455KMT +2456MIA +2457BSY +2458BLO +2459DWI +2460CKX +2461SCS +2462YSZ +2463ESP +2464YZM +2465ELT +2466ADE +2467AFS +2468NER +2469ALB +2470LKT +2471PRL +2472TAH +2473EZA +2474YLS +2475YMH +2476ETE +2477XFF +2478ZES +2479TTH +2480ABC +2481MOK +2482QBH +2483MSR +2484WKK +2485NWY +2486KTT +2487EEE +2488TBE +2489THS +2490LHL +2491WJQ +2492KLA +2493CIY +2494CPS +2495ITE +2496RAK +2497JCH +2498LOO +2499SUR +2500SRE +2501MYL +2502SOW +2503JMS +2504TGX +2505HYF +2506CNL +2507GDT +2508ENI +2509CLB +2510KDS +2511KHO +2512JSP +2513OON +2514HOY +2515CMJ +2516TOR +2517RYG +2518TTT +2519TIN +2520YHH +2521DMW +2522PIN +2523IRA +2524HCM +2525NHY +2526HTM +2527MSL +2528GBF +2529SZU +2530LXQ +2531YGL +2532OBJ +2533MMA +2534JWL +2535GPA +2536MSA +2537VHM +2538KSO +2539ENN +2540KLG +2541BEL +2542CAO +2543ONE +2544MTR +2545YIS +2546TYE +2547TWQ +2548WYI +2549KLW +2550LES +2551EYL +2552YML +2553JNY +2554GAI +2555FER +2556KWZ +2557HIG +2558CAF +2559CCI +2560KKA +2561SBL +2562TSE +2563YAP +2564PCA +2565CNM +2566THE +2567KKH +2568NYI +2569NOR +2570SXQ +2571FCL +2572LTA +2573PTH +2574XYC +2575DDE +2576WYL +2577KOC +2578AZT +2579YIN +2580HEU +2581MEN +2582USA +2583MLE +2584WEO +2585IPA +2586AFM +2587SCE +2588FHP +2589FUS +2590PAY +2591VIS +2592 +2593WON +2594MOO +2595IDO +2596UNS +2597OWN +2598IMX +2599EPT +2600BAL +2601BEE +2602ITY +2603DEN +2604SZE +2605LID +2606TAR +2607PCB +2608QUE +2609HYI +2610DAY +2611PSO +2612HIN +2613ZYS +2614YGE +2615OWC +2616DEE +2617GLO +2618HOO +2619NLM +2620WIL +2621FYF +2622DEX +2623KEO +2624VUN +2625QUA +2626TXQ +2627FRA +2628GAN +2629ONE +2630YEE +2631MIN +2632LEN +2633TTY +2634PLT +2635KUG +2636EPM +2637EEA +2638PGP +2639NUH +2640KIO +2641RLO +2642VHK +2643TTV +2644EZL +2645NRA +2646KYP +2647HTC +2648XWT +2649TAW +2650KKB +2651MKT +2652KCU +2653SCP +2654YEL +2655IUE \ No newline at end of file From 6c63f37d0ca99cc1cde0ec8c649363568dd1af0d Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:39:15 +0000 Subject: [PATCH 03/13] Debug calculation formula to avoid infinity when log(0) is encountered --- .../rep_exchange__survival_analysis.sql | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/models/marts/reporting/rep_exchange__survival_analysis.sql b/models/marts/reporting/rep_exchange__survival_analysis.sql index 046dd4d..15fea1d 100644 --- a/models/marts/reporting/rep_exchange__survival_analysis.sql +++ b/models/marts/reporting/rep_exchange__survival_analysis.sql @@ -17,7 +17,7 @@ transaction_orders AS ( -- LOGIC -- duration_table cte computes the survival time and customer churned indicator duration_table AS ( - + SELECT companies.company_id, --subject companies.company_created_datetime, -- event start datetime @@ -107,7 +107,6 @@ compute_censored_subjects AS ( ), - compute_probability AS ( SELECT @@ -115,17 +114,27 @@ compute_probability AS ( -- The survival probability represents the probability of customers that will not churn up to a specific tenure -- Example: survival_day = 61, survival_prob = 95%. For customers with 61 days of tenure, the customer has a 95% chance of not churning. - EXP(SUM(LN(1 - events / at_risk)) OVER ( - ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - )) AS survival_probability, + -- When events / at_risk = 1, we replace value 1 with a value very near to 1, to prevent log 0 which causes infinity value + EXP(SUM(LN + ( + CASE WHEN (1 - events / at_risk) = 0 THEN 0.999999 ELSE (1 - events / at_risk) END + )) + OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )) AS survival_probability, - 100 * (1 - EXP(SUM(LN(1 - events / at_risk)) OVER ( - ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ))) AS conversion_percentage, + 100 * (1 - EXP(SUM(LN + ( + CASE WHEN (1 - events / at_risk) = 0 THEN 0.999999 ELSE (1 - events / at_risk) END + )) + OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ))) AS conversion_percentage, - SUM(events / at_risk) OVER ( + SUM(events / at_risk) + OVER ( ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ) AS cumulative_hazard, + ) AS cumulative_hazard, '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime From b285a9546d83ba99d074d72717ecb9c6b269ec76 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:40:01 +0000 Subject: [PATCH 04/13] Shipping data model - cohort analysis development --- .../rep_shipping__cohort_analysis.sql | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__cohort_analysis.sql diff --git a/models/marts/reporting/rep_shipping__cohort_analysis.sql b/models/marts/reporting/rep_shipping__cohort_analysis.sql new file mode 100644 index 0000000..a35ffac --- /dev/null +++ b/models/marts/reporting/rep_shipping__cohort_analysis.sql @@ -0,0 +1,212 @@ +-- SOURCE: https://www.holistics.io/blog/calculate-cohort-retention-analysis-with-sql/ + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + + +-- LOGIC +company_register_month_year AS ( + + SELECT + companies.sub_company_id, + companies.sub_company_marking_id, + companies.sub_company_created_datetime, + + IFF( + MAX(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) IS NULL, + 1, + 0 + ) AS is_never_order_company, + + DATE(DATE_TRUNC( + 'MONTH', + companies.sub_company_created_datetime + )) AS register_month_year, + + DATE(DATE_TRUNC( + 'MONTH', + MIN(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) + )) AS first_order_month_year + + FROM companies + + LEFT JOIN origin_warehouse_shipping_packing_lists + ON (companies.sub_company_id = origin_warehouse_shipping_packing_lists.sub_company_id) + + GROUP BY + companies.sub_company_id, + companies.sub_company_marking_id, + companies.sub_company_created_datetime, + register_month_year + +), + +company_order_count_from_registration_date AS ( + + SELECT + origin_warehouse_shipping_packing_lists.sub_company_id, + + DATEDIFF( + MONTH, + company_register_month_year.sub_company_created_datetime, + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime + ) AS order_month + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN company_register_month_year + ON (origin_warehouse_shipping_packing_lists.sub_company_id = company_register_month_year.sub_company_id) + + GROUP BY + origin_warehouse_shipping_packing_lists.sub_company_id, + order_month + +), + +company_first_order_from_registration_date AS ( + + SELECT + origin_warehouse_shipping_packing_lists.sub_company_id, + DATE(DATE_TRUNC('MONTH', sub_company_created_datetime)) AS sub_company_created_datetime, + + MIN(DATEDIFF( + MONTH, + company_register_month_year.sub_company_created_datetime, + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime + )) AS first_order_month + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN company_register_month_year + ON (origin_warehouse_shipping_packing_lists.sub_company_id = company_register_month_year.sub_company_id) + + GROUP BY + origin_warehouse_shipping_packing_lists.sub_company_id, + sub_company_created_datetime + +), + +cohort_size_by_month_year AS ( + + SELECT + register_month_year, + SUM(is_never_order_company) AS count_never_order_company, + COUNT(register_month_year) AS count_total_company + + FROM company_register_month_year + + GROUP BY register_month_year + + ORDER BY register_month_year + +), + +order_retention AS ( + + SELECT + company_register_month_year.register_month_year, + company_order_count_from_registration_date.order_month, + COUNT(company_register_month_year.register_month_year) AS count_retained_company + + FROM company_order_count_from_registration_date + + LEFT JOIN company_register_month_year + ON (company_order_count_from_registration_date.sub_company_id = company_register_month_year.sub_company_id) + + GROUP BY + company_register_month_year.register_month_year, + company_order_count_from_registration_date.order_month + +), + +first_time_order_retention AS ( + + SELECT + sub_company_created_datetime, + first_order_month, + COUNT(first_order_month) AS count_first_order_company + + FROM company_first_order_from_registration_date + + GROUP BY + sub_company_created_datetime, + first_order_month + +), + +cohort_analysis_table AS ( + + SELECT + order_retention.register_month_year, + order_retention.order_month, + order_retention.count_retained_company AS retained_company, + + cohort_size_by_month_year.count_total_company AS total_registered_company, + cohort_size_by_month_year.count_never_order_company, + + first_time_order_retention.count_first_order_company AS count_first_order_company, + + DIV0( + retained_company, + total_registered_company + ) AS retention_rate_overall, + + DIV0( + retained_company, + ( total_registered_company - cohort_size_by_month_year.count_never_order_company ) + ) AS retention_rate_exclude_never_order_company, + + DIV0( + count_first_order_company, + ( total_registered_company - count_never_order_company ) + ) AS first_order_percentage, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM order_retention + + LEFT JOIN cohort_size_by_month_year + ON (order_retention.register_month_year = cohort_size_by_month_year.register_month_year) + + LEFT JOIN first_time_order_retention + ON (order_retention.register_month_year = first_time_order_retention.sub_company_created_datetime) + AND (order_retention.order_month = first_time_order_retention.first_order_month) + + ORDER BY + order_retention.register_month_year, + order_retention.order_month + +), + + +-- FINAL +final_rep_shipping__cohort_analysis AS ( + + SELECT + -- dimensions + register_month_year, + total_registered_company, + order_month, + + -- measures + count_never_order_company, + count_first_order_company, + retained_company, + retention_rate_overall, + retention_rate_exclude_never_order_company, + first_order_percentage, + + -- metadata + _dbt_ran_datetime + + FROM cohort_analysis_table + +) + +SELECT * FROM final_rep_shipping__cohort_analysis \ No newline at end of file From 4514c35db401d1e628142feb3e247f0d206f0751 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:41:36 +0000 Subject: [PATCH 05/13] Shipping data model - company engagement behavior --- ...shipping__company_engagement_behaviors.sql | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__company_engagement_behaviors.sql diff --git a/models/marts/reporting/rep_shipping__company_engagement_behaviors.sql b/models/marts/reporting/rep_shipping__company_engagement_behaviors.sql new file mode 100644 index 0000000..095da25 --- /dev/null +++ b/models/marts/reporting/rep_shipping__company_engagement_behaviors.sql @@ -0,0 +1,236 @@ +-- AVAILABLE FILTER VALUE +{% set day_use_to_churn = 120 %} + + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + + +-- LOGIC +remove_supplier_company AS ( + + SELECT * FROM companies + + WHERE + business_type IN ('IMPORTER', 'ONLINE_SELLER') + +), + +order_date_row_rank AS ( + + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY sub_company_id ORDER BY origin_warehouse_shipping_order_created_datetime, origin_warehouse_shipping_packing_list_id + ) AS row_rank_index + + FROM origin_warehouse_shipping_packing_lists + +), + +get_orders_of_next_and_previous_order_date AS ( + + SELECT + order_date_row_rank.*, + companies.sub_company_created_datetime, + companies.sub_company_marking_id, + companies.is_migrated_company, + previous_order_date_row_rank_clone.origin_warehouse_shipping_order_created_datetime AS previous_order_created_datetime + + FROM order_date_row_rank + + LEFT JOIN companies + ON (order_date_row_rank.sub_company_id = companies.sub_company_id) + + LEFT JOIN order_date_row_rank AS next_order_date_row_rank_clone + ON (order_date_row_rank.row_rank_index = next_order_date_row_rank_clone.row_rank_index - 1 + AND order_date_row_rank.sub_company_id = next_order_date_row_rank_clone.sub_company_id) + + LEFT JOIN order_date_row_rank AS previous_order_date_row_rank_clone + ON (order_date_row_rank.row_rank_index = previous_order_date_row_rank_clone.row_rank_index + 1 + AND order_date_row_rank.sub_company_id = previous_order_date_row_rank_clone.sub_company_id) + +), + +-- Register / Churn / Register; Churn +company_register_events AS ( + + SELECT + remove_supplier_company.sub_company_id, + remove_supplier_company.sub_company_marking_id, + remove_supplier_company.is_migrated_company, + remove_supplier_company.sub_company_created_datetime AS event_datetime, + 'Register' AS register_event, + + IFF( + MAX(origin_warehouse_shipping_packing_lists.sub_company_id) IS NULL, + 1, + 0 + ) AS is_never_order_company, + + IFF( + DATEDIFF( + DAY, + remove_supplier_company.sub_company_created_datetime, + COALESCE(MIN(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime), CURRENT_DATE()) + ) >= {{day_use_to_churn}}, + 'Churn', + NULL + ) AS churn_event, + + RTRIM( + CONCAT( + CASE WHEN register_event IS NOT NULL THEN 'Register, ' ELSE '' END, + CASE WHEN churn_event IS NOT NULL THEN 'Churn, ' ELSE '' END + ), + ', ' + ) AS concat_event + + FROM remove_supplier_company + + LEFT JOIN origin_warehouse_shipping_packing_lists + ON (remove_supplier_company.sub_company_id = origin_warehouse_shipping_packing_lists.sub_company_id) + + GROUP BY + remove_supplier_company.sub_company_id, + remove_supplier_company.sub_company_marking_id, + remove_supplier_company.is_migrated_company, + remove_supplier_company.sub_company_created_datetime + +), + +-- FirstTimeOrder / Repeat / Churn / Reactive +first_time_repeat_churn_reactive_order_events AS ( + + SELECT + sub_company_id, + sub_company_marking_id, + is_migrated_company, + 0 AS is_never_order_company, + origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_order_status, + origin_warehouse_shipping_order_created_datetime AS event_datetime, + service_type, + total_package_CBM, + + IFF(is_first_time_sub_company = 1, 'First Time Order', NULL) AS first_time_order_event, + IFF(is_first_time_sub_company = 0, 'Repeat', NULL) AS repeat_event, + IFF( + DATEDIFF( + DAY, + origin_warehouse_shipping_order_created_datetime, + COALESCE(next_origin_warehouse_shipping_order_created_datetime, CURRENT_DATE()) + ) >= {{day_use_to_churn}}, + 'Churn', + NULL + ) AS churn_event, + + IFF( + DATEDIFF( + DAY, + COALESCE(previous_order_created_datetime, sub_company_created_datetime), + origin_warehouse_shipping_order_created_datetime + ) >= {{day_use_to_churn}}, + 'Reactive', + NULL + ) AS reactive_event, + + RTRIM(CONCAT( + CASE WHEN first_time_order_event IS NOT NULL THEN 'First Time Order, ' ELSE '' END, + CASE WHEN repeat_event IS NOT NULL THEN 'Repeat, ' ELSE '' END, + CASE WHEN reactive_event IS NOT NULL THEN 'Reactive, ' ELSE '' END, + CASE WHEN churn_event IS NOT NULL THEN 'Churn, ' ELSE '' END + ), + ', ' + ) AS concat_event + + FROM get_orders_of_next_and_previous_order_date +), + +union_event_tables AS ( + + SELECT + sub_company_id, + sub_company_marking_id, + is_migrated_company, + is_never_order_company, + NULL AS origin_warehouse_shipping_packing_list_id, + NULL AS origin_warehouse_shipping_order_status, + NULL AS service_type, + NULL AS total_package_CBM, + NULL AS first_time_order_event, + NULL AS repeat_event, + NULL AS reactive_event, + register_event, + churn_event, + concat_event, + event_datetime + + + FROM company_register_events + + UNION + + SELECT + sub_company_id, + sub_company_marking_id, + is_migrated_company, + is_never_order_company, + origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_order_status, + service_type, + total_package_CBM, + first_time_order_event, + repeat_event, + reactive_event, + NULL AS register_event, + churn_event, + concat_event, + event_datetime + + FROM first_time_repeat_churn_reactive_order_events + +), + + +-- FINAL +final__rep_shipping__company_engagement_behaviors AS ( + + SELECT + -- id + sub_company_id, + sub_company_marking_id, + origin_warehouse_shipping_packing_list_id + + -- dimension + is_migrated_company, + is_never_order_company, + origin_warehouse_shipping_order_status, + service_type, + first_time_order_event, + repeat_event, + reactive_event, + register_event, + churn_event, + concat_event, + + -- measure + total_package_CBM, + + -- date/time + event_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM union_event_tables + +) + +SELECT * FROM final__rep_shipping__company_engagement_behaviors \ No newline at end of file From cfc1e36d9a1885b1687985612eee5c35e26018ff Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:42:08 +0000 Subject: [PATCH 06/13] Shipping data model - company lists --- .../reporting/rep_shipping__company_lists.sql | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__company_lists.sql diff --git a/models/marts/reporting/rep_shipping__company_lists.sql b/models/marts/reporting/rep_shipping__company_lists.sql new file mode 100644 index 0000000..dae613b --- /dev/null +++ b/models/marts/reporting/rep_shipping__company_lists.sql @@ -0,0 +1,96 @@ +-- AVAILABLE FILTER VALUE +-- SOURCE: preset_custom_filter Column: day_use_to_churn +{% set day_use_to_churn = 120 %} + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + + +-- LOGIC +company_agg AS ( + + SELECT + companies.sub_company_id, + companies.sub_company_marking_id, + companies.is_migrated_company, + companies.sub_company_created_datetime, + + COUNT(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id) AS count_order, + MAX(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) AS last_order_created_datetime, + COALESCE(last_order_created_datetime, sub_company_created_datetime) AS last_activity_datetime, + + COUNT(IFF( + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_status IN ('APPROVED', 'COMPLETED'), + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id, + null + )) AS count_approved_completed_order, + + IFF(MAX( + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime IS NULL + ), 1, 0 + ) AS is_never_order_company, + + IFF(DATEDIFF( + day, last_activity_datetime, CURRENT_DATE() + ) >= {{day_use_to_churn}}, + 1, 0 + ) AS is_churn_company, + + IFF(is_churn_company = 1, + DATEDIFF(day, sub_company_created_datetime, last_activity_datetime) + {{day_use_to_churn}}, + null + ) AS create_to_churn_day, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies + + LEFT JOIN origin_warehouse_shipping_packing_lists + ON (companies.sub_company_id = origin_warehouse_shipping_packing_lists.sub_company_id) + + GROUP BY + companies.sub_company_id, + companies.sub_company_marking_id, + companies.is_migrated_company, + companies.sub_company_created_datetime + +), + + +-- FINAL +final__rep_shipping__company_lists AS ( + + SELECT + -- id + sub_company_id, + sub_company_marking_id, + + -- dimensions + is_migrated_company, + is_never_order_company, + is_churn_company, + + -- measures + count_order, + count_approved_completed_order, + create_to_churn_day, + + -- date/time + sub_company_created_datetime, + last_order_created_datetime, + last_activity_datetime, + + -- metadata + _dbt_ran_datetime + + FROM company_agg + +) + +SELECT * FROM final__rep_shipping__company_lists \ No newline at end of file From a4c4fc83cb07154e2325e2471ca2aa2d5f7fdd12 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:42:53 +0000 Subject: [PATCH 07/13] Shipping data model - rfm --- models/marts/reporting/rep_shipping__rfm.sql | 303 +++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__rfm.sql diff --git a/models/marts/reporting/rep_shipping__rfm.sql b/models/marts/reporting/rep_shipping__rfm.sql new file mode 100644 index 0000000..95c29c6 --- /dev/null +++ b/models/marts/reporting/rep_shipping__rfm.sql @@ -0,0 +1,303 @@ +-- AVAILABLE FILTER VALUE +-- Alot of companies do not have +{% set custom_one_year_from_date = modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")).date() - modules.datetime.timedelta(365) %} +{% set custom_one_year_to_date = modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")).date() %} + +-- RFM list values defined using percentile from dashboard [SHIPPING] rfm +{% set recency_list_lifetime = [120,240,360,480] %} +{% set frequency_list_lifetime = [1,2,8,24] %} +{% set monetary_list_lifetime = [0.74,3.01,7.87,27.27] %} + +{% set recency_list_one_year = [120,240,360,480] %} +{% set frequency_list_one_year = [1,2,7,16] %} +{% set monetary_list_one_year = [0.76,3.00,7.36,22.80] %} + + +-- IMPORT +WITH origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + +companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + + +-- LOGIC +company_shipping_order_one_year AS( + + SELECT + origin_warehouse_shipping_packing_lists.sub_company_id, + companies.sub_company_marking_id, + COUNT(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id) AS number_of_packing_list_one_year, + MAX(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) AS last_packing_list_created_datetime_one_year, + DATEDIFF(day, last_packing_list_created_datetime_one_year, CURRENT_DATE()) AS day_after_last_packing_list_one_year, + SUM(origin_warehouse_shipping_packing_lists.total_package_cbm) AS total_package_cbm_one_year + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN companies + ON (origin_warehouse_shipping_packing_lists.sub_company_id = companies.sub_company_id) + + WHERE + origin_warehouse_shipping_order_status IN ('COMPLETED','APPROVED') + AND + origin_warehouse_shipping_order_created_datetime >= '{{ custom_one_year_from_date }}' + AND + origin_warehouse_shipping_order_created_datetime < '{{ custom_one_year_to_date }}' + + GROUP BY + origin_warehouse_shipping_packing_lists.sub_company_id, + companies.sub_company_marking_id + +), + +company_rfm AS ( + + SELECT + origin_warehouse_shipping_packing_lists.sub_company_id, + companies.sub_company_marking_id, + + MAX(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) AS last_packing_list_created_datetime, + + DATEDIFF(day, last_packing_list_created_datetime, CURRENT_DATE() ) AS day_after_last_packing_list_lifetime, + MAX(company_shipping_order_one_year.day_after_last_packing_list_one_year) AS day_after_last_packing_list_one_year, + + COUNT(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id) AS number_of_packing_list_lifetime, + MAX(company_shipping_order_one_year.number_of_packing_list_one_year) AS number_of_packing_list_one_year, + + SUM(origin_warehouse_shipping_packing_lists.total_package_cbm) AS total_package_cbm_lifetime, + MAX(company_shipping_order_one_year.total_package_cbm_one_year) AS total_package_cbm_one_year + + FROM origin_warehouse_shipping_packing_lists + + LEFT JOIN company_shipping_order_one_year + ON (origin_warehouse_shipping_packing_lists.sub_company_id = company_shipping_order_one_year.sub_company_id) + + LEFT JOIN companies + ON (origin_warehouse_shipping_packing_lists.sub_company_id = companies.sub_company_id) + + WHERE + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_status IN ('COMPLETED','APPROVED') + + GROUP BY + origin_warehouse_shipping_packing_lists.sub_company_id, + companies.sub_company_marking_id + +), + +calculate_rfm AS ( + + SELECT + *, + + CASE + WHEN day_after_last_packing_list_lifetime <= {{recency_list_lifetime[0]}} THEN 5 + WHEN (day_after_last_packing_list_lifetime > {{recency_list_lifetime[0]}} AND day_after_last_packing_list_lifetime <= {{recency_list_lifetime[1]}}) THEN 4 + WHEN (day_after_last_packing_list_lifetime > {{recency_list_lifetime[1]}} AND day_after_last_packing_list_lifetime <= {{recency_list_lifetime[2]}}) THEN 3 + WHEN (day_after_last_packing_list_lifetime > {{recency_list_lifetime[2]}} AND day_after_last_packing_list_lifetime <= {{recency_list_lifetime[3]}}) THEN 2 + WHEN day_after_last_packing_list_lifetime > {{recency_list_lifetime[3]}} THEN 1 + ELSE -999 + END AS r_score_lifetime, + + CASE + WHEN number_of_packing_list_lifetime <= {{frequency_list_lifetime[0]}} THEN 1 + WHEN (number_of_packing_list_lifetime > {{frequency_list_lifetime[0]}} AND number_of_packing_list_lifetime <= {{frequency_list_lifetime[1]}}) THEN 2 + WHEN (number_of_packing_list_lifetime > {{frequency_list_lifetime[1]}} AND number_of_packing_list_lifetime <= {{frequency_list_lifetime[2]}}) THEN 3 + WHEN (number_of_packing_list_lifetime > {{frequency_list_lifetime[2]}} AND number_of_packing_list_lifetime <= {{frequency_list_lifetime[3]}}) THEN 4 + WHEN number_of_packing_list_lifetime > {{frequency_list_lifetime[3]}} THEN 5 + ELSE -999 + END AS f_score_lifetime, + + CASE + WHEN total_package_cbm_lifetime <= {{monetary_list_lifetime[0]}} THEN 1 + WHEN (total_package_cbm_lifetime > {{monetary_list_lifetime[0]}} AND total_package_cbm_lifetime <= {{monetary_list_lifetime[1]}}) THEN 2 + WHEN (total_package_cbm_lifetime > {{monetary_list_lifetime[1]}} AND total_package_cbm_lifetime <= {{monetary_list_lifetime[2]}}) THEN 3 + WHEN (total_package_cbm_lifetime > {{monetary_list_lifetime[2]}} AND total_package_cbm_lifetime <= {{monetary_list_lifetime[3]}}) THEN 4 + WHEN total_package_cbm_lifetime > {{monetary_list_lifetime[3]}} THEN 5 + ELSE -999 + END AS m_score_lifetime, + + CASE + WHEN (r_score_lifetime = 5) AND (f_score_lifetime = 5) AND (m_score_lifetime = 5) + THEN '1_King' + + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime = 1) AND (m_score_lifetime = 1) + THEN '10_At Cheap Risk' + + WHEN (r_score_lifetime = 1) AND (f_score_lifetime = 1) AND (m_score_lifetime = 1) + THEN '13_Cheap Lost' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 1 AND 2) AND (m_score_lifetime BETWEEN 4 AND 5) + THEN '4_Whale' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 4 AND 5) AND (m_score_lifetime BETWEEN 1 AND 2) + THEN '3_Sardine' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 3 AND 5) AND (m_score_lifetime BETWEEN 3 AND 5) + THEN '2_Loyal Customer' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 2 AND 5) AND (m_score_lifetime BETWEEN 2 AND 5) + THEN '5_Potention Loyal' + + WHEN (r_score_lifetime = 5) AND (f_score_lifetime = 1) AND (m_score_lifetime BETWEEN 1 AND 2) + THEN '7_New Customer' + + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime BETWEEN 4 AND 5) AND (m_score_lifetime BETWEEN 4 AND 5) + THEN '8_At High Risk' + + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime BETWEEN 1 AND 5) AND (m_score_lifetime BETWEEN 1 AND 5) + THEN '9_At Risk' + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime = 1) AND (m_score_lifetime BETWEEN 1 AND 2) + THEN '9_At Risk' + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime BETWEEN 1 AND 2) AND (m_score_lifetime = 1) + THEN '9_At Risk' + + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 4 AND 5) AND (m_score_lifetime BETWEEN 1 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 1 AND 5) AND (m_score_lifetime BETWEEN 4 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 3 AND 5) AND (m_score_lifetime BETWEEN 3 AND 5) + THEN '11_Cant Lose Them' + + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 1 AND 3) AND (m_score_lifetime BETWEEN 1 AND 3) + THEN '12_Lost' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime = 1) AND (m_score_lifetime BETWEEN 1 AND 3) + THEN '6_Normal Customer' + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 1 AND 3) AND (m_score_lifetime = 1) + THEN '6_Normal Customer' + + ELSE 'ERROR Please Contact DATA Team' + END AS segment_lifetime, + + CASE + WHEN day_after_last_packing_list_one_year IS NULL THEN NULL + WHEN day_after_last_packing_list_one_year <= {{recency_list_one_year[0]}} THEN 5 + WHEN (day_after_last_packing_list_one_year > {{recency_list_one_year[0]}} AND day_after_last_packing_list_one_year <= {{recency_list_one_year[1]}}) THEN 4 + WHEN (day_after_last_packing_list_one_year > {{recency_list_one_year[1]}} AND day_after_last_packing_list_one_year <= {{recency_list_one_year[2]}}) THEN 3 + WHEN (day_after_last_packing_list_one_year > {{recency_list_one_year[2]}} AND day_after_last_packing_list_one_year <= {{recency_list_one_year[3]}}) THEN 2 + WHEN day_after_last_packing_list_one_year > {{recency_list_one_year[3]}} THEN 1 + ELSE -999 + END AS r_score_one_year, + + CASE + WHEN number_of_packing_list_one_year IS NULL THEN NULL + WHEN number_of_packing_list_one_year <= {{frequency_list_one_year[0]}} THEN 1 + WHEN (number_of_packing_list_one_year > {{frequency_list_one_year[0]}} AND number_of_packing_list_one_year <= {{frequency_list_one_year[1]}}) THEN 2 + WHEN (number_of_packing_list_one_year > {{frequency_list_one_year[1]}} AND number_of_packing_list_one_year <= {{frequency_list_one_year[2]}}) THEN 3 + WHEN (number_of_packing_list_one_year > {{frequency_list_one_year[2]}} AND number_of_packing_list_one_year <= {{frequency_list_one_year[3]}}) THEN 4 + WHEN number_of_packing_list_one_year > {{frequency_list_one_year[3]}} THEN 5 + ELSE -999 + END AS f_score_one_year, + + CASE + WHEN total_package_cbm_one_year IS NULL THEN NULL + WHEN total_package_cbm_one_year <= {{monetary_list_one_year[0]}} THEN 1 + WHEN (total_package_cbm_one_year > {{monetary_list_one_year[0]}} AND total_package_cbm_one_year <= {{monetary_list_one_year[1]}}) THEN 2 + WHEN (total_package_cbm_one_year > {{monetary_list_one_year[1]}} AND total_package_cbm_one_year <= {{monetary_list_one_year[2]}}) THEN 3 + WHEN (total_package_cbm_one_year > {{monetary_list_one_year[2]}} AND total_package_cbm_one_year <= {{monetary_list_one_year[3]}}) THEN 4 + WHEN total_package_cbm_one_year > {{monetary_list_one_year[3]}} THEN 5 + ELSE -999 + END AS m_score_one_year, + + CASE + WHEN (r_score_one_year IS NULL) AND (f_score_one_year IS NULL) AND (m_score_one_year IS NULL) + THEN '14_Not Active Between 1 year' + + WHEN (r_score_one_year = 5) AND (f_score_one_year = 5) AND (m_score_one_year = 5) + THEN '1_King' + + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year = 1) AND (m_score_one_year = 1) + THEN '10_At Cheap Risk' + + WHEN (r_score_one_year = 1) AND (f_score_one_year = 1) AND (m_score_one_year = 1) + THEN '13_Cheap Lost' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 1 AND 2) AND (m_score_one_year BETWEEN 4 AND 5) + THEN '4_Whale' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 4 AND 5) AND (m_score_one_year BETWEEN 1 AND 2) + THEN '3_Sardine' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 3 AND 5) AND (m_score_one_year BETWEEN 3 AND 5) + THEN '2_Loyal Customer' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 2 AND 5) AND (m_score_one_year BETWEEN 2 AND 5) + THEN '5_Potention Loyal' + + WHEN (r_score_one_year = 5) AND (f_score_one_year = 1) AND (m_score_one_year BETWEEN 1 AND 2) + THEN '7_New Customer' + + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year BETWEEN 4 AND 5) AND (m_score_one_year BETWEEN 4 AND 5) + THEN '8_At High Risk' + + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year BETWEEN 1 AND 5) AND (m_score_one_year BETWEEN 1 AND 5) + THEN '9_At Risk' + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year = 1) AND (m_score_one_year BETWEEN 1 AND 2) + THEN '9_At Risk' + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year BETWEEN 1 AND 2) AND (m_score_one_year = 1) + THEN '9_At Risk' + + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 4 AND 5) AND (m_score_one_year BETWEEN 1 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 1 AND 5) AND (m_score_one_year BETWEEN 4 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 3 AND 5) AND (m_score_one_year BETWEEN 3 AND 5) + THEN '11_Cant Lose Them' + + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 1 AND 3) AND (m_score_one_year BETWEEN 1 AND 3) + THEN '12_Lost' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year = 1) AND (m_score_one_year BETWEEN 1 AND 3) + THEN '6_Normal Customer' + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 1 AND 3) AND (m_score_one_year = 1) + THEN '6_Normal Customer' + + ELSE 'ERROR Please Contact DATA Team' + END AS segment_one_year, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM company_rfm + +), + + +-- FINAL +final__rep_shipping__rfm AS ( + + SELECT + -- ids + sub_company_id, + sub_company_marking_id, + + -- dimensions + segment_lifetime, + segment_one_year, + + -- measures + day_after_last_packing_list_lifetime, + number_of_packing_list_lifetime, + total_package_cbm_lifetime, + r_score_lifetime, + f_score_lifetime, + m_score_lifetime, + day_after_last_packing_list_one_year, + number_of_packing_list_one_year, + total_package_cbm_one_year, + r_score_one_year, + f_score_one_year, + m_score_one_year, + + -- date/times + last_packing_list_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM calculate_rfm + +) + +SELECT * FROM final__rep_shipping__rfm \ No newline at end of file From 1b60be45220904012cb517bdc18f10e2549fc810 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:43:22 +0000 Subject: [PATCH 08/13] Shipping data model - survival analysis --- .../rep_shipping__survival_analysis.sql | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 models/marts/reporting/rep_shipping__survival_analysis.sql diff --git a/models/marts/reporting/rep_shipping__survival_analysis.sql b/models/marts/reporting/rep_shipping__survival_analysis.sql new file mode 100644 index 0000000..78028f6 --- /dev/null +++ b/models/marts/reporting/rep_shipping__survival_analysis.sql @@ -0,0 +1,183 @@ +-- Survival analysis code reference from https://www.crosstab.io/articles/sql-survival-curves/ + +-- AVAILABLE FILTER VALUE +-- Source:preset_custom_filter Column:day_use_to_churn +{% set day_use_to_churn = 120 %} + + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_shipping__companies') }} +), + +origin_warehouse_shipping_packing_lists AS ( + SELECT * FROM {{ ref('fct_shipping__origin_warehouse_shipping_packing_lists') }} +), + + +-- LOGIC +duration_table AS ( + + SELECT + companies.sub_company_id, + companies.sub_company_marking_id, + companies.sub_company_created_datetime, + COUNT(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id) AS count_order, + + COUNT(IFF( + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_status IN ('APPROVED', 'COMPLETED'), + origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id, + null + )) AS count_completed_order, + + MAX(origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_order_created_datetime) AS last_order_datetime, + COALESCE(last_order_datetime, sub_company_created_datetime) AS last_activity_datetime, + IFF(last_order_datetime IS NULL, 1, 0) AS is_never_order_company, + + -- if is_churn_company = 0, the data will be censored + IFF( + DATEDIFF(day, last_activity_datetime, CURRENT_DATE()) >= {{day_use_to_churn}}, + 1, + 0 + ) AS is_churn_company, + + CASE + WHEN is_churn_company = 1 THEN + DATEDIFF(day, companies.sub_company_created_datetime, last_activity_datetime) + {{day_use_to_churn}} + ELSE + DATEDIFF(day, companies.sub_company_created_datetime, CURRENT_DATE()) + END AS survival_time_days --event duration + + FROM companies + + LEFT JOIN origin_warehouse_shipping_packing_lists + ON (companies.sub_company_id = origin_warehouse_shipping_packing_lists.sub_company_id) + + GROUP BY + companies.sub_company_id, + companies.sub_company_marking_id, + companies.sub_company_created_datetime + + HAVING + -- companies without order will not be relevant to the analysis + is_never_order_company = 0 + +), + +-- the daily_tally cte count the total number of observations at each survival_time_day +-- and the number of company that have churned at that survival_time_day +daily_observation_tally AS ( + + SELECT + survival_time_days, + COUNT(survival_time_days) AS total_number_of_observations, + SUM(is_churn_company) AS events -- considering only churned company + + FROM duration_table + + GROUP BY survival_time_days + + ORDER BY survival_time_days + +), + +-- the cumulative_tally cte counts the number of subjects still at risk of experiencing churn +cumulative_tally AS ( + + SELECT + survival_time_days, + events, + total_number_of_observations, + + ( SELECT COUNT(DISTINCT(sub_company_id)) FROM duration_table ) AS total_number_of_subjects, + + -- cumulative sum of observations at all previous survival_time_days SUBTRACTED by total_number_of_subjects + total_number_of_subjects - COALESCE( + SUM(total_number_of_observations) OVER (ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) + ,0 + ) AS at_risk + + FROM daily_observation_tally + +), + +-- At each survival_time_day, count number of censored subject +-- censored subjects = # subject at risk - churned - # subject at risk in the next duration +compute_censored_subjects AS ( + + SELECT + total_number_of_subjects, + survival_time_days, + at_risk, + total_number_of_observations, + events, + at_risk - events - COALESCE( + LEAD(at_risk, 1) OVER (ORDER BY survival_time_days) + ,0 + ) AS censored + + FROM cumulative_tally + + -- Simply subtracting events from number of observations would incorrectly ignore subjects censored at durations that are dropped from the output table + WHERE events > 0 + +), + +compute_probability AS ( + + SELECT + *, + -- The survival probability represents the probability of customers that will not churn up to a specific tenure + -- Example: survival_day = 96, survival_prob = 95%. For customers with 96 days of tenure, the customer has a 95% chance of not churning. + EXP(SUM( + -- When events / at_risk = 1, we replace value 1 with a value close to 1, to prevent log 0 which causes infinity value + LN(CASE WHEN (1 - events / at_risk) = 0 THEN 0.999999 ELSE (1 - events / at_risk) END) + ) OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) + ) AS survival_probability, + + 100 * (1 - EXP(SUM( + -- When events / at_risk = 1, we replace value 1 with a value close to 1, to prevent log 0 which causes infinity value + LN(CASE WHEN (1 - events / at_risk) = 0 THEN 0.999999 ELSE (1 - events / at_risk) END) + ) OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) + )) AS conversion_percentage, + + SUM(events / at_risk) + OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS cumulative_hazard, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM compute_censored_subjects + +), + + +-- FINAL +final__rep_shipping__survival_analysis AS ( + + SELECT + -- dimension + survival_time_days, + at_risk, + total_number_of_observations, + events, + censored, + + -- measures + survival_probability, + conversion_percentage, + cumulative_hazard, + + -- metadata + _dbt_ran_datetime + + FROM compute_probability + +) + +SELECT * FROM final__rep_shipping__survival_analysis \ No newline at end of file From c453c6189dbd5f44abbf0fdccd9fba3da7941d8b Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:44:17 +0000 Subject: [PATCH 09/13] added is_credit_term and is_migrated_company variable --- .../warehouse/dim_shipping__companies.sql | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/models/marts/warehouse/dim_shipping__companies.sql b/models/marts/warehouse/dim_shipping__companies.sql index f02197a..f5dcd46 100644 --- a/models/marts/warehouse/dim_shipping__companies.sql +++ b/models/marts/warehouse/dim_shipping__companies.sql @@ -14,18 +14,19 @@ company_documents AS ( -- LOGIC latest_company_documents AS ( + SELECT - document_id AS identity_document_id, - parent_company_id AS identity_parent_company_id, - approver_user_id AS identity_approver_user_id, - status AS identity_status, - document_type AS identity_document_type, - identity_number AS identity_identity_number, - approved_datetime AS identity_approved_datetime, - deleted_datetime AS identity_deleted_datetime, - created_datetime AS identity_created_datetime, - updated_datetime AS identity_updated_datetime, - row_number() OVER + document_id AS identity_document_id, + parent_company_id AS identity_parent_company_id, + approver_user_id AS identity_approver_user_id, + status AS identity_status, + document_type AS identity_document_type, + identity_number AS identity_identity_number, + approved_datetime AS identity_approved_datetime, + deleted_datetime AS identity_deleted_datetime, + created_datetime AS identity_created_datetime, + updated_datetime AS identity_updated_datetime, + ROW_NUMBER() OVER (PARTITION BY parent_company_id ORDER BY created_datetime DESC) AS latest_row_number @@ -36,15 +37,19 @@ latest_company_documents AS ( QUALIFY latest_row_number = 1 + ), sub_companies_enrich AS ( + SELECT parent_and_sub_companies.parent_company_id, parent_and_sub_companies.autocount_id, parent_and_sub_companies.sub_company_id, parent_and_sub_companies.sub_company_marking_id, - parent_and_sub_companies.status AS sub_company_status, + parent_and_sub_companies.status AS sub_company_status, + parent_and_sub_companies.is_credit_term_company, + parent_and_sub_companies.is_migrated_company, parent_and_sub_companies.parent_company_name, parent_and_sub_companies.sub_company_name, parent_and_sub_companies.company_type, @@ -64,9 +69,9 @@ sub_companies_enrich AS ( latest_sub_company_billing_addresses.postcode, latest_sub_company_billing_addresses.address_type, latest_sub_company_billing_addresses.is_default_address, - latest_sub_company_billing_addresses.status AS company_address_status, - latest_sub_company_billing_addresses.created_datetime AS company_addresses_created_datetime, - latest_sub_company_billing_addresses.updated_datetime AS company_addresses_updated_datetime, + latest_sub_company_billing_addresses.status AS company_address_status, + latest_sub_company_billing_addresses.created_datetime AS company_addresses_created_datetime, + latest_sub_company_billing_addresses.updated_datetime AS company_addresses_updated_datetime, latest_company_documents.identity_document_id, latest_company_documents.identity_approver_user_id, @@ -87,10 +92,13 @@ sub_companies_enrich AS ( LEFT JOIN latest_company_documents ON (parent_and_sub_companies.parent_company_id = latest_company_documents.identity_parent_company_id) + ), + -- FINAL final__dim_shipping__companies AS ( + SELECT -- ids sub_company_id, @@ -107,6 +115,8 @@ final__dim_shipping__companies AS ( sub_company_name, company_type, business_type, + is_credit_term_company, + is_migrated_company, company_address_status, address_reference, address_line_one, @@ -138,8 +148,8 @@ final__dim_shipping__companies AS ( -- metadata _dbt_ran_datetime - FROM sub_companies_enrich + ) SELECT * FROM final__dim_shipping__companies \ No newline at end of file From 5f1a9b361d301afbd164268150a240f1067595cb Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 09:48:22 +0000 Subject: [PATCH 10/13] added is_credit_term_company variable --- ...e_shipping__sub_company_marking_connections.sql | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql b/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql index 79dbd65..95766c2 100644 --- a/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql +++ b/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql @@ -3,19 +3,23 @@ WITH company_connections AS ( SELECT * FROM {{ source('src_shipping_mysql', 'company_connections') }} ), + -- LOGIC company_connections_rename AS ( SELECT - company_connections.id AS sub_company_marking_connection_id, - company_connections.invitee_id AS sub_company_id, + company_connections.id AS sub_company_marking_connection_id, + company_connections.invitee_id AS sub_company_id, company_connections.invitee_reference AS sub_company_marking_id, company_connections.status, - company_connections.created_at AS created_datetime, - company_connections.updated_at AS updated_datetime, + company_connections.is_credit_term AS is_credit_term_company, + company_connections.created_at AS created_datetime, + company_connections.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM company_connections + ), @@ -30,6 +34,7 @@ final__base_shipping__sub_company_marking_connections AS ( -- dimensions status, + is_credit_term_company, -- measures @@ -41,6 +46,7 @@ final__base_shipping__sub_company_marking_connections AS ( _dbt_ran_datetime FROM company_connections_rename + ) SELECT * FROM final__base_shipping__sub_company_marking_connections \ No newline at end of file From 2912022f846d2cbd96f7648de309d5d92c40b9d5 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 10:29:37 +0000 Subject: [PATCH 11/13] added next order datetime variable --- ..._origin_warehouse_shipping_packing_lists.sql | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql index f597e4f..f5cc8f0 100644 --- a/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql +++ b/models/marts/warehouse/fct_shipping__origin_warehouse_shipping_packing_lists.sql @@ -33,20 +33,20 @@ origin_warehouse_shipping_packing_lists_enrich AS ( SELECT origin_warehouse_shipping_packing_lists.origin_warehouse_shipping_packing_list_id, origin_warehouse_shipping_packing_lists.old_origin_warehouse_shipping_packing_list_id, - origin_warehouse_shipping_packing_lists.has_replaced_by_new_id AS has_replaced_by_new_origin_warehouse_shipping_packing_list_id, + origin_warehouse_shipping_packing_lists.has_replaced_by_new_id AS has_replaced_by_new_origin_warehouse_shipping_packing_list_id, - origin_warehouse_shipping_packing_lists.warehouse_order_number AS warehouse_order_number, + origin_warehouse_shipping_packing_lists.warehouse_order_number AS warehouse_order_number, origin_warehouse_shipping_packing_lists.warehouse_sub_company_id, origin_warehouse_shipping_packing_lists.reference_contract, - origin_warehouse_shipping_packing_lists.status AS origin_warehouse_shipping_order_status, - origin_warehouse_shipping_packing_lists.created_datetime AS origin_warehouse_shipping_order_created_datetime, + origin_warehouse_shipping_packing_lists.status AS origin_warehouse_shipping_order_status, + origin_warehouse_shipping_packing_lists.created_datetime AS origin_warehouse_shipping_order_created_datetime, origin_warehouse_shipping_packing_lists.booking_id, bookings.sub_company_id, bookings.service_type, - bookings.status AS booking_status, - bookings.created_datetime AS booking_created_datetime, + bookings.status AS booking_status, + bookings.created_datetime AS booking_created_datetime, booking_delivery_and_billing_addresses.delivery_address_reference, booking_delivery_and_billing_addresses.delivery_address_line_one, @@ -204,6 +204,10 @@ create_new_column AS ( THEN 1 ELSE 0 END AS is_first_time_sub_company_completed, + + LEAD(origin_warehouse_shipping_order_created_datetime) OVER ( + PARTITION BY sub_company_id ORDER BY (origin_warehouse_shipping_order_created_datetime, CAST(origin_warehouse_shipping_packing_list_id AS INT)) + ) AS next_origin_warehouse_shipping_order_created_datetime, '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime @@ -345,6 +349,7 @@ final__fct_shipping__origin_warehouse_shipping_packing_lists AS ( destination_warehouse_transport_first_estimate_schedule_created_datetime, destination_warehouse_transport_last_estimate_schedule_created_datetime, destination_warehouse_transport_created_datetime, + next_origin_warehouse_shipping_order_created_datetime, -- metadata _dbt_ran_datetime From 20db620d27f9d46f196599576a346ec3df997073 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 10:31:33 +0000 Subject: [PATCH 12/13] added is_migrated_company, company_created_datetime, and next_order_datetime variable --- .../rep_shipping__origin_warehouse_shipping_orders.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql b/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql index f7a83e9..da07ff8 100644 --- a/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql +++ b/models/marts/reporting/rep_shipping__origin_warehouse_shipping_orders.sql @@ -13,9 +13,11 @@ join_tables AS ( SELECT origin_warehouse_shipping_packing_lists.*, companies.sub_company_marking_id, + companies.is_migrated_company, companies.parent_company_id, companies.company_type, - companies.state_name AS sub_company_state_name + companies.sub_company_created_datetime, + companies.state_name AS sub_company_state_name FROM (SELECT * EXCLUDE _dbt_ran_datetime FROM origin_warehouse_shipping_packing_lists) AS origin_warehouse_shipping_packing_lists @@ -47,6 +49,7 @@ create_metrics AS ( FROM join_tables ), + --FINAL final__rep_shipping__origin_warehouse_shipping_orders AS ( SELECT @@ -70,6 +73,7 @@ final__rep_shipping__origin_warehouse_shipping_orders AS ( reference_contract, has_replaced_by_new_origin_warehouse_shipping_packing_list_id, company_type, + is_migrated_company, is_first_time_parent_company, is_first_time_parent_company_completed, is_first_time_sub_company, @@ -156,6 +160,7 @@ final__rep_shipping__origin_warehouse_shipping_orders AS ( destination_warehouse_transport_number_of_eta_datetime_postpone, -- date/times + sub_company_created_datetime, booking_created_datetime, delivery_address_created_datetime, billing_address_created_datetime, @@ -187,6 +192,7 @@ final__rep_shipping__origin_warehouse_shipping_orders AS ( destination_warehouse_transport_first_estimate_schedule_created_datetime, destination_warehouse_transport_last_estimate_schedule_created_datetime, destination_warehouse_transport_created_datetime, + next_origin_warehouse_shipping_order_created_datetime, -- metadata _dbt_ran_datetime From 021cd467dcbb70fb00604fcf0da39aa846f007a1 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Dec 2023 10:32:28 +0000 Subject: [PATCH 13/13] Added logic to identify migrated_companies --- ...stg_shipping__parent_and_sub_companies.sql | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql b/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql index f2e95e1..71f3559 100644 --- a/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql +++ b/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql @@ -26,23 +26,27 @@ status AS ( WHERE category = 'DEFAULT' ), +migrated_sub_company_marking_id AS ( + SELECT * FROM {{ ref('seed_shipping__migrated_company_marking_id') }} +), + -- LOGIC parent_companies_join_sub_companies AS ( SELECT - parent_companies.parent_company_id AS parent_company_id, + parent_companies.parent_company_id AS parent_company_id, parent_companies.autocount_id, - parent_companies.name AS parent_company_name, + parent_companies.name AS parent_company_name, parent_companies.parent_company_type AS company_type, parent_companies.status, - parent_companies.created_datetime AS parent_company_created_datetime, - parent_companies.updated_datetime AS parent_company_updated_datetime, + parent_companies.created_datetime AS parent_company_created_datetime, + parent_companies.updated_datetime AS parent_company_updated_datetime, - sub_companies.sub_company_id AS sub_company_id, - sub_companies.name AS sub_company_name, - sub_companies.sub_company_type AS business_type, - sub_companies.created_datetime AS sub_company_created_datetime, - sub_companies.updated_datetime AS sub_company_updated_datetime + sub_companies.sub_company_id AS sub_company_id, + sub_companies.name AS sub_company_name, + sub_companies.sub_company_type AS business_type, + sub_companies.created_datetime AS sub_company_created_datetime, + sub_companies.updated_datetime AS sub_company_updated_datetime FROM parent_companies @@ -56,39 +60,50 @@ parent_companies_and_sub_companies_map_constants AS ( parent_companies_join_sub_companies.autocount_id, parent_companies_join_sub_companies.parent_company_name, - COALESCE(company_types.name, parent_companies_join_sub_companies.company_type::string) AS company_type, + COALESCE(company_types.name, parent_companies_join_sub_companies.company_type::string) AS company_type, - COALESCE(status.name, parent_companies_join_sub_companies.status::string) AS status, + COALESCE(status.name, parent_companies_join_sub_companies.status::string) AS status, parent_companies_join_sub_companies.parent_company_created_datetime, parent_companies_join_sub_companies.parent_company_updated_datetime, parent_companies_join_sub_companies.sub_company_id, parent_companies_join_sub_companies.sub_company_name, - COALESCE(business_type.name, parent_companies_join_sub_companies.business_type::string) AS business_type, + COALESCE(business_type.name, parent_companies_join_sub_companies.business_type::string) AS business_type, parent_companies_join_sub_companies.sub_company_created_datetime, parent_companies_join_sub_companies.sub_company_updated_datetime, sub_company_marking_connections.sub_company_marking_id, + sub_company_marking_connections.is_credit_term_company, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + CASE + WHEN sub_company_marking_id IN (SELECT * FROM migrated_sub_company_marking_id) + OR + DATE(sub_company_marking_connections.created_datetime) = '2021-08-18' + THEN 1 + ELSE 0 + END AS is_migrated_company, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM parent_companies_join_sub_companies LEFT JOIN sub_company_marking_connections ON (parent_companies_join_sub_companies.sub_company_id = sub_company_marking_connections.sub_company_id) - LEFT JOIN parent_company_types AS company_types + LEFT JOIN parent_company_types AS company_types ON (parent_companies_join_sub_companies.company_type = company_types.id) - LEFT JOIN sub_company_types AS business_type + LEFT JOIN sub_company_types AS business_type ON (parent_companies_join_sub_companies.business_type = business_type.id) LEFT JOIN status ON (parent_companies_join_sub_companies.status = status.id) + ), + -- FINAL final__stg_shipping__parent_and_sub_companies AS ( @@ -101,6 +116,8 @@ final__stg_shipping__parent_and_sub_companies AS ( -- dimensions status, + is_credit_term_company, + is_migrated_company, parent_company_name, sub_company_name, company_type, @@ -118,6 +135,7 @@ final__stg_shipping__parent_and_sub_companies AS ( _dbt_ran_datetime FROM parent_companies_and_sub_companies_map_constants + ) -SELECT * FROM final__stg_shipping__parent_and_sub_companies +SELECT * FROM final__stg_shipping__parent_and_sub_companies \ No newline at end of file