From 362cec7fa049e11cad74cf22e41afaddf4e1475a Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Sun, 24 Sep 2023 11:47:14 +0000 Subject: [PATCH 01/11] Clean wrong data --- .../exchange/stg_exchange__companies.sql | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/models/staging/exchange/stg_exchange__companies.sql b/models/staging/exchange/stg_exchange__companies.sql index 3cfe5b0..3b7d27e 100644 --- a/models/staging/exchange/stg_exchange__companies.sql +++ b/models/staging/exchange/stg_exchange__companies.sql @@ -34,8 +34,7 @@ companies_map_constant AS ( companies.deleted_datetime, companies.created_datetime, - companies.updated_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + companies.updated_datetime FROM companies @@ -49,6 +48,40 @@ companies_map_constant AS ( ON (companies.status = status.id) ), +/* +Data cleaning for the 4 company_id (1405, 2030, 3095, 4275) which has a company registration date later than the first order_created_datetime. +Median number of days to placing first order (4 days) is used to revert the company created datetime. +*/ + +data_clean AS ( + SELECT + company_id, + company_marking_id, + autocount_id, + name, + company_type, + business_type, + status, + deleted_datetime, + updated_datetime, + + CASE + -- 2030 has an order placed 5 months before the registration date, so using the first order_datetime as base + WHEN company_id = 2030 THEN DATEADD(day, -4, TO_TIMESTAMP_TZ('2021-06-28T14:04:30+08:00')) + + WHEN company_id = 1405 THEN DATEADD(day, -4, created_datetime) + WHEN company_id = 3095 THEN DATEADD(day, -4, created_datetime) + WHEN company_id = 4275 THEN DATEADD(day, -4, created_datetime) + ELSE created_datetime + END AS created_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies_map_constant + +), + + -- FINAL final__stg_companies__exchange AS ( @@ -74,7 +107,7 @@ final__stg_companies__exchange AS ( -- metadata _dbt_ran_datetime - FROM companies_map_constant + FROM data_clean ) From 6e671ca88007dea0129e81de37fbf3fd3c6bb95a Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Sep 2023 07:36:22 +0000 Subject: [PATCH 02/11] Transaction Invoice stg file --- .../stg_exchange__transaction_invoices.sql | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 models/staging/exchange/stg_exchange__transaction_invoices.sql diff --git a/models/staging/exchange/stg_exchange__transaction_invoices.sql b/models/staging/exchange/stg_exchange__transaction_invoices.sql new file mode 100644 index 0000000..092889c --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_invoices.sql @@ -0,0 +1,145 @@ +/* DOCS +Invoice is issued by CIEF to customers for customer book keeping. +An invoice is a document that consolidates the total amount for multiple orders within a single booking. +Currency shown in invoice is the converted currency. + +For booking_id with multiple invoices, take latest updated_datetime as true. +The duplicates are a bug as confirmed by omair. +*/ + +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +bookings AS ( + SELECT * FROM {{ ref('base_exchange__bookings') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_bookings_status AS ( + + SELECT + bookings.booking_marking_id, + bookings.fix_value, + bookings.fix_currency_id, + transactions.transaction_id AS transaction_invoice_id, + transactions.owner_id AS booking_id, + transactions.issuer_user_id AS issuer_company_id, + transactions.receiver_user_id AS receiver_company_id, + transactions.base_currency_id, + transactions.quote_currency_id, + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(status.name, transactions.status::string) AS status, + transactions.bill_number AS invoice_number, + transactions.base_value, + transactions.quote_value, + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + + transactions.expired_datetime, + transactions.deleted_datetime, + transactions.created_datetime, + transactions.updated_datetime + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + LEFT JOIN bookings + ON (transactions.owner_id = bookings.booking_id) + + WHERE + transactions.owner_type = 'App\\Models\\Booking' + AND + transactions.transaction_type = '2' --INVOICE + +), + +remove_deleted_records AS ( + SELECT + * + FROM transactions_join_transaction_types_payment_methods_bookings_status + WHERE deleted_datetime IS NULL + +), + +remove_duplicated_records AS ( + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC) AS row_index, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM remove_deleted_records + + QUALIFY + row_index = 1 +), + + +-- FINAL +final__stg_exchange__invoices AS ( + + SELECT + -- ids + transaction_invoice_id, + booking_id, + booking_marking_id, + issuer_company_id, + receiver_company_id, + base_currency_id, + quote_currency_id, + fix_currency_id, + + -- dimensions + invoice_number, + transaction_type, + payment_method, + status, + + -- measures + fix_value, + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + deleted_datetime, + created_datetime, + expired_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM remove_duplicated_records + +) + +SELECT * FROM final__stg_exchange__invoices \ No newline at end of file From d1c0ff2319e7ca5d7dcbbaa07f1ea272f2a107ce Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Sep 2023 07:36:34 +0000 Subject: [PATCH 03/11] PO stg file --- ..._exchange__transaction_purchase_orders.sql | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 models/staging/exchange/stg_exchange__transaction_purchase_orders.sql diff --git a/models/staging/exchange/stg_exchange__transaction_purchase_orders.sql b/models/staging/exchange/stg_exchange__transaction_purchase_orders.sql new file mode 100644 index 0000000..26ff2af --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_purchase_orders.sql @@ -0,0 +1,149 @@ +/* DOCS +"IS_AUTO_GENERATED_PURCHASE_ORDER" +- 1 indicates PO auto generated by system, having prefix 'XPO' +- 0 indicates PO submitted by customer, having prefix 'PO' + +Purchase Order document does not perform currency conversion, and it is a document to be submitted by the customer. +Therefore, base currency and quote currency is the same. +The PO is based on the fix currency. + +The 'fix currency' feature enables customers to choose their primary currency when placing an order, +ensuring that it remains fixed while allowing the converted currency to fluctuate. + +Currency is converted only in invoice. + +Note to downstream user: +This table contains duplicate of booking_id, due to deleted entries. +Filter out deleted entries and select only status = 'APPROVED' for use. +*/ + +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +bookings AS ( + SELECT * FROM {{ ref('base_exchange__bookings') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_bookings_status AS ( + + SELECT + bookings.booking_marking_id, + bookings.fix_value, + bookings.fix_currency_id AS purchase_order_currency_id, + + transactions.transaction_id AS transaction_purchase_order_id, + transactions.owner_id AS booking_id, + transactions.issuer_user_id AS issuer_company_id, + transactions.receiver_user_id AS receiver_company_id, + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(status.name, transactions.status::string) AS status, + transactions.bill_number AS purchase_order_number, + transactions.base_value AS purchase_order_amount, + + transactions.expired_datetime, + transactions.deleted_datetime, + transactions.created_datetime, + transactions.updated_datetime, + + CASE + WHEN purchase_order_number LIKE 'XP%' THEN 1 ELSE 0 + END AS is_auto_generated_purchase_order + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + LEFT JOIN bookings + ON (transactions.owner_id = bookings.booking_id) + + WHERE + transactions.owner_type = 'App\\Models\\Booking' + AND + transactions.transaction_type = '7' --PURCHASE_ORDER + +), + +remove_deleted_records AS ( + SELECT + * + FROM transactions_join_transaction_types_payment_methods_bookings_status + WHERE deleted_datetime IS NULL +), + + +remove_duplicated_records AS ( + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC) AS row_index, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM remove_deleted_records + + QUALIFY + row_index = 1 +), + + +-- FINAL +final__stg_exchange__purchase_orders AS ( + + SELECT + -- ids + transaction_purchase_order_id, + booking_id, + booking_marking_id, + issuer_company_id, + receiver_company_id, + purchase_order_currency_id + + -- dimensions + is_auto_generated_purchase_order, + purchase_order_number, + transaction_type, + payment_method, + status, + + -- measures + fix_value, + purchase_order_amount, + + -- date/times + deleted_datetime, + created_datetime, + expired_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM remove_duplicated_records + +) + +SELECT * FROM final__stg_exchange__purchase_orders \ No newline at end of file From c1efd26ef5dcc90c2993b21aac2a78d072371318 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 29 Sep 2023 07:37:04 +0000 Subject: [PATCH 04/11] Change 5 working day to 7 working day --- models/marts/warehouse/dim__dates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/marts/warehouse/dim__dates.py b/models/marts/warehouse/dim__dates.py index 4e17fdc..d9d0a55 100644 --- a/models/marts/warehouse/dim__dates.py +++ b/models/marts/warehouse/dim__dates.py @@ -62,7 +62,7 @@ def model(dbt, session): df_work["ADD_1_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 0), axis=1) df_work["ADD_2_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 0), axis=1) df_work["ADD_3_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 0), axis=1) - df_work["ADD_5_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 5, df_holiday_list, 0), axis=1) + df_work["ADD_7_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 0), axis=1) df_work["ADD_30_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 0), axis=1) df_work["ADD_90_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 0), axis=1) df_work["ADD_365_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 0), axis=1) @@ -72,7 +72,7 @@ def model(dbt, session): df_work["ADD_1_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 1), axis=1) df_work["ADD_2_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 1), axis=1) df_work["ADD_3_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 1), axis=1) - df_work["ADD_5_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 5, df_holiday_list, 1), axis=1) + df_work["ADD_7_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 1), axis=1) df_work["ADD_30_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 1), axis=1) df_work["ADD_90_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 1), axis=1) df_work["ADD_365_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 1), axis=1) From 7c377b793a4230188af8784121f9c622744eba81 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 3 Oct 2023 06:46:23 +0000 Subject: [PATCH 05/11] spacing readjust --- .../exchange/stg_exchange__transaction_cost_logs.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/staging/exchange/stg_exchange__transaction_cost_logs.sql b/models/staging/exchange/stg_exchange__transaction_cost_logs.sql index 332bf6b..952cc9f 100644 --- a/models/staging/exchange/stg_exchange__transaction_cost_logs.sql +++ b/models/staging/exchange/stg_exchange__transaction_cost_logs.sql @@ -20,23 +20,22 @@ status AS ( WHERE category = 'DEFAULT' ), + -- LOGIC transactions_generate_transaction_log_id AS ( SELECT - MD5_NUMBER_LOWER64(CONCAT(transactions.transaction_id, transactions.created_datetime)) AS transaction_log_id, + MD5_NUMBER_LOWER64(CONCAT(transaction_id, created_datetime)) AS transaction_log_id, * FROM transactions ), - transaction_arch_logs_union_transactions AS ( SELECT * FROM transactions_generate_transaction_log_id UNION SELECT * FROM transaction_arch_logs ), - union_transaction_rename_column_and_column_value AS ( SELECT @@ -96,6 +95,7 @@ union_transaction_rename_column_and_column_value AS ( transaction_arch_logs_union_transactions.transaction_type = '3' --BILL ), + -- FINAL final__stg_exchange__transaction_log_costs AS ( From ce381e8c743c59f0e8207b738ecd3c6b076aa830 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 3 Oct 2023 06:50:43 +0000 Subject: [PATCH 06/11] split deduplicate function in intermediate --- .../stg_exchange__transaction_costs.sql | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/models/staging/exchange/stg_exchange__transaction_costs.sql b/models/staging/exchange/stg_exchange__transaction_costs.sql index cf26246..2b4ff03 100644 --- a/models/staging/exchange/stg_exchange__transaction_costs.sql +++ b/models/staging/exchange/stg_exchange__transaction_costs.sql @@ -73,27 +73,6 @@ transactions_join_transaction_types_payment_methods_currencies_status AS ( ), -remove_deleted_row AS ( - SELECT - * - - FROM transactions_join_transaction_types_payment_methods_currencies_status - - WHERE deleted_datetime IS NULL -), - --- It will seldom offur system error, so we assume that the latest cost is correct. But it is not a correct assumption -remove_system_error_duplicate_row AS ( - SELECT - *, - ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index - - FROM remove_deleted_row - - QUALIFY - row_number_index = 1 -), - -- FINAL final__stg_exchange__transaction_costs AS ( @@ -130,7 +109,7 @@ final__stg_exchange__transaction_costs AS ( -- metadata _dbt_ran_datetime - FROM remove_system_error_duplicate_row + FROM transactions_join_transaction_types_payment_methods_currencies_status ) -SELECT * FROM remove_system_error_duplicate_row \ No newline at end of file +SELECT * FROM final__stg_exchange__transaction_costs \ No newline at end of file From 48870313cad52b6458ca55f944534075d6873106 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 10 Oct 2023 05:22:41 +0000 Subject: [PATCH 07/11] Added Model --- ..._transaction_cost_with_latest_cost_ids.sql | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql diff --git a/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql b/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql new file mode 100644 index 0000000..02a701d --- /dev/null +++ b/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql @@ -0,0 +1,158 @@ +-- VARIABLES +{% set order_status = dbt_utils.get_column_values( + table=ref('stg_exchange__transaction_order_logs'), + column='status') +%} + + +-- IMPORTS +WITH transaction_cost_logs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} +), + +transaction_costs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} +), + + +-- LOGIC +transaction_cost_status_datetime AS ( + + SELECT + transaction_cost_id, + company_id, + MIN(created_datetime) AS cost_created_datetime, + + {%- for status in order_status %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM transaction_cost_logs + + GROUP BY + transaction_cost_id, + company_id + +), + +join_cost_and_logs AS ( + + SELECT + transaction_costs.transaction_cost_id, + transaction_costs.transaction_order_id, + transaction_costs.supplier_company_id, + transaction_costs.bank_id, + transaction_costs.base_currency_id, + transaction_costs.quote_currency_id, + transaction_costs.transaction_type, + transaction_costs.payment_method, + transaction_costs.status, + transaction_costs.payment_reference, + transaction_costs.bill_number, + transaction_costs.base_value, + transaction_costs.quote_value, + transaction_costs.base_to_quote_currency_exchange_rate, + transaction_costs.base_tax, + transaction_costs.base_service_charge, + transaction_costs.expired_datetime, + transaction_costs.deleted_datetime, + transaction_costs.created_datetime, + transaction_costs.updated_datetime, + + transaction_cost_status_datetime.company_id, + transaction_cost_status_datetime.cost_created_datetime, + transaction_cost_status_datetime.cost_completed_datetime, + transaction_cost_status_datetime.cost_pending_submission_datetime, + transaction_cost_status_datetime.cost_approved_datetime, + transaction_cost_status_datetime.cost_pending_verification_datetime, + transaction_cost_status_datetime.cost_rejected_datetime, + transaction_cost_status_datetime.cost_suspended_datetime, + transaction_cost_status_datetime.cost_expired_datetime + + FROM transaction_costs + + LEFT JOIN transaction_cost_status_datetime + ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id) + +), + +remove_deleted_expired_costs AS ( + + SELECT + * + + FROM join_cost_and_logs + + WHERE + deleted_datetime IS NULL + AND + expired_datetime IS NULL + +), + +-- Omair mentioned a bug in system causing duplicates. Take cost with latest updated_datetime as true. +remove_system_error_duplicate_row AS ( + + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM remove_deleted_expired_costs + + QUALIFY + row_number_index = 1 + +), + + +-- FINAL +final__int_exchange__transaction_cost_get_latest_cost_ids AS ( + + SELECT + -- ids + transaction_cost_id, + transaction_order_id, + supplier_company_id, + company_id, + bank_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + cost_created_datetime, + cost_completed_datetime, + cost_pending_submission_datetime, + cost_approved_datetime, + cost_pending_verification_datetime, + cost_rejected_datetime, + cost_suspended_datetime, + cost_expired_datetime, + + -- metadata + _dbt_ran_datetime + + FROM remove_system_error_duplicate_row + +) + +SELECT * FROM final__int_exchange__transaction_cost_get_latest_cost_ids \ No newline at end of file From c27858b932ac78da47d952aee11e81722f84d97a Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 10 Oct 2023 05:22:45 +0000 Subject: [PATCH 08/11] Added Model --- .../rep_exchange__cohort_analysis.sql | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 models/marts/reporting/rep_exchange__cohort_analysis.sql diff --git a/models/marts/reporting/rep_exchange__cohort_analysis.sql b/models/marts/reporting/rep_exchange__cohort_analysis.sql new file mode 100644 index 0000000..93227cb --- /dev/null +++ b/models/marts/reporting/rep_exchange__cohort_analysis.sql @@ -0,0 +1,187 @@ +-- SOURCE: https://www.holistics.io/blog/calculate-cohort-retention-analysis-with-sql/ + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + + +-- LOGIC +company_register_month_year AS ( + + SELECT + companies.company_id, + companies.company_created_datetime, + IFF(MAX(transaction_orders.order_created_datetime) IS NULL, 1, 0) AS is_never_order_company, + DATE(DATE_TRUNC('MONTH', companies.company_created_datetime)) AS register_month_year, + DATE(DATE_TRUNC('MONTH', MIN(transaction_orders.order_created_datetime))) AS first_order_month_year + + FROM companies + + LEFT JOIN transaction_orders + ON (companies.company_id = transaction_orders.company_id) + + GROUP BY + companies.company_id, + companies.company_created_datetime, + register_month_year + +), + +-- Identify orders placed from each company, from the time of company registration +-- In cohort analysis, we do not consider number of orders by each user, just whether has_placed_orders or not_placed_orders in each month +company_order_count_from_registration_date AS ( + + SELECT + transaction_orders.company_id, + DATEDIFF(MONTH, company_register_month_year.company_created_datetime, transaction_orders.order_created_datetime) AS order_month + + FROM transaction_orders + + LEFT JOIN company_register_month_year + ON (transaction_orders.company_id = company_register_month_year.company_id) + + GROUP BY + transaction_orders.company_id, + order_month + +), + +company_first_order_count_from_registration_date AS ( + + SELECT + transaction_orders.company_id, + DATE(DATE_TRUNC('MONTH', company_created_datetime)) AS company_created_datetime, + MIN(DATEDIFF(MONTH, company_register_month_year.company_created_datetime, transaction_orders.order_created_datetime)) AS first_order_month + + FROM transaction_orders + + LEFT JOIN company_register_month_year + ON (transaction_orders.company_id = company_register_month_year.company_id) + + GROUP BY + transaction_orders.company_id, + company_created_datetime + +), + +cohort_size_by_month_year AS ( + + SELECT + register_month_year, + SUM(is_never_order_company) AS count_never_order_companies, + COUNT(register_month_year) AS count_total_companies + + FROM company_register_month_year + + GROUP BY register_month_year + + ORDER BY register_month_year + +), + +order_retention AS ( + + SELECT + company_register_month_year.register_month_year, + company_order_count_from_registration_date.order_month, + COUNT(company_register_month_year.register_month_year) AS count_retained_companies + + FROM company_order_count_from_registration_date + + LEFT JOIN company_register_month_year + ON (company_order_count_from_registration_date.company_id = company_register_month_year.company_id) + + GROUP BY + company_register_month_year.register_month_year, + company_order_count_from_registration_date.order_month +), + +first_time_order_retention AS ( + SELECT + company_created_datetime, + first_order_month, + COUNT(first_order_month) AS count_first_order_companies + + FROM company_first_order_count_from_registration_date + + GROUP BY + company_created_datetime, + first_order_month + +), + +cohort_analysis_table AS ( + + SELECT + order_retention.register_month_year, + order_retention.order_month, + order_retention.count_retained_companies AS retained_companies, + + cohort_size_by_month_year.count_total_companies AS total_registered_company, + cohort_size_by_month_year.count_never_order_companies, + + first_time_order_retention.count_first_order_companies AS count_first_order_companies, + + DIV0( + retained_companies, + total_registered_company + ) AS retention_rate_overall, + + DIV0( + retained_companies, + ( total_registered_company - cohort_size_by_month_year.count_never_order_companies ) + ) AS retention_rate_exclude_never_order_company, + + DIV0( + count_first_order_companies, + ( total_registered_company - count_never_order_companies ) + ) AS first_order_percentage, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM order_retention + + LEFT JOIN cohort_size_by_month_year + ON (order_retention.register_month_year = cohort_size_by_month_year.register_month_year) + + LEFT JOIN first_time_order_retention + ON (order_retention.register_month_year = first_time_order_retention.company_created_datetime) + AND (order_retention.order_month = first_time_order_retention.first_order_month) + + ORDER BY + order_retention.register_month_year, + order_retention.order_month + +), + + +-- FINAL +final_rep_exchange__cohort_analysis AS ( + + SELECT + -- dimensions + register_month_year, + total_registered_company, + order_month, + + -- measures + count_never_order_companies, + count_first_order_companies, + retained_companies, + retention_rate_overall, + retention_rate_exclude_never_order_company, + first_order_percentage, + + -- metadata + _dbt_ran_datetime + + FROM cohort_analysis_table + +) + +SELECT * FROM final_rep_exchange__cohort_analysis \ No newline at end of file From 5d9f357837caa4e1846f3f47676b90f11022e234 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 10 Oct 2023 05:23:03 +0000 Subject: [PATCH 09/11] Add Business Term status --- .../reporting/rep_exchange__daily_orders.sql | 118 ++++++++++++------ 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql index 93f63b9..8452a9c 100644 --- a/models/marts/reporting/rep_exchange__daily_orders.sql +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -1,3 +1,23 @@ +-- VARIABLES +{% set var_service_type = [ + '1688 PAYMENT', + '1 DAY TRANSFER', + '3 DAYS TRANSFER', + 'Enterprise to Enterprise 公打公'] %} + +{% set var_working_days_including_base_date = [ + 'add_1_working_day_included_base_date', + 'add_1_working_day_included_base_date', + 'add_3_working_day_included_base_date', + 'add_7_working_day_included_base_date'] %} + +{% set var_working_days_excluding_base_date = [ + 'add_1_working_day_excluded_base_date', + 'add_1_working_day_excluded_base_date', + 'add_3_working_day_excluded_base_date', + 'add_7_working_day_excluded_base_date'] %} + + -- IMPORT WITH orders AS ( SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} @@ -25,6 +45,8 @@ fct_and_dim_joins AS ( orders.booking_id, orders.booking_marking_id, orders.company_id, + orders.transaction_status, + orders.document_status, companies.name AS company_name, companies.company_marking_id AS company_marking_id, @@ -40,6 +62,7 @@ fct_and_dim_joins AS ( companies.has_wallet AS company_has_wallet, companies.latitude AS company_latitude, companies.longitude AS company_longitude, + companies.company_created_datetime AS company_created_datetime, orders.user_id, orders.bank_id, @@ -91,63 +114,56 @@ fct_and_dim_joins AS ( -- Estimated delivery datetime based on CIEF internal SLA (cutoff time 4pm) CASE - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' 23:59:59 +08:00') - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' 23:59:59 +08:00') + {% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %} + + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' 23:59:59 +08:00') + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' 23:59:59 +08:00') + + {% endfor %} - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_included_base_date || ' 23:59:59 +08:00') - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_excluded_base_date || ' 23:59:59 +08:00') + ELSE NULL - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' 23:59:59 +08:00') - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' 23:59:59 +08:00') - - ELSE null END AS estimated_order_delivery_datetime_website_sla, --- Estimated delivery datetime based on customer expectation CASE - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1 DAY TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + {% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %} + + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + + {% endfor %} - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_included_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '3 DAYS TRANSFER' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.add_3_working_day_excluded_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + ELSE NULL - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_included_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '1688 PAYMENT' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.add_1_working_day_excluded_base_date || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - - ELSE null END AS estimated_order_delivery_datetime_customer_expectation, -- On time delivery boolean (CIEF internal SLA) CASE - WHEN order_completed_datetime IS NULL THEN NULL - WHEN estimated_order_delivery_datetime_website_sla > order_completed_datetime THEN 1 + -- On-time boolean is null when order is not complete + WHEN account_verified_payment_datetime IS NULL THEN NULL + WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1 ELSE 0 + END AS is_on_time_delivery_website_sla, -- On time delivery boolean (Customer Expectation) CASE - WHEN order_completed_datetime IS NULL THEN NULL - WHEN estimated_order_delivery_datetime_customer_expectation > order_completed_datetime THEN 1 + -- On-time boolean is null when order is not complete + WHEN account_verified_payment_datetime IS NULL THEN NULL + WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1 ELSE 0 + END AS is_on_time_delivery_customer_expectation, orders.order_approved_datetime, orders.order_completed_datetime, orders.order_rejected_datetime, orders.order_suspended_datetime, - orders.next_order_created_datetime, orders.order_expected_expired_datetime, orders.order_expired_datetime, @@ -169,6 +185,8 @@ fct_and_dim_joins AS ( orders.cost_tax_rm, orders.cost_service_charge_rm, orders.total_cost_value_rm, + orders.purchase_order_number, + orders.invoice_number, orders.cost_created_datetime, orders.cost_pending_submission_datetime, orders.cost_pending_verification_datetime, @@ -176,6 +194,19 @@ fct_and_dim_joins AS ( orders.cost_completed_datetime, orders.cost_rejected_datetime, + orders.customer_made_booking_datetime, + orders.customer_placed_order_datetime, + orders.customer_made_payment_datetime, + orders.account_verified_payment_datetime, + orders.first_white_form_generated_datetime, + orders.last_white_form_generated_datetime, + orders.operation_uploaded_bank_slip_datetime, + orders.first_purchase_order_uploaded_datetime, + orders.last_purchase_order_uploaded_datetime, + orders.operation_generated_invoice_datetime, + orders.account_rejected_payment_datetime, + orders.next_order_created_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM orders @@ -218,6 +249,8 @@ final__rep_exchange__daily_orders AS ( -- dimensions company_name, supplier_company_name, + transaction_status, + document_status, fix_currency_name, quote_currency_name, base_currency_name, @@ -253,6 +286,8 @@ final__rep_exchange__daily_orders AS ( is_first_time_booking_user_completed, is_on_time_delivery_website_sla, is_on_time_delivery_customer_expectation, + purchase_order_number, + invoice_number, -- measures order_base_to_quote_currency_exchange_rate, @@ -279,6 +314,7 @@ final__rep_exchange__daily_orders AS ( estimate_booking_value_rm, -- date/times + company_created_datetime, order_created_datetime, order_created_week, order_expected_expired_datetime, @@ -289,7 +325,6 @@ final__rep_exchange__daily_orders AS ( order_rejected_datetime, order_suspended_datetime, order_expired_datetime, - next_order_created_datetime, cost_created_datetime, cost_pending_submission_datetime, cost_pending_verification_datetime, @@ -302,6 +337,19 @@ final__rep_exchange__daily_orders AS ( booking_suspended_datetime, estimated_order_delivery_datetime_website_sla, estimated_order_delivery_datetime_customer_expectation, + + customer_made_booking_datetime, + customer_placed_order_datetime, + customer_made_payment_datetime, + account_verified_payment_datetime, + first_white_form_generated_datetime, + last_white_form_generated_datetime, + operation_uploaded_bank_slip_datetime, + first_purchase_order_uploaded_datetime, + last_purchase_order_uploaded_datetime, + operation_generated_invoice_datetime, + account_rejected_payment_datetime, + next_order_created_datetime, -- metadata _dbt_ran_datetime @@ -310,4 +358,4 @@ final__rep_exchange__daily_orders AS ( ) -SELECT * FROM final__rep_exchange__daily_orders +SELECT * FROM final__rep_exchange__daily_orders \ No newline at end of file From 9206cfa02e33d9d4531c7e214935234bae03274b Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 10 Oct 2023 05:23:11 +0000 Subject: [PATCH 10/11] Added Model --- .../rep_exchange__survival_analysis.sql | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 models/marts/reporting/rep_exchange__survival_analysis.sql diff --git a/models/marts/reporting/rep_exchange__survival_analysis.sql b/models/marts/reporting/rep_exchange__survival_analysis.sql new file mode 100644 index 0000000..046dd4d --- /dev/null +++ b/models/marts/reporting/rep_exchange__survival_analysis.sql @@ -0,0 +1,160 @@ +-- Survival analysis code reference from https://www.crosstab.io/articles/sql-survival-curves/ + +-- AVAILABLE FILTER VALUE +-- Source:preset_custom_filter Column:day_use_to_churn +{% set day_use_to_churn = 61 %} + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + + +-- LOGIC +-- duration_table cte computes the survival time and customer churned indicator +duration_table AS ( + + SELECT + companies.company_id, --subject + companies.company_created_datetime, -- event start datetime + COUNT(transaction_orders.order_id) AS count_order, + COUNT(IFF(transaction_orders.order_status = 'COMPLETED', transaction_orders.order_id, null)) AS count_completed_order, + MAX(transaction_orders.order_created_datetime) AS last_order_datetime, + COALESCE(last_order_datetime, company_created_datetime) AS last_activity_datetime, + IFF(last_order_datetime IS NULL, 1, 0) AS is_never_order_company, + + -- if is_churn_company = 0, the data will be censored + IFF(DATEDIFF(day, last_activity_datetime, CURRENT_DATE()) >= {{day_use_to_churn}}, 1, 0 ) AS is_churn_company, + + CASE + WHEN is_churn_company = 1 THEN + DATEDIFF(day, companies.company_created_datetime, last_activity_datetime ) + {{day_use_to_churn}} + ELSE + DATEDIFF(day, companies.company_created_datetime, CURRENT_DATE()) + END AS survival_time_days -- event duration + + FROM companies + + LEFT JOIN transaction_orders + ON (companies.company_id = transaction_orders.company_id) + + GROUP BY + companies.company_id, + companies.company_created_datetime + + HAVING + -- companies without order will not be relevant to the analysis + is_never_order_company = 0 + +), + +-- the daily_tally cte count the total number of observations at each survival_time_day +-- and the number of company that have churned at that survival_time_day +daily_observation_tally AS ( + + SELECT + survival_time_days, + COUNT(survival_time_days) AS total_number_of_observations, + SUM(is_churn_company) AS events -- considering only churned company + + FROM duration_table + + GROUP BY survival_time_days + + ORDER BY survival_time_days + +), + +-- the cumulative_tally cte counts the number of subjects still at risk of experiencing churn +cumulative_tally AS ( + + SELECT + survival_time_days, + events, + total_number_of_observations, + + ( SELECT COUNT( DISTINCT(company_id) ) FROM duration_table ) AS total_number_of_subjects, + + -- cumulative sum of observations at all previous survival_time_days SUBTRACTED by total_number_of_subjects + total_number_of_subjects - COALESCE(SUM(total_number_of_observations) OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0 + ) AS at_risk + + FROM daily_observation_tally + +), + +-- At each survival_time_day, count number of censored subject +-- censored subjects = # subject at risk - churned - # subject at risk in the next duration +compute_censored_subjects AS ( + + SELECT + total_number_of_subjects, + survival_time_days, + at_risk, + total_number_of_observations, + events, + at_risk - events - COALESCE(LEAD(at_risk, 1) OVER (ORDER BY survival_time_days), 0) AS censored + + FROM cumulative_tally + + -- Simply subtracting events from number of observations would incorrectly ignore subjects censored at durations that are dropped from the output table + WHERE events > 0 + +), + + +compute_probability AS ( + + SELECT + *, + + -- The survival probability represents the probability of customers that will not churn up to a specific tenure + -- Example: survival_day = 61, survival_prob = 95%. For customers with 61 days of tenure, the customer has a 95% chance of not churning. + EXP(SUM(LN(1 - events / at_risk)) OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )) AS survival_probability, + + 100 * (1 - EXP(SUM(LN(1 - events / at_risk)) OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ))) AS conversion_percentage, + + SUM(events / at_risk) OVER ( + ORDER BY survival_time_days ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS cumulative_hazard, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM compute_censored_subjects + +), + + +-- FINAL +final__rep_exchange__survival_analysis AS ( + + SELECT + -- dimension + survival_time_days, + at_risk, + total_number_of_observations, + events, + censored, + + -- measures + survival_probability, + conversion_percentage, + cumulative_hazard, + + -- metadata + _dbt_ran_datetime + + FROM compute_probability + +) + +SELECT * FROM final__rep_exchange__survival_analysis \ No newline at end of file From 6fa24f0a962b2ab0792efb329c305bdccd29cfae Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 10 Oct 2023 05:23:46 +0000 Subject: [PATCH 11/11] Because cost move to INT, delete some code --- .../fct_exchange__transaction_orders.sql | 243 +++++++++++++----- 1 file changed, 182 insertions(+), 61 deletions(-) diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql index 9fa1162..2e59b19 100644 --- a/models/marts/warehouse/fct_exchange__transaction_orders.sql +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -9,6 +9,7 @@ column='status') %} + -- IMPORTS WITH transaction_orders AS ( SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} @@ -19,11 +20,7 @@ transaction_order_logs AS ( ), transaction_costs AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} -), - -transaction_cost_logs AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} + SELECT * FROM {{ ref('int_exchange__transaction_cost_with_latest_cost_ids') }} ), bookings AS ( @@ -34,13 +31,24 @@ booking_logs AS ( SELECT * FROM {{ ref('stg_exchange__booking_logs') }} ), +purchase_orders AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_purchase_orders') }} +), + +invoices AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_invoices') }} +), + + -- LOGIC order_logs_join_bookings AS ( + SELECT transaction_order_logs.*, bookings.service_type, ROW_NUMBER() OVER (PARTITION BY transaction_order_id - ORDER BY transaction_order_logs.updated_datetime DESC, + ORDER BY + transaction_order_logs.updated_datetime DESC, transaction_order_logs.transaction_order_id DESC, transaction_order_logs.transaction_order_log_id DESC) AS rank_index @@ -177,22 +185,7 @@ transaction_order_datetime_imputation AS ( END AS new_order_completed_datetime FROM transaction_order_order_status_datetime -), -transaction_cost_status_datetime AS ( - - SELECT - transaction_cost_id, - MIN(created_datetime) AS cost_created_datetime, - - {%- for status in order_status %} - MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} - {% endfor %} - - FROM transaction_cost_logs - - GROUP BY - transaction_cost_id ), booking_status_datetime AS ( @@ -209,6 +202,7 @@ booking_status_datetime AS ( GROUP BY booking_id + ), transaction_orders_lists AS ( @@ -284,63 +278,65 @@ transaction_orders_lists AS ( LEFT JOIN transaction_order_datetime_imputation ON (transaction_orders.transaction_order_id = transaction_order_datetime_imputation.transaction_order_id) + ), transaction_costs_lists AS ( SELECT - transaction_costs.transaction_cost_id, - transaction_costs.transaction_order_id, - transaction_costs.supplier_company_id, - transaction_costs.bank_id, - transaction_costs.base_currency_id, - transaction_costs.quote_currency_id, - transaction_costs.transaction_type, - transaction_costs.payment_method, - transaction_costs.status, - transaction_costs.payment_reference, - transaction_costs.bill_number, - transaction_costs.base_value, - transaction_costs.quote_value, - transaction_costs.base_to_quote_currency_exchange_rate, - transaction_costs.base_tax, - transaction_costs.base_service_charge, + transaction_cost_id, + transaction_order_id, + supplier_company_id, + bank_id, + base_currency_id, + quote_currency_id, + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, -- use for future if base currency id other from '1' - IFF(transaction_costs.base_currency_id = '1', - transaction_costs.base_value, + IFF(base_currency_id = '1', + base_value, -9999999999) AS value_rm, -- use for future if base currency id other from '1' - IFF(transaction_costs.base_currency_id = '1', - transaction_costs.base_tax, + IFF(base_currency_id = '1', + base_tax, -9999999999) AS tax_rm, -- use for future if base currency id other from '1' - IFF(transaction_costs.base_currency_id = '1', - transaction_costs.base_service_charge, + IFF(base_currency_id = '1', + base_service_charge, -9999999999) AS service_charge_rm, (value_rm + service_charge_rm + tax_rm) AS total_value_rm, - transaction_costs.expired_datetime, - transaction_costs.deleted_datetime, - transaction_costs.created_datetime, - transaction_costs.updated_datetime, - - {%- for status in order_status %} - transaction_cost_status_datetime.cost_{{status.lower()}}_datetime, - {% endfor %} - transaction_cost_status_datetime.cost_created_datetime + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + cost_created_datetime, + cost_completed_datetime, + cost_pending_submission_datetime, + cost_approved_datetime, + cost_pending_verification_datetime, + cost_rejected_datetime, + cost_suspended_datetime, + cost_expired_datetime FROM transaction_costs - LEFT JOIN transaction_cost_status_datetime - ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id) - ), bookings_lists AS ( + SELECT bookings.*, @@ -370,6 +366,7 @@ bookings_lists AS ( LEFT JOIN booking_status_datetime ON (bookings.booking_id = booking_status_datetime.booking_id) + ), bookings_orders_costs_join AS ( @@ -452,9 +449,7 @@ bookings_orders_costs_join AS ( transaction_costs_lists.cost_pending_verification_datetime, transaction_costs_lists.cost_approved_datetime, transaction_costs_lists.cost_completed_datetime, - transaction_costs_lists.cost_rejected_datetime, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + transaction_costs_lists.cost_rejected_datetime FROM transaction_orders_lists @@ -466,6 +461,112 @@ bookings_orders_costs_join AS ( ), +purchase_orders_invoices_join AS ( + + SELECT + bookings_orders_costs_join.*, + invoices.created_datetime AS invoice_created_datetime, + purchase_orders.created_datetime AS purchase_order_created_datetime, + purchase_orders.updated_datetime AS purchase_order_updated_datetime, + + CASE + WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.purchase_order_number + END AS purchase_order_number, + CASE + WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.status + END AS purchase_order_status, + + CASE + WHEN cost_status IN ('APPROVED') THEN invoices.invoice_number + END AS invoice_number, + CASE + WHEN cost_status IN ('APPROVED') THEN invoices.status + END AS invoice_status + + FROM bookings_orders_costs_join + + LEFT JOIN purchase_orders + ON (bookings_orders_costs_join.booking_id = purchase_orders.booking_id) + + LEFT JOIN invoices + ON (bookings_orders_costs_join.booking_id = invoices.booking_id) + +), + +purchase_order_invoice_status_redefined AS ( + + SELECT + *, + + COALESCE( 'COST_' ||cost_status, 'ORDER_' || order_status ) AS transaction_temp_status, + + -- Transaction status tracks the orders and costs + CASE + + WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IS NULL + THEN 'PENDING_GENERATE_WHITE_FORM' + WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IN ('PENDING_VERIFICATION') + THEN 'GENERATED_WHITE_FORM' + WHEN transaction_temp_status IN ('COST_APPROVED', 'COST_COMPLETED') + THEN 'UPLOADED_BANK_SLIP' + WHEN transaction_temp_status IN ('ORDER_APPROVED') + THEN 'VERIFIED_PAYMENT_STATEMENT' + WHEN transaction_temp_status IN ('ORDER_PENDING_VERIFICATION') + THEN 'PENDING_VERIFICATION_PAYMENT_STATEMENT' + WHEN transaction_temp_status IN ('ORDER_PENDING_SUBMISSION') + THEN 'PENDING_SUBMISSION_PAYMENT_STATEMENT' + + END AS transaction_temp_status_2, + + -- Document status tracks the purchase orders and invoices + CASE + + WHEN order_status IN ('REJECTED', 'SUSPENDED', 'EXPIRED') + THEN null + WHEN ( purchase_order_status IS NULL OR purchase_order_status IN ('PENDING_SUBMISSION') ) AND purchase_order_number IS NULL + THEN 'PENDING_SUBMISSION_PURCHASE_ORDER' + WHEN purchase_order_status IN ('PENDING_VERIFICATION') + THEN 'PENDING_VERIFICATION_PURCHASE_ORDER' + WHEN purchase_order_number IS NOT NULL AND purchase_order_status IN ( 'PENDING_VERIFICATION', 'PENDING_SUBMISSION' ) AND purchase_order_number IS NOT NULL + THEN 'PENDING_RESUBMISSION_PURCHASE_ORDER' + WHEN purchase_order_status IN ('APPROVED') AND invoice_number IS NULL AND cost_status IN ('APPROVED', 'COMPLETED') + THEN 'PENDING_GENERATE_INVOICE' + WHEN purchase_order_status IN ('APPROVED') AND invoice_status IS NULL + THEN 'APPROVED_PURCHASE_ORDER' + WHEN invoice_status IN ('APPROVED') AND cost_status IN ('COMPLETED', 'APPROVED') + THEN 'GENERATED_INVOICE' + + END AS document_status, + + COALESCE(transaction_temp_status_2, transaction_temp_status ) AS transaction_status + + FROM purchase_orders_invoices_join + +), + +renaming_datetime_columns AS ( + + SELECT + *, + booking_created_datetime AS customer_made_booking_datetime, + order_created_datetime AS customer_placed_order_datetime, + order_pending_verification_datetime AS customer_made_payment_datetime, + order_approved_datetime AS account_verified_payment_datetime, + order_completed_datetime AS first_white_form_generated_datetime, + cost_pending_verification_datetime AS last_white_form_generated_datetime, + COALESCE(cost_approved_datetime, cost_completed_datetime) AS operation_uploaded_bank_slip_datetime, + purchase_order_created_datetime AS first_purchase_order_uploaded_datetime, + purchase_order_updated_datetime AS last_purchase_order_uploaded_datetime, + invoice_created_datetime AS operation_generated_invoice_datetime, + order_rejected_datetime AS account_rejected_payment_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM purchase_order_invoice_status_redefined + +), + + -- FINAL final_fct_exchange__new_orders AS ( @@ -485,6 +586,8 @@ final_fct_exchange__new_orders AS ( base_currency_id, -- dimensions + transaction_status, + document_status, service_type, booking_status, is_first_time_booking_company, @@ -505,6 +608,10 @@ final_fct_exchange__new_orders AS ( cost_status, cost_payment_reference, cost_bill_number, + purchase_order_status, + purchase_order_number, + invoice_status, + invoice_number, -- measures estimate_booking_base_to_quote_currency_exchange_rate, @@ -520,6 +627,7 @@ final_fct_exchange__new_orders AS ( order_service_charge_rm, order_tax_rm, total_order_value_rm, + cost_base_value, cost_quote_value, cost_base_to_quote_currency_exchange_rate, @@ -548,18 +656,31 @@ final_fct_exchange__new_orders AS ( order_suspended_datetime, order_expired_datetime, - next_order_created_datetime, cost_created_datetime, cost_pending_submission_datetime, cost_pending_verification_datetime, cost_approved_datetime, cost_completed_datetime, - cost_rejected_datetime, + cost_rejected_datetime, + + customer_made_booking_datetime, + customer_placed_order_datetime, + customer_made_payment_datetime, + account_verified_payment_datetime, + first_white_form_generated_datetime, + last_white_form_generated_datetime, + operation_uploaded_bank_slip_datetime, + first_purchase_order_uploaded_datetime, + last_purchase_order_uploaded_datetime, + operation_generated_invoice_datetime, + account_rejected_payment_datetime, + next_order_created_datetime, -- metadata _dbt_ran_datetime - FROM bookings_orders_costs_join + FROM renaming_datetime_columns + ) SELECT * FROM final_fct_exchange__new_orders \ No newline at end of file