From dd42248cddd516071b4a3f1b76a69734e2f34da0 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Wed, 6 Mar 2024 03:42:37 +0000 Subject: [PATCH 1/9] build base wallets --- .../base/base_exchange__wallet_logs.sql | 58 +++++++++++++++++++ .../exchange/base/base_exchange__wallets.sql | 56 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 models/staging/exchange/base/base_exchange__wallet_logs.sql create mode 100644 models/staging/exchange/base/base_exchange__wallets.sql diff --git a/models/staging/exchange/base/base_exchange__wallet_logs.sql b/models/staging/exchange/base/base_exchange__wallet_logs.sql new file mode 100644 index 0000000..c5b13d5 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__wallet_logs.sql @@ -0,0 +1,58 @@ +-- IMPORT +WITH source AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'wallet_logs') }} +), + + +-- LOGIC +renamed AS ( + + SELECT + id AS wallet_log_id, + wallet_id, + owner_type, + owner_id, + code, + currency_id, + amount, + deleted_at AS deleted_datetime, + created_at AS created_datetime, + updated_at AS updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM source + +), + + +-- FINAL +final__base_exchange__wallet_logs AS ( + + SELECT + -- ids + wallet_log_id, + wallet_id, + owner_id, + currency_id, + + -- dimensions + owner_type, + code, + + -- measures + amount, + + -- date/time + created_datetime, + updated_datetime, + deleted_datetime, + + -- metadata + _dbt_ran_datetime + + FROM renamed + +) + +SELECT * FROM final__base_exchange__wallet_logs \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__wallets.sql b/models/staging/exchange/base/base_exchange__wallets.sql new file mode 100644 index 0000000..ead58e7 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__wallets.sql @@ -0,0 +1,56 @@ +-- IMPORT +WITH source AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'wallets') }} +), + + +-- LOGIC +renamed AS ( + + SELECT + id AS wallet_id, + owner_type, + owner_id, + code, + currency_id, + amount, + deleted_at AS deleted_datetime, + created_at AS created_datetime, + updated_at AS updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM source + +), + + +-- FINAL +final__base_exchange__wallets AS ( + + SELECT + -- ids + wallet_id, + owner_id, + currency_id, + + -- dimensions + owner_type, + code, + + -- measures + amount, + + -- date/time + created_datetime, + updated_datetime, + deleted_datetime, + + -- metadata + _dbt_ran_datetime + + FROM renamed + +) + +SELECT * FROM final__base_exchange__wallets \ No newline at end of file From 3c0ced171af49f5178e4ff9a0207a0c453b0ac4e Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Tue, 12 Mar 2024 01:08:06 +0000 Subject: [PATCH 2/9] Build cash flow analysis --- .../base/base_exchange__wallet_logs.sql | 2 +- .../exchange/base/base_exchange__wallets.sql | 2 +- .../stg_exchange__wallet_balance_logs.sql | 58 +++++++++++++++++++ .../stg_exchange__wallet_balances.sql | 56 ++++++++++++++++++ 4 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 models/staging/exchange/stg_exchange__wallet_balance_logs.sql create mode 100644 models/staging/exchange/stg_exchange__wallet_balances.sql diff --git a/models/staging/exchange/base/base_exchange__wallet_logs.sql b/models/staging/exchange/base/base_exchange__wallet_logs.sql index c5b13d5..cab7d77 100644 --- a/models/staging/exchange/base/base_exchange__wallet_logs.sql +++ b/models/staging/exchange/base/base_exchange__wallet_logs.sql @@ -37,8 +37,8 @@ final__base_exchange__wallet_logs AS ( currency_id, -- dimensions - owner_type, code, + owner_type, -- measures amount, diff --git a/models/staging/exchange/base/base_exchange__wallets.sql b/models/staging/exchange/base/base_exchange__wallets.sql index ead58e7..cfdbc29 100644 --- a/models/staging/exchange/base/base_exchange__wallets.sql +++ b/models/staging/exchange/base/base_exchange__wallets.sql @@ -35,8 +35,8 @@ final__base_exchange__wallets AS ( currency_id, -- dimensions - owner_type, code, + owner_type, -- measures amount, diff --git a/models/staging/exchange/stg_exchange__wallet_balance_logs.sql b/models/staging/exchange/stg_exchange__wallet_balance_logs.sql new file mode 100644 index 0000000..8116af9 --- /dev/null +++ b/models/staging/exchange/stg_exchange__wallet_balance_logs.sql @@ -0,0 +1,58 @@ +-- IMPORT +WITH wallet_logs AS ( + SELECT * FROM {{ ref('base_exchange__wallet_logs') }} +), + + +-- LOGIC +renamed AS ( + + SELECT + wallet_log_id, + wallet_id, + owner_id AS company_id, + currency_id, + code, + amount AS wallet_balance, + created_datetime, + updated_datetime, + deleted_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM wallet_logs + + WHERE owner_type = 'App\\Models\\Company' + +), + + +-- FINAL +final__base_exchange__wallet_logs AS ( + + SELECT + --ids + wallet_log_id, + wallet_id, + company_id, + currency_id, + + --dimensions + code, + + --measures + wallet_balance, + + --date/times + created_datetime, + updated_datetime, + deleted_datetime, + + --metadata + _dbt_ran_datetime + + FROM renamed + +) + +SELECT * FROM final__base_exchange__wallet_logs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__wallet_balances.sql b/models/staging/exchange/stg_exchange__wallet_balances.sql new file mode 100644 index 0000000..9409da4 --- /dev/null +++ b/models/staging/exchange/stg_exchange__wallet_balances.sql @@ -0,0 +1,56 @@ +-- IMPORT +WITH wallets AS ( + SELECT * FROM {{ ref('base_exchange__wallets') }} +), + + +-- LOGIC +renamed AS ( + + SELECT + wallet_id, + owner_id AS company_id, + currency_id, + code, + amount AS wallet_balance, + created_datetime, + updated_datetime, + deleted_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM wallets + + WHERE owner_type = 'App\\Models\\Company' + +), + + +-- FINAL +final__stg__exchange__wallets AS ( + + SELECT + --ids + wallet_id, + company_id, + currency_id, + + --dimensions + code, + + --measures + wallet_balance, + + --date/times + created_datetime, + updated_datetime, + deleted_datetime, + + --metadata + _dbt_ran_datetime + + FROM renamed + +) + +SELECT * FROM final__stg__exchange__wallets \ No newline at end of file From 0550be2ed3c2861257d44ba06286fa5e6c947f7e Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 15 Mar 2024 07:15:38 +0000 Subject: [PATCH 3/9] build wallet cash flow analysis --- ...ep_exchange__wallet_cash_flow_analysis.sql | 82 ++++++ ...ct_exchange__wallet_daily_transactions.sql | 266 ++++++++++++++++++ 2 files changed, 348 insertions(+) create mode 100644 models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql create mode 100644 models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql diff --git a/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql b/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql new file mode 100644 index 0000000..574812b --- /dev/null +++ b/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql @@ -0,0 +1,82 @@ +-- IMPORT +WITH daily_wallet_transactions AS ( + SELECT * FROM {{ ref('fct_exchange__wallet_daily_transactions') }} +), + +companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + + +-- LOGIC +daily_wallet_transaction_enrich AS ( + + SELECT + daily_wallet_transactions.* EXCLUDE _dbt_ran_datetime, + + companies.company_marking_id, + companies.company_type, + companies.business_type, + companies.exchange_rate_segment, + companies.is_migrated_company, + companies.company_created_datetime, + + MD5_NUMBER_LOWER64(CONCAT(daily_wallet_transactions.wallet_id, transaction_date))::STRING AS wallet_cash_flow_transaction_id, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM daily_wallet_transactions + + LEFT JOIN companies + ON ( daily_wallet_transactions.company_id = companies.company_id) + +), + + +-- FINAL +final__rep_exchange__wallet_cash_flow_analysis AS ( + + SELECT + --ids + wallet_cash_flow_transaction_id, + wallet_id, + company_id, + company_marking_id, + currency_id, + first_wallet_log_id, + last_wallet_log_id, + + --dimensions + company_type, + business_type, + exchange_rate_segment, + is_migrated_company, + is_latest_wallet_balance, + + --measures + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, + prev_daily_last_wallet_balance, + daily_first_wallet_balance, + daily_last_wallet_balance, + total_daily_transaction_credit, + total_daily_transaction_debit, + + --date/time + transaction_date, + company_created_datetime, + wallet_registration_datetime, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime, + + --metadata + _dbt_ran_datetime + + FROM daily_wallet_transaction_enrich + +) + +SELECT * FROM final__rep_exchange__wallet_cash_flow_analysis \ No newline at end of file diff --git a/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql b/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql new file mode 100644 index 0000000..556e406 --- /dev/null +++ b/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql @@ -0,0 +1,266 @@ +-- IMPORT +WITH wallet_balance_logs AS ( + SELECT * FROM {{ ref('stg_exchange__wallet_balance_logs') }} +), + +latest_wallet_balances AS ( + SELECT * FROM {{ ref('stg_exchange__wallet_balances') }} +), + +wallet_transaction_logs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_wallet_logs') }} + WHERE status IN ('APPROVED') +), + + +-- LOGIC +-- wallet transactions +daily_transaction_log AS ( + + SELECT + wallet_id, + company_id, + base_currency_id, + quote_currency_id, + DATE(updated_datetime) AS transaction_date, + + MIN(created_datetime) AS first_daily_transaction_start_datetime, + MAX(created_datetime) AS last_daily_transaction_start_datetime, + + MIN(updated_datetime) AS first_daily_transaction_completed_datetime, + MAX(updated_datetime) AS last_daily_transaction_completed_datetime, + + SUM(CASE + WHEN transaction_type IN ('PAYMENT', 'DEBIT_NOTE') + THEN -( base_value ) + ELSE 0 + END) AS total_daily_transaction_debit, + + SUM(CASE + WHEN transaction_type IN ('TOP_UP', 'CREDIT_NOTE') + THEN base_value + ELSE 0 + END) AS total_daily_transaction_credit, + + COUNT(CASE + WHEN transaction_type IN ('TOP_UP', 'CREDIT_NOTE') + THEN wallet_id + END) AS count_total_daily_transaction_credit, + + COUNT(CASE + WHEN transaction_type IN ('PAYMENT', 'DEBIT_NOTE') + THEN wallet_id + END) AS count_total_daily_transaction_debit + + FROM wallet_transaction_logs + + GROUP BY + wallet_id, + company_id, + base_currency_id, + quote_currency_id, + transaction_date + +), + +-- wallet balances +wallet_balance_list AS ( + + SELECT + wallet_log_id, + wallet_id, + company_id, + currency_id, + wallet_balance, + created_datetime AS wallet_registration_datetime, + updated_datetime + FROM wallet_balance_logs + + UNION + + SELECT + MD5_NUMBER_LOWER64(CONCAT(wallet_id, wallet_balance, updated_datetime)) AS wallet_log_id, + wallet_id, + company_id, + currency_id, + wallet_balance, + created_datetime AS wallet_registration_datetime, + updated_datetime + FROM latest_wallet_balances + +), + +wallet_balance_list_enrich AS ( + + SELECT + wallet_log_id, + wallet_id, + company_id, + currency_id, + wallet_balance, + wallet_registration_datetime, + updated_datetime, + DATE(updated_datetime) AS updated_date, + LAG(wallet_balance) OVER (PARTITION BY wallet_id ORDER BY updated_datetime) AS prev_wallet_balance, + LEAD(wallet_balance) OVER (PARTITION BY wallet_id ORDER BY updated_datetime) AS next_wallet_balance + + FROM wallet_balance_list + +), + +wallet_balance_daily_aggregate AS ( + + SELECT + DISTINCT + wallet_id, + company_id, + currency_id, + wallet_registration_datetime, + updated_date, + COUNT(wallet_id) OVER (PARTITION BY wallet_id, updated_date) AS count_daily_wallet_activity, + FIRST_VALUE(wallet_log_id) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS first_wallet_log_id, + LAST_VALUE(wallet_log_id) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS last_wallet_log_id, + + FIRST_VALUE(wallet_balance) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS daily_first_wallet_balance, + LAST_VALUE(wallet_balance) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS daily_last_wallet_balance, + + FROM wallet_balance_list_enrich + +), + +wallet_balance_list_join_wallet_transaction AS ( + + SELECT + COALESCE(wallet_balance_daily_aggregate.updated_date, daily_transaction_log.transaction_date) AS transaction_date, + COALESCE(wallet_balance_daily_aggregate.wallet_id, daily_transaction_log.wallet_id) AS wallet_id, + COALESCE(wallet_balance_daily_aggregate.company_id, daily_transaction_log.company_id) AS company_id, + COALESCE(wallet_balance_daily_aggregate.currency_id, daily_transaction_log.base_currency_id) AS currency_id, + + -- Wallet data + COALESCE(wallet_balance_daily_aggregate.count_daily_wallet_activity, 0) AS count_daily_wallet_activity, + COALESCE(wallet_balance_daily_aggregate.first_wallet_log_id, NULL) AS first_wallet_log_id, + COALESCE(wallet_balance_daily_aggregate.last_wallet_log_id, NULL) AS last_wallet_log_id, + COALESCE(wallet_balance_daily_aggregate.daily_first_wallet_balance, NULL) AS daily_first_wallet_balance, + COALESCE(wallet_balance_daily_aggregate.daily_last_wallet_balance, NULL) AS daily_last_wallet_balance, + COALESCE(wallet_balance_daily_aggregate.wallet_registration_datetime, NULL) AS wallet_registration_datetime, + + -- Transaction data + COALESCE(daily_transaction_log.first_daily_transaction_start_datetime, NULL) AS first_daily_transaction_start_datetime, + COALESCE(daily_transaction_log.last_daily_transaction_start_datetime, NULL) AS last_daily_transaction_start_datetime, + COALESCE(daily_transaction_log.first_daily_transaction_completed_datetime, NULL) AS first_daily_transaction_completed_datetime, + COALESCE(daily_transaction_log.last_daily_transaction_completed_datetime, NULL) AS last_daily_transaction_completed_datetime, + COALESCE(daily_transaction_log.total_daily_transaction_debit, NULL) AS total_daily_transaction_debit, + COALESCE(daily_transaction_log.total_daily_transaction_credit, NULL) AS total_daily_transaction_credit, + COALESCE(daily_transaction_log.count_total_daily_transaction_credit, 0) AS count_total_daily_transaction_credit, + COALESCE(daily_transaction_log.count_total_daily_transaction_debit, 0) AS count_total_daily_transaction_debit, + + FROM wallet_balance_daily_aggregate + + FULL OUTER JOIN daily_transaction_log + ON (wallet_balance_daily_aggregate.wallet_id = daily_transaction_log.wallet_id) + AND (wallet_balance_daily_aggregate.updated_date = daily_transaction_log.transaction_date) + +), + +data_cleaning AS ( + + SELECT + wallet_id, + company_id, + currency_id, + first_wallet_log_id, + last_wallet_log_id, + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, + + LAG(daily_last_wallet_balance) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_daily_last_wallet_balance, + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date DESC) = 1 + THEN 1 + ELSE 0 + END AS is_latest_wallet_balance, + + CASE + /* + When opening balance does not equal to previous day's closing balance + Take previous day's closing balance as opening balance + */ + WHEN + daily_first_wallet_balance != prev_daily_last_wallet_balance + AND + daily_first_wallet_balance IS NOT NULL + THEN prev_daily_last_wallet_balance + /* + When there is transaction, but no wallet balance log + Take daily credit as first wallet balance + */ + WHEN daily_first_wallet_balance IS NULL + THEN total_daily_transaction_credit + + ELSE daily_first_wallet_balance + END AS daily_first_wallet_balance, + + CASE + WHEN daily_last_wallet_balance IS NULL + THEN ( total_daily_transaction_credit + total_daily_transaction_debit ) + ELSE daily_last_wallet_balance + END AS daily_last_wallet_balance, + + total_daily_transaction_credit, + total_daily_transaction_debit, + wallet_registration_datetime, + transaction_date, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM wallet_balance_list_join_wallet_transaction + +), + + +-- FINAL +final__fct_exchange__daily_wallet_transactions AS ( + + SELECT + --ids + wallet_id, + company_id, + currency_id, + first_wallet_log_id, + last_wallet_log_id, + + --dimensions + is_latest_wallet_balance, + + --measures + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, + prev_daily_last_wallet_balance, + daily_first_wallet_balance, + daily_last_wallet_balance, + total_daily_transaction_credit, + total_daily_transaction_debit, + + --date/times + wallet_registration_datetime, + transaction_date, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime, + + --metadata + _dbt_ran_datetime + + FROM data_cleaning + +) + +SELECT * FROM final__fct_exchange__daily_wallet_transactions \ No newline at end of file From 1b107599f64c415b71b523f5318fda5a2a15671a Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Fri, 22 Mar 2024 06:27:27 +0000 Subject: [PATCH 4/9] updated logic to calculate wallet balance --- ...ep_exchange__wallet_cash_flow_analysis.sql | 11 +- ...ct_exchange__wallet_daily_transactions.sql | 178 ++++++-- .../stg_exchange__transaction_wallets.sql | 413 ++++++++++++++---- 3 files changed, 486 insertions(+), 116 deletions(-) diff --git a/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql b/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql index 574812b..e3ab9d6 100644 --- a/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql +++ b/models/marts/reporting/rep_exchange__wallet_cash_flow_analysis.sql @@ -57,12 +57,15 @@ final__rep_exchange__wallet_cash_flow_analysis AS ( count_daily_wallet_activity, count_total_daily_transaction_credit, count_total_daily_transaction_debit, - prev_daily_last_wallet_balance, - daily_first_wallet_balance, - daily_last_wallet_balance, total_daily_transaction_credit, total_daily_transaction_debit, - + original_daily_first_wallet_balance, + original_daily_last_wallet_balance, + daily_0_origin_first_wallet_balance, + daily_0_origin_last_wallet_balance, + daily_calculated_first_wallet_balance, + daily_calculated_last_wallet_balance, + --date/time transaction_date, company_created_datetime, diff --git a/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql b/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql index 556e406..ebd9dbf 100644 --- a/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql +++ b/models/marts/warehouse/fct_exchange__wallet_daily_transactions.sql @@ -162,7 +162,7 @@ wallet_balance_list_join_wallet_transaction AS ( ), -data_cleaning AS ( +minor_data_imputation AS ( SELECT wallet_id, @@ -173,54 +173,154 @@ data_cleaning AS ( count_daily_wallet_activity, count_total_daily_transaction_credit, count_total_daily_transaction_debit, - - LAG(daily_last_wallet_balance) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_daily_last_wallet_balance, - + daily_last_wallet_balance, + CASE WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date DESC) = 1 THEN 1 ELSE 0 END AS is_latest_wallet_balance, - - CASE - /* - When opening balance does not equal to previous day's closing balance - Take previous day's closing balance as opening balance - */ - WHEN - daily_first_wallet_balance != prev_daily_last_wallet_balance - AND - daily_first_wallet_balance IS NOT NULL - THEN prev_daily_last_wallet_balance - /* - When there is transaction, but no wallet balance log - Take daily credit as first wallet balance - */ - WHEN daily_first_wallet_balance IS NULL - THEN total_daily_transaction_credit - - ELSE daily_first_wallet_balance - END AS daily_first_wallet_balance, CASE - WHEN daily_last_wallet_balance IS NULL - THEN ( total_daily_transaction_credit + total_daily_transaction_debit ) - ELSE daily_last_wallet_balance - END AS daily_last_wallet_balance, + WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date) = 1 + THEN 1 + ELSE 0 + END AS is_earliest_wallet_balance, + /* + Initialize for older records without wallet balance. (2 scenario) + 1. Set closing balance as 0 + 2. Set closing balance as difference between credit and debit + */ + CASE + WHEN is_earliest_wallet_balance = 1 + THEN COALESCE(daily_first_wallet_balance, 0) + ELSE daily_first_wallet_balance + END daily_first_wallet_balance, + + CASE + WHEN is_earliest_wallet_balance = 1 + THEN COALESCE(daily_last_wallet_balance, 0) + ELSE daily_last_wallet_balance + END daily_0_origin_last_wallet_balance, + + CASE + WHEN is_earliest_wallet_balance = 1 + THEN COALESCE(daily_last_wallet_balance, total_daily_transaction_credit + total_daily_transaction_debit) + ELSE daily_last_wallet_balance + END daily_calculated_last_wallet_balance, + + + CASE + WHEN wallet_registration_datetime IS NULL + THEN ( + SELECT MIN(wallet_registration_datetime) + FROM wallet_balance_list_join_wallet_transaction t1 + WHERE t1.wallet_id = wallet_balance_list_join_wallet_transaction.wallet_id + ) + END AS new_wallet_registration_datetime, + + COALESCE(total_daily_transaction_credit, 0) AS total_daily_transaction_credit, + COALESCE(total_daily_transaction_debit, 0) AS total_daily_transaction_debit, + transaction_date, + LEAD(transaction_date) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS next_transaction_date, + LAG(transaction_date) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_transaction_date, + wallet_registration_datetime, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime + + FROM wallet_balance_list_join_wallet_transaction + +), + +recursive_imputation AS ( + + SELECT + *, + + daily_first_wallet_balance AS new_daily_0_origin_first_wallet_balance, + daily_0_origin_last_wallet_balance AS new_daily_0_origin_last_wallet_balance, + + daily_first_wallet_balance AS new_daily_calculated_first_wallet_balance, + daily_calculated_last_wallet_balance AS new_daily_calculated_last_wallet_balance, + + FROM minor_data_imputation + + WHERE is_earliest_wallet_balance = 1 + + UNION ALL + + SELECT + t1.*, + + t2.new_daily_0_origin_last_wallet_balance AS new_daily_0_origin_first_wallet_balance, + t2.new_daily_0_origin_last_wallet_balance + t1.total_daily_transaction_credit + t1.total_daily_transaction_debit AS new_daily_0_origin_last_wallet_balance, + + t2.new_daily_calculated_last_wallet_balance AS new_daily_calculated_first_wallet_balance, + t2.new_daily_calculated_last_wallet_balance + t1.total_daily_transaction_credit + t1.total_daily_transaction_debit AS new_daily_calculated_last_wallet_balance + + + FROM minor_data_imputation t1 + + INNER JOIN recursive_imputation t2 + + ON ( + t1.wallet_id = t2.wallet_id + AND + t2.transaction_date = t1.prev_transaction_date + ) + +), + +casting_and_organize AS ( + + SELECT + wallet_id, + company_id, + currency_id, + first_wallet_log_id::STRING AS first_wallet_log_id, + last_wallet_log_id::STRING AS last_wallet_log_id, + is_earliest_wallet_balance, + is_latest_wallet_balance, + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, total_daily_transaction_credit, total_daily_transaction_debit, - wallet_registration_datetime, + LAG(daily_last_wallet_balance) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_daily_last_wallet_balance, + + COALESCE(CASE + WHEN prev_daily_last_wallet_balance IS NULL + THEN NULL + WHEN daily_first_wallet_balance != prev_daily_last_wallet_balance + THEN prev_daily_last_wallet_balance + ELSE daily_first_wallet_balance + END, 0) AS original_daily_first_wallet_balance, + + COALESCE(daily_last_wallet_balance, 0) AS original_daily_last_wallet_balance, + + COALESCE(new_daily_0_origin_first_wallet_balance, 0) AS daily_0_origin_first_wallet_balance, + COALESCE(new_daily_0_origin_last_wallet_balance, 0) AS daily_0_origin_last_wallet_balance, + + COALESCE(new_daily_calculated_first_wallet_balance, 0) AS daily_calculated_first_wallet_balance, + COALESCE(new_daily_calculated_last_wallet_balance, 0) AS daily_calculated_last_wallet_balance, + + prev_transaction_date, transaction_date, + next_transaction_date, first_daily_transaction_start_datetime, first_daily_transaction_completed_datetime, last_daily_transaction_start_datetime, last_daily_transaction_completed_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM wallet_balance_list_join_wallet_transaction + COALESCE(wallet_registration_datetime, new_wallet_registration_datetime) AS wallet_registration_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + FROM recursive_imputation + ), @@ -236,30 +336,34 @@ final__fct_exchange__daily_wallet_transactions AS ( last_wallet_log_id, --dimensions + is_earliest_wallet_balance, is_latest_wallet_balance, --measures count_daily_wallet_activity, count_total_daily_transaction_credit, count_total_daily_transaction_debit, - prev_daily_last_wallet_balance, - daily_first_wallet_balance, - daily_last_wallet_balance, total_daily_transaction_credit, total_daily_transaction_debit, + original_daily_first_wallet_balance, + original_daily_last_wallet_balance, + daily_0_origin_first_wallet_balance, + daily_0_origin_last_wallet_balance, + daily_calculated_first_wallet_balance, + daily_calculated_last_wallet_balance, --date/times - wallet_registration_datetime, transaction_date, first_daily_transaction_start_datetime, first_daily_transaction_completed_datetime, last_daily_transaction_start_datetime, last_daily_transaction_completed_datetime, + wallet_registration_datetime, --metadata _dbt_ran_datetime - FROM data_cleaning + FROM casting_and_organize ) diff --git a/models/staging/exchange/stg_exchange__transaction_wallets.sql b/models/staging/exchange/stg_exchange__transaction_wallets.sql index 3e8cf3a..0271377 100644 --- a/models/staging/exchange/stg_exchange__transaction_wallets.sql +++ b/models/staging/exchange/stg_exchange__transaction_wallets.sql @@ -18,93 +18,356 @@ status AS ( -- LOGIC -transactions_join_transaction_types_payment_methods_currencies_status AS ( +-- wallet transactions +daily_transaction_log AS ( SELECT - transactions.transaction_id AS transaction_wallet_id, - IFF(transactions.owner_type = 'App\\Models\\Wallet', transactions.owner_id, null) AS wallet_id, - - COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, - - transactions.receiver_user_id AS company_id, - - COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, - - transactions.payment_reference, - transactions.bill_number, - - transactions.base_value, - transactions.quote_value, - - transactions.base_currency_id, - transactions.quote_currency_id, - - transactions.base_to_quote_currency_exchange_rate, - transactions.base_tax, - transactions.base_service_charge, - - transactions.expired_datetime, - - COALESCE(status.name, transactions.status::string) AS status, - - transactions.deleted_datetime, - transactions.created_datetime, - transactions.updated_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM transactions - - LEFT JOIN transaction_types - ON (transactions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transactions.payment_method = payment_methods.id) - - LEFT JOIN status - ON (transactions.status = status.id) - - - WHERE - transactions.owner_type = 'App\\Models\\Wallet' -), - - --- FINAL -final__stg_exchange__transaction_wallets AS ( - - SELECT - -- ids - transaction_wallet_id, wallet_id, company_id, base_currency_id, quote_currency_id, + DATE(updated_datetime) AS transaction_date, + + MIN(created_datetime) AS first_daily_transaction_start_datetime, + MAX(created_datetime) AS last_daily_transaction_start_datetime, + + MIN(updated_datetime) AS first_daily_transaction_completed_datetime, + MAX(updated_datetime) AS last_daily_transaction_completed_datetime, + + SUM(CASE + WHEN transaction_type IN ('PAYMENT', 'DEBIT_NOTE') + THEN -( base_value ) + ELSE 0 + END) AS total_daily_transaction_debit, + + SUM(CASE + WHEN transaction_type IN ('TOP_UP', 'CREDIT_NOTE') + THEN base_value + ELSE 0 + END) AS total_daily_transaction_credit, - -- dimensions - transaction_type, - payment_method, - status, - payment_reference, - bill_number, + COUNT(CASE + WHEN transaction_type IN ('TOP_UP', 'CREDIT_NOTE') + THEN wallet_id + END) AS count_total_daily_transaction_credit, - -- measures - base_value, - quote_value, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, + COUNT(CASE + WHEN transaction_type IN ('PAYMENT', 'DEBIT_NOTE') + THEN wallet_id + END) AS count_total_daily_transaction_debit + + FROM wallet_transaction_logs - -- date/times - expired_datetime, - deleted_datetime, - created_datetime, + GROUP BY + wallet_id, + company_id, + base_currency_id, + quote_currency_id, + transaction_date + +), + +-- wallet balances +wallet_balance_list AS ( + + SELECT + wallet_log_id, + wallet_id, + company_id, + currency_id, + wallet_balance, + created_datetime AS wallet_registration_datetime, + updated_datetime + FROM wallet_balance_logs + + UNION + + SELECT + MD5_NUMBER_LOWER64(CONCAT(wallet_id, wallet_balance, updated_datetime)) AS wallet_log_id, + wallet_id, + company_id, + currency_id, + wallet_balance, + created_datetime AS wallet_registration_datetime, + updated_datetime + FROM latest_wallet_balances + +), + +wallet_balance_list_enrich AS ( + + SELECT + wallet_log_id, + wallet_id, + company_id, + currency_id, + wallet_balance, + wallet_registration_datetime, updated_datetime, + DATE(updated_datetime) AS updated_date, + LAG(wallet_balance) OVER (PARTITION BY wallet_id ORDER BY updated_datetime) AS prev_wallet_balance, + LEAD(wallet_balance) OVER (PARTITION BY wallet_id ORDER BY updated_datetime) AS next_wallet_balance + + FROM wallet_balance_list - -- metadata +), + +wallet_balance_daily_aggregate AS ( + + SELECT + DISTINCT + wallet_id, + company_id, + currency_id, + wallet_registration_datetime, + updated_date, + COUNT(wallet_id) OVER (PARTITION BY wallet_id, updated_date) AS count_daily_wallet_activity, + FIRST_VALUE(wallet_log_id) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS first_wallet_log_id, + LAST_VALUE(wallet_log_id) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS last_wallet_log_id, + + FIRST_VALUE(wallet_balance) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS daily_first_wallet_balance, + LAST_VALUE(wallet_balance) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS daily_last_wallet_balance, + + FROM wallet_balance_list_enrich + +), + +wallet_balance_list_join_wallet_transaction AS ( + + SELECT + COALESCE(wallet_balance_daily_aggregate.updated_date, daily_transaction_log.transaction_date) AS transaction_date, + COALESCE(wallet_balance_daily_aggregate.wallet_id, daily_transaction_log.wallet_id) AS wallet_id, + COALESCE(wallet_balance_daily_aggregate.company_id, daily_transaction_log.company_id) AS company_id, + COALESCE(wallet_balance_daily_aggregate.currency_id, daily_transaction_log.base_currency_id) AS currency_id, + + -- Wallet data + COALESCE(wallet_balance_daily_aggregate.count_daily_wallet_activity, 0) AS count_daily_wallet_activity, + COALESCE(wallet_balance_daily_aggregate.first_wallet_log_id, NULL) AS first_wallet_log_id, + COALESCE(wallet_balance_daily_aggregate.last_wallet_log_id, NULL) AS last_wallet_log_id, + COALESCE(wallet_balance_daily_aggregate.daily_first_wallet_balance, NULL) AS daily_first_wallet_balance, + COALESCE(wallet_balance_daily_aggregate.daily_last_wallet_balance, NULL) AS daily_last_wallet_balance, + COALESCE(wallet_balance_daily_aggregate.wallet_registration_datetime, NULL) AS wallet_registration_datetime, + + -- Transaction data + COALESCE(daily_transaction_log.first_daily_transaction_start_datetime, NULL) AS first_daily_transaction_start_datetime, + COALESCE(daily_transaction_log.last_daily_transaction_start_datetime, NULL) AS last_daily_transaction_start_datetime, + COALESCE(daily_transaction_log.first_daily_transaction_completed_datetime, NULL) AS first_daily_transaction_completed_datetime, + COALESCE(daily_transaction_log.last_daily_transaction_completed_datetime, NULL) AS last_daily_transaction_completed_datetime, + COALESCE(daily_transaction_log.total_daily_transaction_debit, NULL) AS total_daily_transaction_debit, + COALESCE(daily_transaction_log.total_daily_transaction_credit, NULL) AS total_daily_transaction_credit, + COALESCE(daily_transaction_log.count_total_daily_transaction_credit, 0) AS count_total_daily_transaction_credit, + COALESCE(daily_transaction_log.count_total_daily_transaction_debit, 0) AS count_total_daily_transaction_debit, + + FROM wallet_balance_daily_aggregate + + FULL OUTER JOIN daily_transaction_log + ON (wallet_balance_daily_aggregate.wallet_id = daily_transaction_log.wallet_id) + AND (wallet_balance_daily_aggregate.updated_date = daily_transaction_log.transaction_date) + +), + +minor_data_imputation AS ( + + SELECT + wallet_id, + company_id, + currency_id, + first_wallet_log_id, + last_wallet_log_id, + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, + daily_last_wallet_balance, + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date DESC) = 1 + THEN 1 + ELSE 0 + END AS is_latest_wallet_balance, + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date) = 1 + THEN 1 + ELSE 0 + END AS is_earliest_wallet_balance, + + /* + Initialize for older records without wallet balance. (2 scenario) + 1. Set closing balance as 0 + 2. Set closing balance as difference between credit and debit + */ + CASE + WHEN is_earliest_wallet_balance = 1 + THEN COALESCE(daily_first_wallet_balance, 0) + ELSE daily_first_wallet_balance + END daily_first_wallet_balance, + + CASE + WHEN is_earliest_wallet_balance = 1 + THEN COALESCE(daily_last_wallet_balance, 0) + ELSE daily_last_wallet_balance + END daily_0_origin_last_wallet_balance, + + CASE + WHEN is_earliest_wallet_balance = 1 + THEN COALESCE(daily_last_wallet_balance, total_daily_transaction_credit + total_daily_transaction_debit) + ELSE daily_last_wallet_balance + END daily_calculated_last_wallet_balance, + + + CASE + WHEN wallet_registration_datetime IS NULL + THEN ( + SELECT MIN(wallet_registration_datetime) + FROM wallet_balance_list_join_wallet_transaction t1 + WHERE t1.wallet_id = wallet_balance_list_join_wallet_transaction.wallet_id + ) + END AS new_wallet_registration_datetime, + + total_daily_transaction_credit, + total_daily_transaction_debit, + transaction_date, + LEAD(transaction_date) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS next_transaction_date, + LAG(transaction_date) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_transaction_date, + wallet_registration_datetime, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime + + FROM wallet_balance_list_join_wallet_transaction + +), + +recursive_imputation AS ( + + SELECT + *, + + daily_first_wallet_balance AS new_daily_0_origin_first_wallet_balance, + daily_0_origin_last_wallet_balance AS new_daily_0_origin_last_wallet_balance, + + daily_first_wallet_balance AS new_daily_calculated_first_wallet_balance, + daily_calculated_last_wallet_balance AS new_daily_calculated_last_wallet_balance, + + FROM minor_data_imputation + + WHERE is_earliest_wallet_balance = 1 + + UNION ALL + + SELECT + t1.*, + + t2.new_daily_0_origin_last_wallet_balance AS new_daily_0_origin_first_wallet_balance, + t2.new_daily_0_origin_last_wallet_balance + t1.total_daily_transaction_credit + t1.total_daily_transaction_debit AS new_daily_0_origin_last_wallet_balance, + + t2.new_daily_calculated_last_wallet_balance AS new_daily_calculated_first_wallet_balance, + t2.new_daily_calculated_last_wallet_balance + t1.total_daily_transaction_credit + t1.total_daily_transaction_debit AS new_daily_calculated_last_wallet_balance + + + FROM minor_data_imputation t1 + + INNER JOIN recursive_imputation t2 + + ON ( + t1.wallet_id = t2.wallet_id + AND + t2.transaction_date = t1.prev_transaction_date + ) + +), + +casting_and_organize AS ( + + SELECT + wallet_id, + company_id, + currency_id, + first_wallet_log_id::STRING AS first_wallet_log_id, + last_wallet_log_id::STRING AS last_wallet_log_id, + is_earliest_wallet_balance + is_latest_wallet_balance, + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, + total_daily_transaction_credit, + total_daily_transaction_debit, + LAG(daily_last_wallet_balance) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_daily_last_wallet_balance, + + CASE + WHEN prev_daily_last_wallet_balance IS NULL + THEN NULL + WHEN daily_first_wallet_balance != prev_daily_last_wallet_balance + THEN prev_daily_last_wallet_balance + ELSE daily_first_wallet_balance + END AS original_daily_first_wallet_balance, + + daily_last_wallet_balance AS original_daily_last_wallet_balance, + + new_daily_0_origin_first_wallet_balance AS daily_0_origin_first_wallet_balance, + new_daily_0_origin_last_wallet_balance AS daily_0_origin_last_wallet_balance, + + new_daily_calculated_first_wallet_balance AS daily_calculated_first_wallet_balance, + new_daily_calculated_last_wallet_balance AS daily_calculated_last_wallet_balance, + + prev_transaction_date, + transaction_date, + next_transaction_date, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime, + + COALESCE(wallet_registration_datetime, new_wallet_registration_datetime) AS wallet_registration_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM recursive_imputation + +), + + +-- FINAL +final__fct_exchange__daily_wallet_transactions AS ( + + SELECT + --ids + wallet_id, + company_id, + currency_id, + first_wallet_log_id, + last_wallet_log_id, + + --dimensions + is_latest_wallet_balance, + + --measures + count_daily_wallet_activity, + count_total_daily_transaction_credit, + count_total_daily_transaction_debit, + total_daily_transaction_credit, + total_daily_transaction_debit, + original_daily_first_wallet_balance, + original_daily_last_wallet_balance, + daily_0_origin_first_wallet_balance, + daily_0_origin_last_wallet_balance, + daily_calculated_first_wallet_balance, + daily_calculated_last_wallet_balance, + + --date/times + transaction_date, + first_daily_transaction_start_datetime, + first_daily_transaction_completed_datetime, + last_daily_transaction_start_datetime, + last_daily_transaction_completed_datetime, + wallet_registration_datetime, + + --metadata _dbt_ran_datetime - FROM transactions_join_transaction_types_payment_methods_currencies_status + FROM casting_and_organize -) +) -SELECT * FROM final__stg_exchange__transaction_wallets \ No newline at end of file +SELECT * FROM final__fct_exchange__daily_wallet_transactions \ No newline at end of file From 9de1c91e502e82b45713cb7f30830806ab0c36c0 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Mon, 1 Apr 2024 02:20:04 +0000 Subject: [PATCH 5/9] Fixed corrupted stage model --- .../stg_exchange__transaction_wallets.sql | 392 +++--------------- 1 file changed, 65 insertions(+), 327 deletions(-) diff --git a/models/staging/exchange/stg_exchange__transaction_wallets.sql b/models/staging/exchange/stg_exchange__transaction_wallets.sql index 0271377..fa6c6f8 100644 --- a/models/staging/exchange/stg_exchange__transaction_wallets.sql +++ b/models/staging/exchange/stg_exchange__transaction_wallets.sql @@ -18,356 +18,94 @@ status AS ( -- LOGIC --- wallet transactions -daily_transaction_log AS ( +transactions_join_transaction_types_payment_methods_currencies_status AS ( SELECT - wallet_id, - company_id, - base_currency_id, - quote_currency_id, - DATE(updated_datetime) AS transaction_date, + transactions.transaction_id AS transaction_wallet_id, + IFF(transactions.owner_type = 'App\\Models\\Wallet', transactions.owner_id, null) AS wallet_id, + + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + + transactions.receiver_user_id AS company_id, + + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + + transactions.payment_reference, + transactions.bill_number, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + transactions.quote_currency_id, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + + transactions.expired_datetime, - MIN(created_datetime) AS first_daily_transaction_start_datetime, - MAX(created_datetime) AS last_daily_transaction_start_datetime, - - MIN(updated_datetime) AS first_daily_transaction_completed_datetime, - MAX(updated_datetime) AS last_daily_transaction_completed_datetime, - - SUM(CASE - WHEN transaction_type IN ('PAYMENT', 'DEBIT_NOTE') - THEN -( base_value ) - ELSE 0 - END) AS total_daily_transaction_debit, - - SUM(CASE - WHEN transaction_type IN ('TOP_UP', 'CREDIT_NOTE') - THEN base_value - ELSE 0 - END) AS total_daily_transaction_credit, + COALESCE(status.name, transactions.status::string) AS status, - COUNT(CASE - WHEN transaction_type IN ('TOP_UP', 'CREDIT_NOTE') - THEN wallet_id - END) AS count_total_daily_transaction_credit, + transactions.deleted_datetime, + transactions.created_datetime, + transactions.updated_datetime, - COUNT(CASE - WHEN transaction_type IN ('PAYMENT', 'DEBIT_NOTE') - THEN wallet_id - END) AS count_total_daily_transaction_debit - - FROM wallet_transaction_logs + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - GROUP BY - wallet_id, - company_id, - base_currency_id, - quote_currency_id, - transaction_date - -), + FROM transactions --- wallet balances -wallet_balance_list AS ( + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) - SELECT - wallet_log_id, - wallet_id, - company_id, - currency_id, - wallet_balance, - created_datetime AS wallet_registration_datetime, - updated_datetime - FROM wallet_balance_logs - - UNION - - SELECT - MD5_NUMBER_LOWER64(CONCAT(wallet_id, wallet_balance, updated_datetime)) AS wallet_log_id, - wallet_id, - company_id, - currency_id, - wallet_balance, - created_datetime AS wallet_registration_datetime, - updated_datetime - FROM latest_wallet_balances + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) -), + LEFT JOIN status + ON (transactions.status = status.id) -wallet_balance_list_enrich AS ( + WHERE + transactions.owner_type = 'App\\Models\\Wallet' - SELECT - wallet_log_id, - wallet_id, - company_id, - currency_id, - wallet_balance, - wallet_registration_datetime, - updated_datetime, - DATE(updated_datetime) AS updated_date, - LAG(wallet_balance) OVER (PARTITION BY wallet_id ORDER BY updated_datetime) AS prev_wallet_balance, - LEAD(wallet_balance) OVER (PARTITION BY wallet_id ORDER BY updated_datetime) AS next_wallet_balance - - FROM wallet_balance_list - -), - -wallet_balance_daily_aggregate AS ( - - SELECT - DISTINCT - wallet_id, - company_id, - currency_id, - wallet_registration_datetime, - updated_date, - COUNT(wallet_id) OVER (PARTITION BY wallet_id, updated_date) AS count_daily_wallet_activity, - FIRST_VALUE(wallet_log_id) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS first_wallet_log_id, - LAST_VALUE(wallet_log_id) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS last_wallet_log_id, - - FIRST_VALUE(wallet_balance) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS daily_first_wallet_balance, - LAST_VALUE(wallet_balance) OVER (PARTITION BY wallet_id, updated_date ORDER BY updated_datetime) AS daily_last_wallet_balance, - - FROM wallet_balance_list_enrich - -), - -wallet_balance_list_join_wallet_transaction AS ( - - SELECT - COALESCE(wallet_balance_daily_aggregate.updated_date, daily_transaction_log.transaction_date) AS transaction_date, - COALESCE(wallet_balance_daily_aggregate.wallet_id, daily_transaction_log.wallet_id) AS wallet_id, - COALESCE(wallet_balance_daily_aggregate.company_id, daily_transaction_log.company_id) AS company_id, - COALESCE(wallet_balance_daily_aggregate.currency_id, daily_transaction_log.base_currency_id) AS currency_id, - - -- Wallet data - COALESCE(wallet_balance_daily_aggregate.count_daily_wallet_activity, 0) AS count_daily_wallet_activity, - COALESCE(wallet_balance_daily_aggregate.first_wallet_log_id, NULL) AS first_wallet_log_id, - COALESCE(wallet_balance_daily_aggregate.last_wallet_log_id, NULL) AS last_wallet_log_id, - COALESCE(wallet_balance_daily_aggregate.daily_first_wallet_balance, NULL) AS daily_first_wallet_balance, - COALESCE(wallet_balance_daily_aggregate.daily_last_wallet_balance, NULL) AS daily_last_wallet_balance, - COALESCE(wallet_balance_daily_aggregate.wallet_registration_datetime, NULL) AS wallet_registration_datetime, - - -- Transaction data - COALESCE(daily_transaction_log.first_daily_transaction_start_datetime, NULL) AS first_daily_transaction_start_datetime, - COALESCE(daily_transaction_log.last_daily_transaction_start_datetime, NULL) AS last_daily_transaction_start_datetime, - COALESCE(daily_transaction_log.first_daily_transaction_completed_datetime, NULL) AS first_daily_transaction_completed_datetime, - COALESCE(daily_transaction_log.last_daily_transaction_completed_datetime, NULL) AS last_daily_transaction_completed_datetime, - COALESCE(daily_transaction_log.total_daily_transaction_debit, NULL) AS total_daily_transaction_debit, - COALESCE(daily_transaction_log.total_daily_transaction_credit, NULL) AS total_daily_transaction_credit, - COALESCE(daily_transaction_log.count_total_daily_transaction_credit, 0) AS count_total_daily_transaction_credit, - COALESCE(daily_transaction_log.count_total_daily_transaction_debit, 0) AS count_total_daily_transaction_debit, - - FROM wallet_balance_daily_aggregate - - FULL OUTER JOIN daily_transaction_log - ON (wallet_balance_daily_aggregate.wallet_id = daily_transaction_log.wallet_id) - AND (wallet_balance_daily_aggregate.updated_date = daily_transaction_log.transaction_date) - -), - -minor_data_imputation AS ( - - SELECT - wallet_id, - company_id, - currency_id, - first_wallet_log_id, - last_wallet_log_id, - count_daily_wallet_activity, - count_total_daily_transaction_credit, - count_total_daily_transaction_debit, - daily_last_wallet_balance, - - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date DESC) = 1 - THEN 1 - ELSE 0 - END AS is_latest_wallet_balance, - - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY wallet_id ORDER BY transaction_date) = 1 - THEN 1 - ELSE 0 - END AS is_earliest_wallet_balance, - - /* - Initialize for older records without wallet balance. (2 scenario) - 1. Set closing balance as 0 - 2. Set closing balance as difference between credit and debit - */ - CASE - WHEN is_earliest_wallet_balance = 1 - THEN COALESCE(daily_first_wallet_balance, 0) - ELSE daily_first_wallet_balance - END daily_first_wallet_balance, - - CASE - WHEN is_earliest_wallet_balance = 1 - THEN COALESCE(daily_last_wallet_balance, 0) - ELSE daily_last_wallet_balance - END daily_0_origin_last_wallet_balance, - - CASE - WHEN is_earliest_wallet_balance = 1 - THEN COALESCE(daily_last_wallet_balance, total_daily_transaction_credit + total_daily_transaction_debit) - ELSE daily_last_wallet_balance - END daily_calculated_last_wallet_balance, - - - CASE - WHEN wallet_registration_datetime IS NULL - THEN ( - SELECT MIN(wallet_registration_datetime) - FROM wallet_balance_list_join_wallet_transaction t1 - WHERE t1.wallet_id = wallet_balance_list_join_wallet_transaction.wallet_id - ) - END AS new_wallet_registration_datetime, - - total_daily_transaction_credit, - total_daily_transaction_debit, - transaction_date, - LEAD(transaction_date) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS next_transaction_date, - LAG(transaction_date) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_transaction_date, - wallet_registration_datetime, - first_daily_transaction_start_datetime, - first_daily_transaction_completed_datetime, - last_daily_transaction_start_datetime, - last_daily_transaction_completed_datetime - - FROM wallet_balance_list_join_wallet_transaction - -), - -recursive_imputation AS ( - - SELECT - *, - - daily_first_wallet_balance AS new_daily_0_origin_first_wallet_balance, - daily_0_origin_last_wallet_balance AS new_daily_0_origin_last_wallet_balance, - - daily_first_wallet_balance AS new_daily_calculated_first_wallet_balance, - daily_calculated_last_wallet_balance AS new_daily_calculated_last_wallet_balance, - - FROM minor_data_imputation - - WHERE is_earliest_wallet_balance = 1 - - UNION ALL - - SELECT - t1.*, - - t2.new_daily_0_origin_last_wallet_balance AS new_daily_0_origin_first_wallet_balance, - t2.new_daily_0_origin_last_wallet_balance + t1.total_daily_transaction_credit + t1.total_daily_transaction_debit AS new_daily_0_origin_last_wallet_balance, - - t2.new_daily_calculated_last_wallet_balance AS new_daily_calculated_first_wallet_balance, - t2.new_daily_calculated_last_wallet_balance + t1.total_daily_transaction_credit + t1.total_daily_transaction_debit AS new_daily_calculated_last_wallet_balance - - - FROM minor_data_imputation t1 - - INNER JOIN recursive_imputation t2 - - ON ( - t1.wallet_id = t2.wallet_id - AND - t2.transaction_date = t1.prev_transaction_date - ) - -), - -casting_and_organize AS ( - - SELECT - wallet_id, - company_id, - currency_id, - first_wallet_log_id::STRING AS first_wallet_log_id, - last_wallet_log_id::STRING AS last_wallet_log_id, - is_earliest_wallet_balance - is_latest_wallet_balance, - count_daily_wallet_activity, - count_total_daily_transaction_credit, - count_total_daily_transaction_debit, - total_daily_transaction_credit, - total_daily_transaction_debit, - LAG(daily_last_wallet_balance) OVER (PARTITION BY wallet_id ORDER BY transaction_date) AS prev_daily_last_wallet_balance, - - CASE - WHEN prev_daily_last_wallet_balance IS NULL - THEN NULL - WHEN daily_first_wallet_balance != prev_daily_last_wallet_balance - THEN prev_daily_last_wallet_balance - ELSE daily_first_wallet_balance - END AS original_daily_first_wallet_balance, - - daily_last_wallet_balance AS original_daily_last_wallet_balance, - - new_daily_0_origin_first_wallet_balance AS daily_0_origin_first_wallet_balance, - new_daily_0_origin_last_wallet_balance AS daily_0_origin_last_wallet_balance, - - new_daily_calculated_first_wallet_balance AS daily_calculated_first_wallet_balance, - new_daily_calculated_last_wallet_balance AS daily_calculated_last_wallet_balance, - - prev_transaction_date, - transaction_date, - next_transaction_date, - first_daily_transaction_start_datetime, - first_daily_transaction_completed_datetime, - last_daily_transaction_start_datetime, - last_daily_transaction_completed_datetime, - - COALESCE(wallet_registration_datetime, new_wallet_registration_datetime) AS wallet_registration_datetime, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM recursive_imputation - ), -- FINAL -final__fct_exchange__daily_wallet_transactions AS ( +final__stg_exchange__transaction_wallets AS ( SELECT - --ids + -- ids + transaction_wallet_id, wallet_id, company_id, - currency_id, - first_wallet_log_id, - last_wallet_log_id, - - --dimensions - is_latest_wallet_balance, + base_currency_id, + quote_currency_id, - --measures - count_daily_wallet_activity, - count_total_daily_transaction_credit, - count_total_daily_transaction_debit, - total_daily_transaction_credit, - total_daily_transaction_debit, - original_daily_first_wallet_balance, - original_daily_last_wallet_balance, - daily_0_origin_first_wallet_balance, - daily_0_origin_last_wallet_balance, - daily_calculated_first_wallet_balance, - daily_calculated_last_wallet_balance, - - --date/times - transaction_date, - first_daily_transaction_start_datetime, - first_daily_transaction_completed_datetime, - last_daily_transaction_start_datetime, - last_daily_transaction_completed_datetime, - wallet_registration_datetime, + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, - --metadata + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata _dbt_ran_datetime - FROM casting_and_organize + FROM transactions_join_transaction_types_payment_methods_currencies_status ) -SELECT * FROM final__fct_exchange__daily_wallet_transactions \ No newline at end of file +SELECT * FROM final__stg_exchange__transaction_wallets \ No newline at end of file From 5fabf779d9a494a39238f117b2f582903d6fcc9c Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Mon, 1 Apr 2024 09:58:59 +0000 Subject: [PATCH 6/9] Changed comparison competitor --- .../rep_exchange__exchange_rate_analysis.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql b/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql index 34ae90d..3b87142 100644 --- a/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql +++ b/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql @@ -59,7 +59,7 @@ restructure_and_expand_table_cief_vs_world_rate AS ( ), -restructure_and_expand_table_cief_1688_vs_c2m_altr AS ( +restructure_and_expand_table_cief_1688_vs_c2m_tbdf AS ( SELECT t1.provider, @@ -83,7 +83,7 @@ restructure_and_expand_table_cief_1688_vs_c2m_altr AS ( LEFT JOIN currency_exchange_rate_compilation t2 ON ( t1.exchange_rate_date = t2.exchange_rate_date ) AND t2.provider = 'C2M' - AND t2.service_type = 'ALTR' + AND t2.service_type = 'TBDF' WHERE t1.provider = 'CIEF' @@ -94,7 +94,7 @@ restructure_and_expand_table_cief_1688_vs_c2m_altr AS ( ), -restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf AS ( +restructure_and_expand_table_cief_1_day_transfer_vs_c2m_altr AS ( SELECT t1.provider, @@ -118,7 +118,7 @@ restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf AS ( LEFT JOIN currency_exchange_rate_compilation t2 ON ( t1.exchange_rate_date = t2.exchange_rate_date ) AND t2.provider = 'C2M' - AND t2.service_type = 'TBDF' + AND t2.service_type = 'ALTR' CROSS JOIN get_base_myr_value_for_comparison @@ -171,9 +171,9 @@ union_expanded_tables AS ( SELECT * FROM restructure_and_expand_table_cief_vs_world_rate UNION ALL - SELECT * FROM restructure_and_expand_table_cief_1688_vs_c2m_altr + SELECT * FROM restructure_and_expand_table_cief_1688_vs_c2m_tbdf UNION ALL - SELECT * FROM restructure_and_expand_table_cief_1_day_transfer_vs_c2m_tbdf + SELECT * FROM restructure_and_expand_table_cief_1_day_transfer_vs_c2m_altr UNION ALL SELECT * FROM restructure_and_expand_table_cief_3_days_vs_sd From 75910a77fe41c374fe3454bb3c4d3a4300084a8e Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Wed, 3 Apr 2024 06:10:11 +0000 Subject: [PATCH 7/9] Added metrics --- .../rep_exchange__exchange_rate_analysis.sql | 80 ++++++++++++++----- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql b/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql index 3b87142..e7e5f0a 100644 --- a/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql +++ b/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql @@ -179,13 +179,15 @@ union_expanded_tables AS ( ), -rate_comparison AS ( +exchange_amount_compute AS ( SELECT *, + -- CIEF exchanged CNY after deducting service charge ( (comparison_base_value_myr / 1.02) * myr_to_cny_currency_exchange_rate ) AS cief_amount_cny, + -- Competitor exchanged CNY after deducting service charge CASE WHEN competitor_name = 'WORLD RATE' THEN comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate @@ -197,6 +199,25 @@ rate_comparison AS ( THEN ( comparison_base_value_myr * competitor_myr_to_cny_currency_exchange_rate ) - handling_fee_cny END AS competitor_amount_cny, + -- Median and Average difference in CNY amount exchanged comparing CIEF to competitor + MEDIAN(1 - DIV0(competitor_amount_cny, cief_amount_cny)) OVER ( + PARTITION BY service_type, exchange_rate_date, competitor_name, competitor_service_type, competitor_payment_method + ) AS median_amount_percent_difference, + + median_amount_percent_difference - 0.5 * median_amount_percent_difference AS upper_bound_median_amount_percent_difference, + median_amount_percent_difference + 0.5 * median_amount_percent_difference AS lower_bound_median_amount_percent_difference, + + AVG(1 - DIV0(competitor_amount_cny, cief_amount_cny)) OVER ( + PARTITION BY service_type, exchange_rate_date, competitor_name, competitor_service_type, competitor_payment_method + ) AS average_amount_percent_difference, + + average_amount_percent_difference - 0.5 * average_amount_percent_difference AS upper_bound_average_amount_percent_difference, + average_amount_percent_difference + 0.5 * average_amount_percent_difference AS lower_bound_average_amount_percent_difference, + + AVG(1 - DIV0(COMPETITOR_AMOUNT_CNY, CIEF_AMOUNT_CNY)) OVER ( + PARTITION BY service_type, exchange_rate_date, payment_method, competitor_name, competitor_service_type, competitor_payment_method + ) AS comparison_average_percent_difference, + MAX(exchange_rate_date) OVER ( PARTITION BY competitor_name, payment_method, competitor_payment_method) AS latest_exchange_rate_date FROM union_expanded_tables @@ -227,23 +248,31 @@ order_metrics AS ( exchange_rate_join_order_metrics AS ( SELECT - rate_comparison.provider, - rate_comparison.payment_method, - rate_comparison.service_type, - rate_comparison.myr_to_cny_currency_exchange_rate, - rate_comparison.exchange_rate_date, - rate_comparison.competitor_name, - rate_comparison.start_range_myr, - rate_comparison.end_range_myr, - rate_comparison.competitor_payment_method, - rate_comparison.competitor_service_type, - rate_comparison.handling_fee_myr, - rate_comparison.handling_fee_cny, - rate_comparison.competitor_myr_to_cny_currency_exchange_rate, - rate_comparison.comparison_base_value_myr, - rate_comparison.cief_amount_cny, - rate_comparison.competitor_amount_cny, - rate_comparison.latest_exchange_rate_date, + exchange_amount_compute.provider, + exchange_amount_compute.payment_method, + exchange_amount_compute.service_type, + exchange_amount_compute.myr_to_cny_currency_exchange_rate, + exchange_amount_compute.exchange_rate_date, + exchange_amount_compute.competitor_name, + exchange_amount_compute.start_range_myr, + exchange_amount_compute.end_range_myr, + exchange_amount_compute.competitor_payment_method, + exchange_amount_compute.competitor_service_type, + exchange_amount_compute.handling_fee_myr, + exchange_amount_compute.handling_fee_cny, + exchange_amount_compute.competitor_myr_to_cny_currency_exchange_rate, + exchange_amount_compute.comparison_base_value_myr, + exchange_amount_compute.cief_amount_cny, + exchange_amount_compute.competitor_amount_cny, + exchange_amount_compute.latest_exchange_rate_date, + exchange_amount_compute.median_amount_percent_difference, + exchange_amount_compute.upper_bound_median_amount_percent_difference, + exchange_amount_compute.lower_bound_median_amount_percent_difference, + exchange_amount_compute.average_amount_percent_difference, + exchange_amount_compute.upper_bound_average_amount_percent_difference, + exchange_amount_compute.lower_bound_average_amount_percent_difference, + exchange_amount_compute.comparison_average_percent_difference, + order_metrics.daily_sales_cny, order_metrics.total_daily_sales_cny, order_metrics.daily_sales_myr, @@ -253,14 +282,14 @@ exchange_rate_join_order_metrics AS ( '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - FROM rate_comparison + FROM exchange_amount_compute LEFT JOIN order_metrics ON ( - rate_comparison.exchange_rate_date = order_metrics.order_date - AND rate_comparison.service_type = order_metrics.service_type - AND rate_comparison.payment_method = order_metrics.order_payment_method + exchange_amount_compute.exchange_rate_date = order_metrics.order_date + AND exchange_amount_compute.service_type = order_metrics.service_type + AND exchange_amount_compute.payment_method = order_metrics.order_payment_method ) ), @@ -296,6 +325,13 @@ final__rep_exchange__exchange_rate_analysis AS ( total_daily_sales_cny, daily_sales_myr, total_daily_sales_myr, + median_amount_percent_difference, + upper_bound_median_amount_percent_difference, + lower_bound_median_amount_percent_difference, + average_amount_percent_difference, + upper_bound_average_amount_percent_difference, + lower_bound_average_amount_percent_difference, + comparison_average_percent_difference, -- date/times exchange_rate_date, From 6cbc293e2561e71f7c552110822da93b0a2bdbb1 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Fri, 5 Apr 2024 03:14:14 +0000 Subject: [PATCH 8/9] Accommodate new service type and update on-time logic --- models/marts/reporting/rep_exchange__daily_orders.sql | 6 ++++-- models/marts/warehouse/fct_exchange__transaction_orders.sql | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql index 9ffc8fc..d7a5a8e 100644 --- a/models/marts/reporting/rep_exchange__daily_orders.sql +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -209,8 +209,9 @@ fct_and_dim_joins AS ( -- On time delivery boolean (CIEF internal SLA) CASE -- On-time boolean is null when order is not complete - WHEN service_type IN ('BA') THEN NULL + WHEN service_type IN ('BA', 'RECEIVED-ON-BEHALF') THEN NULL WHEN transaction_status NOT IN ('UPLOADED_BANK_SLIP') THEN NULL + WHEN order_created_datetime < '2023-01-01' THEN NULL WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1 WHEN estimated_order_delivery_datetime_website_sla < orders.operation_uploaded_bank_slip_datetime THEN 0 ELSE -999 @@ -219,8 +220,9 @@ fct_and_dim_joins AS ( -- On time delivery boolean (Customer Expectation) CASE -- On-time boolean is null when order is not complete - WHEN service_type IN ('BA') THEN NULL + WHEN service_type IN ('BA', 'RECEIVED-ON-BEHALF') THEN NULL WHEN transaction_status NOT IN ('UPLOADED_BANK_SLIP') THEN NULL + WHEN order_created_datetime < '2023-01-01' THEN NULL WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1 WHEN estimated_order_delivery_datetime_customer_expectation < orders.operation_uploaded_bank_slip_datetime THEN 0 ELSE -999 diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql index 1a15160..d954e12 100644 --- a/models/marts/warehouse/fct_exchange__transaction_orders.sql +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -536,6 +536,8 @@ purchase_order_invoice_status_redefined AS ( CASE WHEN order_status IN ('REFUNDED') THEN 'ORDER_REFUNDED' + WHEN order_status IN ('EXPIRED') + THEN 'ORDER_EXPIRED' WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IS NULL THEN 'PENDING_GENERATE_WHITE_FORM' WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IN ('PENDING_VERIFICATION') From ff552ebd0d6b2133019eca223f060e98be68a6ae Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Mon, 13 May 2024 02:21:08 +0000 Subject: [PATCH 9/9] renamed to new service type --- .../reporting/rep_exchange__daily_orders.sql | 4 ++-- .../rep_exchange__exchange_rate_analysis.sql | 14 ++++++------- .../fct_exchange__transaction_orders.sql | 20 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql index d7a5a8e..99d1e6e 100644 --- a/models/marts/reporting/rep_exchange__daily_orders.sql +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -6,9 +6,9 @@ -- working days mapping for each service type {% set service_type_working_day_mapping = { - '1 DAY TRANSFER': 1, + 'BANK TRANSFER (EXPRESS)': 1, '1688 PAYMENT': 1, - '3 DAYS TRANSFER': 3, + 'BANK TRANSFER (SAVER)': 3, 'ENTERPRISE TO ENTERPRISE 公打公': 7 } %} diff --git a/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql b/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql index e7e5f0a..03fb93d 100644 --- a/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql +++ b/models/marts/reporting/rep_exchange__exchange_rate_analysis.sql @@ -5,7 +5,7 @@ WITH currency_exchange_rate_compilation AS ( orders AS ( SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} - WHERE service_type NOT IN ('BA') + WHERE service_type NOT IN ('BA', 'RECEIVED-ON-BEHALF') ), @@ -94,7 +94,7 @@ restructure_and_expand_table_cief_1688_vs_c2m_tbdf AS ( ), -restructure_and_expand_table_cief_1_day_transfer_vs_c2m_altr AS ( +restructure_and_expand_table_cief_bank_transfer_express_vs_c2m_altr AS ( SELECT t1.provider, @@ -127,11 +127,11 @@ restructure_and_expand_table_cief_1_day_transfer_vs_c2m_altr AS ( AND competitor_name IN ('C2M') AND - t1.service_type IN ('1 DAY TRANSFER') + t1.service_type IN ('BANK TRANSFER (EXPRESS)') ), -restructure_and_expand_table_cief_3_days_vs_sd AS ( +restructure_and_expand_table_cief_bank_transfer_saver_vs_sd AS ( SELECT t1.provider, @@ -163,7 +163,7 @@ restructure_and_expand_table_cief_3_days_vs_sd AS ( AND competitor_name IN ('SD') AND - t1.service_type IN ('3 DAYS TRANSFER') + t1.service_type IN ('BANK TRANSFER (SAVER)') ), @@ -173,9 +173,9 @@ union_expanded_tables AS ( UNION ALL SELECT * FROM restructure_and_expand_table_cief_1688_vs_c2m_tbdf UNION ALL - SELECT * FROM restructure_and_expand_table_cief_1_day_transfer_vs_c2m_altr + SELECT * FROM restructure_and_expand_table_cief_bank_transfer_express_vs_c2m_altr UNION ALL - SELECT * FROM restructure_and_expand_table_cief_3_days_vs_sd + SELECT * FROM restructure_and_expand_table_cief_bank_transfer_saver_vs_sd ), diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql index d954e12..4320a17 100644 --- a/models/marts/warehouse/fct_exchange__transaction_orders.sql +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -159,35 +159,35 @@ transaction_order_datetime_imputation AS ( THEN COALESCE(order_completed_datetime, DATEADD(minute, 433, new_order_approved_datetime)) /* - Imputation for order_approved_datetime for 1 day transfer + Imputation for order_approved_datetime for BANK TRANSFER (EXPRESS) - If payment_method is 'cash', add median of 59 minute to order_approved datetime - If payment_method is 'wallet', add median of 206 minute to order_approved datetime - If payment_method is 'cheque', reuse order_completed datetime, all orders have completed datetime - If payment_method is 'payment_gateway', add median of 214.5 minute to order_approved datetime */ - WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'CASH' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (EXPRESS)' AND payment_method = 'CASH' THEN COALESCE(order_completed_datetime, DATEADD(minute, 59, new_order_approved_datetime)) - WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'WALLET' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (EXPRESS)' AND payment_method = 'WALLET' THEN COALESCE(order_completed_datetime, DATEADD(minute, 206, new_order_approved_datetime)) - WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'CHEQUE' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (EXPRESS)' AND payment_method = 'CHEQUE' THEN order_completed_datetime - WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'PAYMENT_GATEWAY' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (EXPRESS)' AND payment_method = 'PAYMENT_GATEWAY' THEN COALESCE(order_completed_datetime, DATEADD(minute, 214.5, new_order_approved_datetime)) /* - Imputation for order_approved_datetime for 3 days transfer + Imputation for order_approved_datetime for BANK TRANSFER (SAVER) - If payment_method is 'cash', add median of 2702 minute to order_approved datetime - If payment_method is 'wallet', add median of 4502.92 minute to order_approved datetime - If payment_method is 'cheque', reuse order_completed datetime, all orders have completed datetime - If payment_method is 'payment_gateway', add median of 5253 minute to order_approved datetime */ - WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'CASH' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (SAVER)' AND payment_method = 'CASH' THEN COALESCE(order_completed_datetime, DATEADD(minute, 2702, new_order_approved_datetime)) - WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'WALLET' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (SAVER)' AND payment_method = 'WALLET' THEN COALESCE(order_completed_datetime, DATEADD(minute, 4502.92, new_order_approved_datetime)) - WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'CHEQUE' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (SAVER)' AND payment_method = 'CHEQUE' THEN order_completed_datetime - WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'PAYMENT_GATEWAY' + WHEN order_created_datetime >= '2023-01-01' AND service_type = 'BANK TRANSFER (SAVER)' AND payment_method = 'PAYMENT_GATEWAY' THEN COALESCE(order_completed_datetime, DATEADD(minute, 5253, new_order_approved_datetime)) END AS new_order_completed_datetime