From 2bee6b91ef14409010e769c63238493c457b25cc Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Thu, 7 Mar 2024 02:04:48 +0000 Subject: [PATCH 1/2] Redefined on-time logic --- .../reporting/rep_exchange__daily_orders.sql | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql index 7e8ce9d..9ffc8fc 100644 --- a/models/marts/reporting/rep_exchange__daily_orders.sql +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -27,7 +27,7 @@ -- IMPORT WITH orders AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} ), companies AS ( @@ -190,10 +190,16 @@ fct_and_dim_joins AS ( CASE {% for (service_type, working_day) in service_type_work_day_dict.items() %} - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + WHEN + orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' + AND + DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + WHEN + orders.order_created_datetime >= '2023-01-01' + AND + orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) {% endfor %} @@ -203,19 +209,21 @@ fct_and_dim_joins AS ( -- On time delivery boolean (CIEF internal SLA) CASE -- On-time boolean is null when order is not complete - WHEN account_verified_payment_datetime IS NULL THEN NULL - WHEN estimated_order_delivery_datetime_website_sla IS NULL THEN NULL + WHEN service_type IN ('BA') THEN NULL + WHEN transaction_status NOT IN ('UPLOADED_BANK_SLIP') THEN NULL WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1 - ELSE 0 + WHEN estimated_order_delivery_datetime_website_sla < orders.operation_uploaded_bank_slip_datetime THEN 0 + ELSE -999 END AS is_on_time_delivery_website_sla, -- On time delivery boolean (Customer Expectation) CASE -- On-time boolean is null when order is not complete - WHEN account_verified_payment_datetime IS NULL THEN NULL - WHEN estimated_order_delivery_datetime_customer_expectation IS NULL THEN NULL + WHEN service_type IN ('BA') THEN NULL + WHEN transaction_status NOT IN ('UPLOADED_BANK_SLIP') THEN NULL WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1 - ELSE 0 + WHEN estimated_order_delivery_datetime_customer_expectation < orders.operation_uploaded_bank_slip_datetime THEN 0 + ELSE -999 END AS is_on_time_delivery_customer_expectation, '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime @@ -242,6 +250,7 @@ fct_and_dim_joins AS ( ), + --FINAL final__rep_exchange__daily_orders AS ( From 715f51ee35032d72834981e033313c1ab9ac7b93 Mon Sep 17 00:00:00 2001 From: CIEF ACC1 Date: Thu, 7 Mar 2024 02:05:22 +0000 Subject: [PATCH 2/2] Total order/cost value formula redefined --- ..._transaction_cost_with_latest_cost_ids.sql | 6 +- ...hange__transaction_orders_get_user_ids.sql | 6 ++ .../warehouse/dim_exchange__companies.sql | 29 ++++-- .../fct_exchange__transaction_costs.sql | 16 ++- .../fct_exchange__transaction_orders.sql | 98 ++++++++++++------- .../stg_exchange__transaction_cost_logs.sql | 26 ++--- .../stg_exchange__transaction_costs.sql | 36 +++---- .../stg_exchange__transaction_invoices.sql | 9 +- .../stg_exchange__transaction_order_logs.sql | 24 ++--- .../stg_exchange__transaction_orders.sql | 45 ++++----- 10 files changed, 175 insertions(+), 120 deletions(-) diff --git a/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql b/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql index d1eccd1..aa2d61a 100644 --- a/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql +++ b/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql @@ -30,7 +30,7 @@ temp_orders AS ( -- LOGIC /* Some order may have multiple same cost status. This is due to the accounting department adjusting the exchange rate. -This is particularly true for the 1688 payment, as exchange rate is amended at a later stage instead of during the initial order placement stage. +This is particularly true for the 1688 payment, as the cost rate is amended at a later stage instead of during the initial order placement stage. Thus, it is to be assumed that the datetime from first repeat occurence as the true datetime. */ transaction_cost_status_datetime AS ( @@ -71,6 +71,7 @@ join_cost_and_logs AS ( transaction_costs.base_to_quote_currency_exchange_rate, transaction_costs.base_tax, transaction_costs.base_service_charge, + transaction_costs.total_base_value, transaction_costs.expired_datetime, transaction_costs.deleted_datetime, transaction_costs.created_datetime, @@ -100,6 +101,7 @@ join_cost_and_logs AS ( QUALIFY row_number_index = 1 + ), remove_deleted_expired_costs AS ( @@ -157,6 +159,7 @@ temp_cte_1688_cleaning AS ( remove_system_error_duplicate_row.quote_value, remove_system_error_duplicate_row.base_to_quote_currency_exchange_rate, remove_system_error_duplicate_row.base_tax, + remove_system_error_duplicate_row.total_base_value, remove_system_error_duplicate_row.expired_datetime, remove_system_error_duplicate_row.deleted_datetime, remove_system_error_duplicate_row.created_datetime, @@ -231,6 +234,7 @@ final__int_exchange__transaction_cost_get_latest_cost_ids AS ( base_to_quote_currency_exchange_rate, base_tax, base_service_charge, + total_base_value, -- date/times expired_datetime, diff --git a/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql b/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql index 8e99ad5..e105771 100644 --- a/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql +++ b/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql @@ -7,8 +7,10 @@ bookings AS ( SELECT * FROM {{ ref('stg_exchange__bookings') }} ), + -- LOGICS transaction_orders_join_bookings AS ( + SELECT transaction_orders.*, bookings.user_id @@ -17,11 +19,13 @@ transaction_orders_join_bookings AS ( LEFT JOIN bookings ON (transaction_orders.booking_id = bookings.booking_id) + ), -- FINAL final__int_exchange__transaction_orders_get_user_ids AS ( + SELECT -- ids transaction_order_id, @@ -45,6 +49,7 @@ final__int_exchange__transaction_orders_get_user_ids AS ( base_to_quote_currency_exchange_rate, base_tax, base_service_charge, + total_base_value, -- date/times expired_datetime, @@ -56,6 +61,7 @@ final__int_exchange__transaction_orders_get_user_ids AS ( '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM transaction_orders_join_bookings + ) SELECT * FROM final__int_exchange__transaction_orders_get_user_ids \ No newline at end of file diff --git a/models/marts/warehouse/dim_exchange__companies.sql b/models/marts/warehouse/dim_exchange__companies.sql index dd0028c..f9a9748 100644 --- a/models/marts/warehouse/dim_exchange__companies.sql +++ b/models/marts/warehouse/dim_exchange__companies.sql @@ -38,6 +38,7 @@ transaction_orders AS ( -- LOGIC count_company_user AS ( + SELECT company_id, COUNT(user_id) AS count_user @@ -45,22 +46,26 @@ count_company_user AS ( FROM companies_brg_users GROUP BY company_id + ), company_billing_addresses AS ( + SELECT * FROM company_addresses WHERE - is_billing_address = '1' + is_billing_address = 1 + ), latest_company_upload_identity_documents AS ( + SELECT *, - row_number() OVER + ROW_NUMBER() OVER (PARTITION BY company_id ORDER BY created_datetime DESC) AS latest_row_number @@ -69,14 +74,15 @@ latest_company_upload_identity_documents AS ( WHERE document_type IN ('IDENTITY_CARD','SSM_REGISTRATION') AND - status = 'APPROVED' + status IN ('APPROVED') QUALIFY - latest_row_number = '1' + latest_row_number = 1 ), companies_rename_and_join_table AS ( + SELECT companies.company_id, companies.name, @@ -161,19 +167,23 @@ companies_rename_and_join_table AS ( LEFT JOIN currencies ON (company_wallets.currency_id = currencies.currency_id) + ), lifetime_value AS ( + SELECT companies_rename_and_join_table.*, ROUND( ZEROIFNULL(SUM( - CASE - WHEN transaction_orders.status in ('COMPLETED', 'APPROVED') - THEN ( transaction_orders.base_value + transaction_orders.base_tax + transaction_orders.base_service_charge ) - END)) - ,2) AS company_lifetime_value, + CASE + WHEN transaction_orders.status IN ('COMPLETED', 'APPROVED') + THEN transaction_orders.total_base_value + END + )) + ,2 + ) AS company_lifetime_value, CASE WHEN company_lifetime_value <= {{monetary_list_lifetime[0]}} THEN 1 @@ -256,6 +266,7 @@ lifetime_value AS ( -- FINAL final__dim_exchange__companies AS ( + SELECT -- ids company_id, diff --git a/models/marts/warehouse/fct_exchange__transaction_costs.sql b/models/marts/warehouse/fct_exchange__transaction_costs.sql index e373e74..80e911f 100644 --- a/models/marts/warehouse/fct_exchange__transaction_costs.sql +++ b/models/marts/warehouse/fct_exchange__transaction_costs.sql @@ -11,6 +11,7 @@ transaction_cost_logs AS ( SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} ), + -- LOGICS transaction_cost_status_datetime AS ( @@ -29,6 +30,7 @@ transaction_cost_status_datetime AS ( ), transaction_costs_lists AS ( + SELECT transaction_costs.transaction_cost_id, transaction_costs.transaction_order_id, @@ -46,23 +48,24 @@ transaction_costs_lists AS ( transaction_costs.base_to_quote_currency_exchange_rate, transaction_costs.base_tax, transaction_costs.base_service_charge, + transaction_costs.total_base_value, -- use for future if base currency id other from '1' IFF(transaction_costs.base_currency_id = '1', transaction_costs.base_value, - -9999999999) AS value_rm, + -9999999999) AS value_rm, -- use for future if base currency id other from '1' IFF(transaction_costs.base_currency_id = '1', transaction_costs.base_tax, - -9999999999) AS tax_rm, + -9999999999) AS tax_rm, -- use for future if base currency id other from '1' IFF(transaction_costs.base_currency_id = '1', transaction_costs.base_service_charge, - -9999999999) AS service_charge_rm, + -9999999999) AS service_charge_rm, - (value_rm - service_charge_rm - tax_rm) AS total_value_rm, + (value_rm + service_charge_rm + tax_rm) AS total_value_rm, transaction_costs.expired_datetime, transaction_costs.deleted_datetime, @@ -70,6 +73,7 @@ transaction_costs_lists AS ( transaction_costs.updated_datetime, transaction_cost_status_datetime.cost_created_datetime, + {%- for status in cost_status %} transaction_cost_status_datetime.cost_{{status.lower()}}_datetime, {% endfor %} @@ -80,11 +84,13 @@ transaction_costs_lists AS ( LEFT JOIN transaction_cost_status_datetime ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id) + ), -- FINAL final__fct_exchange__transaction_costs AS ( + SELECT -- ids transaction_cost_id, @@ -110,6 +116,7 @@ final__fct_exchange__transaction_costs AS ( value_rm, tax_rm, service_charge_rm, + total_base_value, total_value_rm, -- date/times @@ -129,6 +136,7 @@ final__fct_exchange__transaction_costs AS ( _dbt_ran_datetime FROM transaction_costs_lists + ) SELECT * FROM final__fct_exchange__transaction_costs \ No newline at end of file diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql index 7115ae6..1a15160 100644 --- a/models/marts/warehouse/fct_exchange__transaction_orders.sql +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -46,16 +46,19 @@ order_logs_join_bookings AS ( SELECT transaction_order_logs.*, bookings.service_type, - ROW_NUMBER() OVER (PARTITION BY transaction_order_id + ROW_NUMBER() OVER ( + PARTITION BY transaction_order_id ORDER BY - transaction_order_logs.updated_datetime DESC, - transaction_order_logs.transaction_order_id DESC, - transaction_order_logs.transaction_order_log_id DESC) AS rank_index + transaction_order_logs.updated_datetime DESC, + transaction_order_logs.transaction_order_id DESC, + transaction_order_logs.transaction_order_log_id DESC + ) AS rank_index FROM transaction_order_logs LEFT JOIN bookings ON (transaction_order_logs.booking_id = bookings.booking_id) + ), transaction_order_order_status_datetime AS ( @@ -69,12 +72,15 @@ transaction_order_order_status_datetime AS ( MIN(created_datetime) AS order_created_datetime, {%- for status in order_status %} + {% if status != 'APPROVED' %} MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS order_{{status.lower()}}_datetime {% else %} MIN(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS order_{{status.lower()}}_datetime {% endif %} + {%- if not loop.last %},{% endif -%} + {% endfor %} FROM order_logs_join_bookings @@ -83,6 +89,7 @@ transaction_order_order_status_datetime AS ( transaction_order_id, payment_method, service_type + ), transaction_order_datetime_imputation AS ( @@ -96,7 +103,8 @@ transaction_order_datetime_imputation AS ( order_created_datetime AS new_order_created_datetime, order_rejected_datetime AS new_order_rejected_datetime, order_suspended_datetime AS new_order_suspended_datetime, - order_expired_datetime AS new_order_expired_datetime, + order_expired_datetime AS new_order_expired_datetime, + order_refunded_datetime AS new_order_refunded_datetime, -- imputation for ORDER_PENDING_SUBMISSION DATETIME COALESCE(order_pending_submission_datetime, order_created_datetime) AS new_order_pending_submission_datetime, @@ -223,44 +231,56 @@ transaction_orders_lists AS ( transaction_orders.bill_number, transaction_orders.base_value, transaction_orders.quote_value, + transaction_orders.total_base_value, + -- use for future if base currency id other from '1' IFF(transaction_orders.base_currency_id = '1', transaction_orders.base_value, - -9999999999) AS value_rm, + -9999999999) AS base_value_rm, transaction_orders.base_tax AS base_tax, + -- use for future if base currency id other from '1' IFF(transaction_orders.base_currency_id = '1', transaction_orders.base_tax, -9999999999) AS tax_rm, transaction_orders.base_service_charge, + -- use for future if base currency id other from '1' IFF(transaction_orders.base_currency_id = '1', transaction_orders.base_service_charge, -9999999999) AS service_charge_rm, - (value_rm - service_charge_rm - tax_rm) AS total_value_rm, + (base_value_rm + service_charge_rm + tax_rm) AS total_base_value_rm, transaction_orders.expired_datetime AS order_expected_expired_datetime, transaction_orders.deleted_datetime AS order_deleted_datetime, transaction_orders.updated_datetime AS order_updated_datetime, {% for column_name in ["company", "user"] %} - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.{{column_name}}_id ORDER BY transaction_orders.created_datetime, transaction_orders.transaction_order_id) = 1 THEN 1 - ELSE 0 - END AS is_first_time_{{column_name}}, + CASE + WHEN + ROW_NUMBER() OVER ( + PARTITION BY transaction_orders.{{column_name}}_id + ORDER BY + transaction_orders.created_datetime, + transaction_orders.transaction_order_id + ) = 1 + THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}, - CASE - WHEN transaction_orders.status = 'COMPLETED' - AND - MIN(IFF(transaction_orders.status = 'COMPLETED', transaction_orders.created_datetime, NULL)) - OVER (PARTITION BY transaction_orders.{{column_name}}_id) - = transaction_orders.created_datetime - THEN 1 - ELSE 0 - END AS is_first_time_{{column_name}}_completed, + CASE + WHEN + transaction_orders.status = 'COMPLETED' + AND + MIN(IFF(transaction_orders.status = 'COMPLETED', transaction_orders.created_datetime, NULL)) OVER ( + PARTITION BY transaction_orders.{{column_name}}_id + ) = transaction_orders.created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}_completed, {% endfor %} transaction_order_datetime_imputation.new_order_created_datetime AS order_created_datetime, @@ -270,9 +290,12 @@ transaction_orders_lists AS ( transaction_order_datetime_imputation.new_order_{{status.lower()}}_datetime AS order_{{status.lower()}}_datetime, {% endfor %} - LEAD(transaction_order_datetime_imputation.new_order_created_datetime) OVER - (PARTITION BY company_id ORDER BY transaction_order_datetime_imputation.new_order_created_datetime, - transaction_order_datetime_imputation.transaction_order_id) AS next_order_created_datetime + LEAD(transaction_order_datetime_imputation.new_order_created_datetime) OVER ( + PARTITION BY company_id + ORDER BY + transaction_order_datetime_imputation.new_order_created_datetime, + transaction_order_datetime_imputation.transaction_order_id + ) AS next_order_created_datetime FROM transaction_orders @@ -318,7 +341,7 @@ transaction_costs_lists AS ( base_service_charge, -9999999999) AS service_charge_rm, - (value_rm - service_charge_rm - tax_rm) AS total_value_rm, + (value_rm + service_charge_rm + tax_rm) AS total_value_rm, expired_datetime, deleted_datetime, @@ -406,15 +429,17 @@ bookings_orders_costs_join AS ( transaction_orders_lists.is_first_time_company_completed AS is_first_time_order_company_completed, transaction_orders_lists.is_first_time_user AS is_first_time_order_user, transaction_orders_lists.is_first_time_user_completed AS is_first_time_order_user_completed, - transaction_orders_lists.base_to_quote_currency_exchange_rate AS order_base_to_quote_currency_exchange_rate , - transaction_orders_lists.base_value AS order_base_value , - transaction_orders_lists.base_service_charge AS order_base_service_charge , - transaction_orders_lists.base_tax AS order_base_tax , - transaction_orders_lists.quote_value AS order_quote_value , - transaction_orders_lists.value_rm AS order_value_rm , - transaction_orders_lists.service_charge_rm AS order_service_charge_rm , - transaction_orders_lists.tax_rm AS order_tax_rm , - transaction_orders_lists.total_value_rm AS total_order_value_rm, + + transaction_orders_lists.base_to_quote_currency_exchange_rate AS order_base_to_quote_currency_exchange_rate, + transaction_orders_lists.base_value AS order_base_value, + transaction_orders_lists.base_service_charge AS order_base_service_charge, + transaction_orders_lists.base_tax AS order_base_tax, + transaction_orders_lists.quote_value AS order_quote_value, + transaction_orders_lists.base_value_rm AS order_value_rm, + transaction_orders_lists.service_charge_rm AS order_service_charge_rm, + transaction_orders_lists.tax_rm AS order_tax_rm, + transaction_orders_lists.total_base_value_rm AS total_order_value_rm, + transaction_orders_lists.order_created_datetime, transaction_orders_lists.order_deleted_datetime, transaction_orders_lists.order_pending_submission_datetime, @@ -422,6 +447,7 @@ bookings_orders_costs_join AS ( transaction_orders_lists.order_approved_datetime, transaction_orders_lists.order_completed_datetime, transaction_orders_lists.order_updated_datetime, + transaction_orders_lists.order_refunded_datetime, transaction_orders_lists.order_rejected_datetime, transaction_orders_lists.order_suspended_datetime, transaction_orders_lists.order_expired_datetime, @@ -476,6 +502,7 @@ purchase_orders_invoices_join AS ( CASE WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.purchase_order_number END AS purchase_order_number, + CASE WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.status END AS purchase_order_status, @@ -483,6 +510,7 @@ purchase_orders_invoices_join AS ( CASE WHEN cost_status IN ('APPROVED') THEN invoices.invoice_number END AS invoice_number, + CASE WHEN cost_status IN ('APPROVED') THEN invoices.status END AS invoice_status @@ -506,7 +534,8 @@ purchase_order_invoice_status_redefined AS ( -- Transaction status tracks the orders and costs CASE - + WHEN order_status IN ('REFUNDED') + THEN 'ORDER_REFUNDED' WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IS NULL THEN 'PENDING_GENERATE_WHITE_FORM' WHEN order_status IN ('COMPLETED', 'APPROVED') AND cost_status IN ('PENDING_VERIFICATION') @@ -661,6 +690,7 @@ final_fct_exchange__new_orders AS ( order_pending_verification_datetime, order_approved_datetime, order_completed_datetime, + order_refunded_datetime, order_rejected_datetime, order_suspended_datetime, order_expired_datetime, diff --git a/models/staging/exchange/stg_exchange__transaction_cost_logs.sql b/models/staging/exchange/stg_exchange__transaction_cost_logs.sql index 952cc9f..dc39c9e 100644 --- a/models/staging/exchange/stg_exchange__transaction_cost_logs.sql +++ b/models/staging/exchange/stg_exchange__transaction_cost_logs.sql @@ -23,32 +23,36 @@ status AS ( -- LOGIC transactions_generate_transaction_log_id AS ( + SELECT MD5_NUMBER_LOWER64(CONCAT(transaction_id, created_datetime)) AS transaction_log_id, * FROM transactions + ), transaction_arch_logs_union_transactions AS ( + SELECT * FROM transactions_generate_transaction_log_id UNION SELECT * FROM transaction_arch_logs + ), union_transaction_rename_column_and_column_value AS ( SELECT - transaction_arch_logs_union_transactions.transaction_log_id AS transaction_cost_log_id, - transaction_arch_logs_union_transactions.transaction_id AS transaction_cost_id, + transaction_arch_logs_union_transactions.transaction_log_id AS transaction_cost_log_id, + transaction_arch_logs_union_transactions.transaction_id AS transaction_cost_id, IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Transaction', transaction_arch_logs_union_transactions.owner_id, null) AS transaction_order_id, - COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type, + COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type, - transaction_arch_logs_union_transactions.issuer_user_id AS supplier_company_id, + transaction_arch_logs_union_transactions.issuer_user_id AS supplier_company_id, transaction_arch_logs_union_transactions.receiver_user_id AS company_id, transaction_arch_logs_union_transactions.recipient_bank_id, @@ -58,23 +62,22 @@ union_transaction_rename_column_and_column_value AS ( transaction_arch_logs_union_transactions.payment_reference, transaction_arch_logs_union_transactions.bill_number, + transaction_arch_logs_union_transactions.base_currency_id, + transaction_arch_logs_union_transactions.quote_currency_id, transaction_arch_logs_union_transactions.base_value, transaction_arch_logs_union_transactions.quote_value, - - transaction_arch_logs_union_transactions.base_currency_id, - - transaction_arch_logs_union_transactions.quote_currency_id, - transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate, transaction_arch_logs_union_transactions.base_tax, transaction_arch_logs_union_transactions.base_service_charge, - transaction_arch_logs_union_transactions.expired_datetime, + ( transaction_arch_logs_union_transactions.base_value + transaction_arch_logs_union_transactions.base_service_charge ) AS total_base_value, COALESCE(status.name, transaction_arch_logs_union_transactions.status::string) AS status, + transaction_arch_logs_union_transactions.expired_datetime, transaction_arch_logs_union_transactions.deleted_datetime, transaction_arch_logs_union_transactions.created_datetime, transaction_arch_logs_union_transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM transaction_arch_logs_union_transactions @@ -87,12 +90,12 @@ union_transaction_rename_column_and_column_value AS ( LEFT JOIN status ON (transaction_arch_logs_union_transactions.status = status.id) - WHERE transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Transaction' AND transaction_arch_logs_union_transactions.transaction_type = '3' --BILL + ), @@ -123,6 +126,7 @@ final__stg_exchange__transaction_log_costs AS ( base_to_quote_currency_exchange_rate, base_tax, base_service_charge, + total_base_value, -- date/times expired_datetime, diff --git a/models/staging/exchange/stg_exchange__transaction_costs.sql b/models/staging/exchange/stg_exchange__transaction_costs.sql index 2b4ff03..e0c4301 100644 --- a/models/staging/exchange/stg_exchange__transaction_costs.sql +++ b/models/staging/exchange/stg_exchange__transaction_costs.sql @@ -21,37 +21,32 @@ status AS ( transactions_join_transaction_types_payment_methods_currencies_status AS ( SELECT - transactions.transaction_id AS transaction_cost_id, - IFF(transactions.owner_type = 'App\\Models\\Transaction', transactions.owner_id, null) AS transaction_order_id, - - COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, - - transactions.issuer_user_id AS supplier_company_id, - + transactions.transaction_id AS transaction_cost_id, + transactions.issuer_user_id AS supplier_company_id, transactions.recipient_bank_id AS bank_id, - - COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + IFF(transactions.owner_type = 'App\\Models\\Transaction', transactions.owner_id, null) AS transaction_order_id, transactions.payment_reference, transactions.bill_number, - - transactions.base_value, - transactions.quote_value, - transactions.base_currency_id, transactions.quote_currency_id, - transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, transactions.base_service_charge, - - transactions.expired_datetime, + transactions.base_value, + transactions.quote_value, + ( transactions.base_value + transactions.base_service_charge ) AS total_base_value, - COALESCE(status.name, transactions.status::string) AS status, - + COALESCE(status.name, transactions.status::string) AS status, + + transactions.expired_datetime, transactions.deleted_datetime, transactions.created_datetime, transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM transactions @@ -65,7 +60,6 @@ transactions_join_transaction_types_payment_methods_currencies_status AS ( LEFT JOIN status ON (transactions.status = status.id) - WHERE transactions.owner_type = 'App\\Models\\Transaction' AND @@ -95,10 +89,11 @@ final__stg_exchange__transaction_costs AS ( -- measures base_value, - quote_value, base_to_quote_currency_exchange_rate, + quote_value, base_tax, base_service_charge, + total_base_value, -- date/times expired_datetime, @@ -110,6 +105,7 @@ final__stg_exchange__transaction_costs AS ( _dbt_ran_datetime FROM transactions_join_transaction_types_payment_methods_currencies_status + ) SELECT * FROM final__stg_exchange__transaction_costs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_invoices.sql b/models/staging/exchange/stg_exchange__transaction_invoices.sql index 092889c..f247317 100644 --- a/models/staging/exchange/stg_exchange__transaction_invoices.sql +++ b/models/staging/exchange/stg_exchange__transaction_invoices.sql @@ -47,11 +47,12 @@ transactions_join_transaction_types_payment_methods_bookings_status AS ( COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, COALESCE(status.name, transactions.status::string) AS status, transactions.bill_number AS invoice_number, - transactions.base_value, + ( transactions.quote_value / transactions.base_to_quote_currency_exchange_rate ) AS base_value, transactions.quote_value, transactions.base_to_quote_currency_exchange_rate, transactions.base_tax, transactions.base_service_charge, + (( transactions.quote_value / transactions.base_to_quote_currency_exchange_rate ) + transactions.base_service_charge ) AS total_base_value, transactions.expired_datetime, transactions.deleted_datetime, @@ -80,6 +81,7 @@ transactions_join_transaction_types_payment_methods_bookings_status AS ( ), remove_deleted_records AS ( + SELECT * FROM transactions_join_transaction_types_payment_methods_bookings_status @@ -88,9 +90,10 @@ remove_deleted_records AS ( ), remove_duplicated_records AS ( + SELECT *, - ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC) AS row_index, + ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC) AS row_index, '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime @@ -98,6 +101,7 @@ remove_duplicated_records AS ( QUALIFY row_index = 1 + ), @@ -128,6 +132,7 @@ final__stg_exchange__invoices AS ( base_to_quote_currency_exchange_rate, base_tax, base_service_charge, + total_base_value, -- date/times deleted_datetime, diff --git a/models/staging/exchange/stg_exchange__transaction_order_logs.sql b/models/staging/exchange/stg_exchange__transaction_order_logs.sql index 618d182..d4824e4 100644 --- a/models/staging/exchange/stg_exchange__transaction_order_logs.sql +++ b/models/staging/exchange/stg_exchange__transaction_order_logs.sql @@ -29,19 +29,17 @@ transactions_generate_transaction_log_id AS ( FROM transactions ), - transaction_arch_logs_union_transactions AS ( SELECT * FROM transactions_generate_transaction_log_id UNION SELECT * FROM transaction_arch_logs ), - union_transaction_rename_column_and_column_value AS ( SELECT - transaction_arch_logs_union_transactions.transaction_log_id AS transaction_order_log_id, - transaction_arch_logs_union_transactions.transaction_id AS transaction_order_id, + transaction_arch_logs_union_transactions.transaction_log_id AS transaction_order_log_id, + transaction_arch_logs_union_transactions.transaction_id AS transaction_order_id, IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Booking', transaction_arch_logs_union_transactions.owner_id, @@ -49,16 +47,15 @@ union_transaction_rename_column_and_column_value AS ( COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type, - transaction_arch_logs_union_transactions.receiver_user_id AS company_id, - + transaction_arch_logs_union_transactions.receiver_user_id AS company_id, transaction_arch_logs_union_transactions.recipient_bank_id, - COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string) AS payment_method, + COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string) AS payment_method, transaction_arch_logs_union_transactions.payment_reference, transaction_arch_logs_union_transactions.bill_number, - transaction_arch_logs_union_transactions.base_value, + ( transaction_arch_logs_union_transactions.base_value - transaction_arch_logs_union_transactions.base_service_charge ) AS base_value, transaction_arch_logs_union_transactions.quote_value, transaction_arch_logs_union_transactions.base_currency_id, @@ -68,14 +65,16 @@ union_transaction_rename_column_and_column_value AS ( transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate, transaction_arch_logs_union_transactions.base_tax, transaction_arch_logs_union_transactions.base_service_charge, - transaction_arch_logs_union_transactions.expired_datetime, + transaction_arch_logs_union_transactions.base_value AS total_base_value, - COALESCE(status.name, transaction_arch_logs_union_transactions.status::string) AS status, + COALESCE(status.name, transaction_arch_logs_union_transactions.status::string) AS status, + transaction_arch_logs_union_transactions.expired_datetime, transaction_arch_logs_union_transactions.deleted_datetime, transaction_arch_logs_union_transactions.created_datetime, transaction_arch_logs_union_transactions.updated_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM transaction_arch_logs_union_transactions @@ -88,11 +87,11 @@ union_transaction_rename_column_and_column_value AS ( LEFT JOIN status ON (transaction_arch_logs_union_transactions.status = status.id) - WHERE transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Booking' AND transaction_arch_logs_union_transactions.transaction_type = '1' --PAYMENT + ), -- FINAL @@ -121,6 +120,7 @@ final__stg_exchange__transaction_order_logs AS ( base_to_quote_currency_exchange_rate, base_tax, base_service_charge, + total_base_value, -- date/times expired_datetime, diff --git a/models/staging/exchange/stg_exchange__transaction_orders.sql b/models/staging/exchange/stg_exchange__transaction_orders.sql index 7f8a529..16ee7f2 100644 --- a/models/staging/exchange/stg_exchange__transaction_orders.sql +++ b/models/staging/exchange/stg_exchange__transaction_orders.sql @@ -1,7 +1,3 @@ --- change name for final__stg_exchange__transaction_costs --- @todo --- @yam - -- IMPORTS WITH transactions AS ( SELECT * FROM {{ ref('base_exchange__transactions') }} @@ -25,37 +21,32 @@ status AS ( transactions_join_transaction_types_payment_methods_status AS ( SELECT - transactions.transaction_id AS transaction_order_id, - IFF(transactions.owner_type = 'App\\Models\\Booking', transactions.owner_id, null) AS booking_id, - - COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, - - transactions.receiver_user_id AS company_id, - + transactions.transaction_id AS transaction_order_id, + transactions.receiver_user_id AS company_id, transactions.recipient_bank_id AS bank_id, - - COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + IFF(transactions.owner_type = 'App\\Models\\Booking', transactions.owner_id, null) AS booking_id, transactions.payment_reference, transactions.bill_number, - - transactions.base_value, - transactions.quote_value, - transactions.base_currency_id, transactions.quote_currency_id, - transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, transactions.base_service_charge, + ( transactions.base_value - transactions.base_service_charge ) AS base_value, -- Tax not implemented in CIEF yet. + transactions.base_value AS total_base_value, + transactions.quote_value, + + COALESCE(status.name, transactions.status::string) AS status, transactions.expired_datetime, - - COALESCE(status.name, transactions.status::string) AS status, - transactions.deleted_datetime, transactions.created_datetime, transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM transactions @@ -69,17 +60,16 @@ transactions_join_transaction_types_payment_methods_status AS ( LEFT JOIN status ON (transactions.status = status.id) - WHERE transactions.owner_type = 'App\\Models\\Booking' - AND + AND transactions.transaction_type = '1' --PAYMENT ), -- FINAL -final__stg_exchange__transaction_costs AS ( +final__stg_exchange__transaction_orders AS ( SELECT -- ids @@ -99,10 +89,11 @@ final__stg_exchange__transaction_costs AS ( -- measures base_value, - quote_value, base_to_quote_currency_exchange_rate, + quote_value, base_tax, base_service_charge, + total_base_value, -- date/times expired_datetime, @@ -115,6 +106,6 @@ final__stg_exchange__transaction_costs AS ( FROM transactions_join_transaction_types_payment_methods_status -) +) -SELECT * FROM final__stg_exchange__transaction_costs \ No newline at end of file +SELECT * FROM final__stg_exchange__transaction_orders \ No newline at end of file