diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql index 4866782..59480dd 100644 --- a/models/marts/warehouse/fct_exchange__transaction_orders.sql +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -1,42 +1,225 @@ --- IMPORTS +-- VARIABLES {% set order_status = dbt_utils.get_column_values( table=ref('stg_exchange__transaction_order_logs'), - column='status') %} + column='status') +%} +{% set booking_status = dbt_utils.get_column_values( + table=ref('stg_exchange__booking_logs'), + column='status') +%} + +-- IMPORTS WITH transaction_orders AS ( - SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} + SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} ), -transaction_order_logs AS ( +transaction_order_logs AS ( SELECT * FROM {{ ref('stg_exchange__transaction_order_logs') }} ), --- LOGICS +transaction_costs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} +), + +transaction_cost_logs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} +), + +bookings AS ( + SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_booking_values') }} +), + +booking_logs AS ( + SELECT * FROM {{ ref('stg_exchange__booking_logs') }} +), + +-- LOGIC +order_logs_join_bookings AS ( + SELECT + transaction_order_logs.*, + bookings.service_type, + ROW_NUMBER() OVER (PARTITION BY transaction_order_id + ORDER BY transaction_order_logs.updated_datetime DESC, + transaction_order_logs.transaction_order_id DESC, + transaction_order_logs.transaction_order_log_id DESC) AS rank_index + + FROM transaction_order_logs + + LEFT JOIN bookings + ON (transaction_order_logs.booking_id = bookings.booking_id) +), + transaction_order_order_status_datetime AS ( SELECT transaction_order_id, - MIN(created_datetime) AS order_created_datetime, + payment_method, + service_type, + MAX(rank_index) AS status_count, + MAX(CASE WHEN rank_index = 1 THEN status END) AS final_order_status, + MIN(created_datetime) AS order_created_datetime, {%- for status in order_status %} - MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS order_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} - {% endfor %} + {% if status != 'APPROVED' %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS order_{{status.lower()}}_datetime + {% else %} + MIN(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS order_{{status.lower()}}_datetime + {% endif %} + {%- if not loop.last %},{% endif -%} + {% endfor %} - FROM transaction_order_logs + FROM order_logs_join_bookings GROUP BY - transaction_order_id + transaction_order_id, + payment_method, + service_type ), +transaction_order_datetime_imputation AS ( + + SELECT + transaction_order_id, + payment_method, + service_type, + final_order_status, + + order_created_datetime AS new_order_created_datetime, + order_rejected_datetime AS new_order_rejected_datetime, + order_suspended_datetime AS new_order_suspended_datetime, + order_expired_datetime AS new_order_expired_datetime, + + -- imputation for ORDER_PENDING_SUBMISSION DATETIME + COALESCE(order_pending_submission_datetime, order_created_datetime) AS new_order_pending_submission_datetime, + + /* + Data manipulation for ORDER_PENDING_VERIFICATION_DATETIME + - To improve data consistency + */ + CASE + -- When final order status is not pending verification, return null + WHEN final_order_status NOT IN ('PENDING_VERIFICATION', 'APPROVED', 'COMPLETED') THEN null + ELSE COALESCE(order_pending_verification_datetime, new_order_pending_submission_datetime) + END AS new_order_pending_verification_datetime, + + + /* + Data manipulation for ORDER_APPROVED_DATETIME + - To improve data consistency + - All orders from year 2023 has order_approved datetime + */ + CASE + -- When final order status is not approved, return null + WHEN order_created_datetime >= '2023-01-01'AND final_order_status NOT IN ('APPROVED', 'COMPLETED') THEN null + ELSE COALESCE(order_approved_datetime, new_order_pending_verification_datetime) + END AS new_order_approved_datetime, + + + /* + Missing value imputation for order_completed_datetime + - Missing value caused by inconsistency and backlog from exchange system + - Imputed dataset used for estimation of on-time delivery + */ + CASE + -- When final order status is not completed, return null, imputation will be done in next code block + WHEN order_created_datetime >= '2023-01-01'AND final_order_status NOT IN ('COMPLETED') THEN null + WHEN order_created_datetime < '2023-01-01' THEN order_completed_datetime + + /* + Imputation for order_approved_datetime for 1688 payment + - If payment_method is 'cash', add median of 53 minute to order_approved datetime + - If payment_method is 'wallet', add median of 569 minute to order_approved datetime + - If payment_method is 'cheque', reuse order_completed datetime, no orders using cheque in 1688 payment + - If payment_method is 'payment_gateway', add median of 433 minute to order_approved datetime + */ + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'CASH' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 53, new_order_approved_datetime)) + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'WALLET' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 569, new_order_approved_datetime)) + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'CHEQUE' + THEN order_completed_datetime + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'PAYMENT_GATEWAY' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 433, new_order_approved_datetime)) + + /* + Imputation for order_approved_datetime for 1 day transfer + - If payment_method is 'cash', add median of 59 minute to order_approved datetime + - If payment_method is 'wallet', add median of 206 minute to order_approved datetime + - If payment_method is 'cheque', reuse order_completed datetime, all orders have completed datetime + - If payment_method is 'payment_gateway', add median of 214.5 minute to order_approved datetime + */ + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'CASH' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 59, new_order_approved_datetime)) + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'WALLET' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 206, new_order_approved_datetime)) + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'CHEQUE' + THEN order_completed_datetime + WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'PAYMENT_GATEWAY' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 214.5, new_order_approved_datetime)) + + /* + Imputation for order_approved_datetime for 3 days transfer + - If payment_method is 'cash', add median of 2702 minute to order_approved datetime + - If payment_method is 'wallet', add median of 4502.92 minute to order_approved datetime + - If payment_method is 'cheque', reuse order_completed datetime, all orders have completed datetime + - If payment_method is 'payment_gateway', add median of 5253 minute to order_approved datetime + */ + WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'CASH' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 2702, new_order_approved_datetime)) + WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'WALLET' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 4502.92, new_order_approved_datetime)) + WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'CHEQUE' + THEN order_completed_datetime + WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'PAYMENT_GATEWAY' + THEN COALESCE(order_completed_datetime, DATEADD(minute, 5253, new_order_approved_datetime)) + + END AS new_order_completed_datetime + + FROM transaction_order_order_status_datetime +), + +transaction_cost_status_datetime AS ( + + SELECT + transaction_cost_id, + MIN(created_datetime) AS cost_created_datetime, + + {%- for status in order_status %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM transaction_cost_logs + + GROUP BY + transaction_cost_id +), + +booking_status_datetime AS ( + + SELECT + booking_id, + MIN(created_datetime) AS booking_created_datetime, + + {%- for status in booking_status %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS booking_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM booking_logs + + GROUP BY + booking_id +), transaction_orders_lists AS ( + SELECT transaction_orders.transaction_order_id, transaction_orders.booking_id, transaction_orders.bank_id, transaction_orders.company_id, transaction_orders.user_id, - transaction_orders.status AS order_status, + transaction_orders.status, transaction_orders.transaction_type, transaction_orders.payment_method, transaction_orders.base_currency_id, @@ -44,37 +227,37 @@ transaction_orders_lists AS ( transaction_orders.base_to_quote_currency_exchange_rate, transaction_orders.payment_reference, transaction_orders.bill_number, - transaction_orders.expired_datetime AS order_expired_datetime, transaction_orders.base_value, transaction_orders.quote_value, -- use for future if base currency id other from '1' IFF(transaction_orders.base_currency_id = '1', transaction_orders.base_value, - -9999999999) AS value_rm, + -9999999999) AS value_rm, - transaction_orders.base_tax AS base_tax, + transaction_orders.base_tax AS base_tax, -- use for future if base currency id other from '1' IFF(transaction_orders.base_currency_id = '1', transaction_orders.base_tax, - -9999999999) AS tax_rm, + -9999999999) AS tax_rm, transaction_orders.base_service_charge, -- use for future if base currency id other from '1' IFF(transaction_orders.base_currency_id = '1', transaction_orders.base_service_charge, - -9999999999) AS service_charge_rm, + -9999999999) AS service_charge_rm, - (value_rm + service_charge_rm + tax_rm) AS total_value_rm, + (value_rm + service_charge_rm + tax_rm) AS total_value_rm, - transaction_orders.deleted_datetime AS order_deleted_datetime, - transaction_orders.updated_datetime AS order_updated_datetime, + transaction_orders.expired_datetime AS order_expected_expired_datetime, + transaction_orders.deleted_datetime AS order_deleted_datetime, + transaction_orders.updated_datetime AS order_updated_datetime, {% for column_name in ["company", "user"] %} CASE - WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.{{column_name}}_id ORDER BY transaction_orders.created_datetime) = 1 THEN 1 + WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.{{column_name}}_id ORDER BY transaction_orders.created_datetime, transaction_orders.transaction_order_id) = 1 THEN 1 ELSE 0 - END AS is_first_time_{{column_name}}, + END AS is_first_time_{{column_name}}, CASE WHEN transaction_orders.status = 'COMPLETED' @@ -84,75 +267,297 @@ transaction_orders_lists AS ( = transaction_orders.created_datetime THEN 1 ELSE 0 - END AS is_first_time_{{column_name}}_completed, + END AS is_first_time_{{column_name}}_completed, {% endfor %} - transaction_order_order_status_datetime.order_created_datetime, + transaction_order_datetime_imputation.new_order_created_datetime AS order_created_datetime, + -- transaction_order_datetime_imputation.order_expired_datetime, {%- for status in order_status %} - transaction_order_order_status_datetime.order_{{status.lower()}}_datetime, + transaction_order_datetime_imputation.new_order_{{status.lower()}}_datetime AS order_{{status.lower()}}_datetime, {% endfor %} - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + LEAD(transaction_order_datetime_imputation.new_order_created_datetime) OVER + (PARTITION BY company_id ORDER BY transaction_order_datetime_imputation.new_order_created_datetime, + transaction_order_datetime_imputation.transaction_order_id) AS next_order_created_datetime FROM transaction_orders - LEFT JOIN transaction_order_order_status_datetime - ON (transaction_orders.transaction_order_id = transaction_order_order_status_datetime.transaction_order_id) + LEFT JOIN transaction_order_datetime_imputation + ON (transaction_orders.transaction_order_id = transaction_order_datetime_imputation.transaction_order_id) +), + +transaction_costs_lists AS ( + + SELECT + transaction_costs.transaction_cost_id, + transaction_costs.transaction_order_id, + transaction_costs.supplier_company_id, + transaction_costs.bank_id, + transaction_costs.base_currency_id, + transaction_costs.quote_currency_id, + transaction_costs.transaction_type, + transaction_costs.payment_method, + transaction_costs.status, + transaction_costs.payment_reference, + transaction_costs.bill_number, + transaction_costs.base_value, + transaction_costs.quote_value, + transaction_costs.base_to_quote_currency_exchange_rate, + transaction_costs.base_tax, + transaction_costs.base_service_charge, + + -- use for future if base currency id other from '1' + IFF(transaction_costs.base_currency_id = '1', + transaction_costs.base_value, + -9999999999) AS value_rm, + + -- use for future if base currency id other from '1' + IFF(transaction_costs.base_currency_id = '1', + transaction_costs.base_tax, + -9999999999) AS tax_rm, + + -- use for future if base currency id other from '1' + IFF(transaction_costs.base_currency_id = '1', + transaction_costs.base_service_charge, + -9999999999) AS service_charge_rm, + + (value_rm + service_charge_rm + tax_rm) AS total_value_rm, + + transaction_costs.expired_datetime, + transaction_costs.deleted_datetime, + transaction_costs.created_datetime, + transaction_costs.updated_datetime, + + {%- for status in order_status %} + transaction_cost_status_datetime.cost_{{status.lower()}}_datetime, + {% endfor %} + transaction_cost_status_datetime.cost_created_datetime + + FROM transaction_costs + + LEFT JOIN transaction_cost_status_datetime + ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id) + +), + +bookings_lists AS ( + SELECT + bookings.*, + + {% for column_name in ["company", "user"] %} + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY bookings.{{column_name}}_id ORDER BY bookings.created_datetime, bookings.booking_id) = 1 THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}, + + CASE + WHEN bookings.status = 'COMPLETED' + AND + MIN(IFF(bookings.status = 'COMPLETED', bookings.created_datetime, NULL)) + OVER (PARTITION BY bookings.{{column_name}}_id) + = bookings.created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}_completed, + {% endfor %} + + booking_status_datetime.booking_created_datetime, + {%- for status in booking_status %} + booking_status_datetime.booking_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM bookings + + LEFT JOIN booking_status_datetime + ON (bookings.booking_id = booking_status_datetime.booking_id) +), + +bookings_orders_costs_join AS ( + + SELECT + bookings_lists.booking_id, + bookings_lists.booking_marking_id, + bookings_lists.user_id, + bookings_lists.fix_currency_id, + bookings_lists.quote_currency_id, + bookings_lists.base_currency_id, + bookings_lists.service_type, + bookings_lists.status AS booking_status, + bookings_lists.is_first_time_company AS is_first_time_booking_company, + bookings_lists.is_first_time_company_completed AS is_first_time_booking_company_completed, + bookings_lists.is_first_time_user AS is_first_time_booking_user, + bookings_lists.is_first_time_user_completed AS is_first_time_booking_user_completed, + bookings_lists.estimate_base_to_quote_currency_exchange_rate AS estimate_booking_base_to_quote_currency_exchange_rate, + bookings_lists.estimate_quote_value AS estimate_booking_quote_value, + bookings_lists.estimate_base_value AS estimate_booking_base_value, + bookings_lists.estimate_value_rm AS estimate_booking_value_rm, + bookings_lists.booking_created_datetime, + bookings_lists.booking_approved_datetime, + bookings_lists.booking_completed_datetime, + bookings_lists.booking_suspended_datetime, + + transaction_orders_lists.transaction_order_id AS order_id, + transaction_orders_lists.company_id, + transaction_orders_lists.bank_id, + transaction_orders_lists.transaction_type AS order_transaction_type, + transaction_orders_lists.payment_method AS order_payment_method, + transaction_orders_lists.status AS order_status, + transaction_orders_lists.payment_reference AS order_payment_reference, + transaction_orders_lists.bill_number AS order_bill_number, + transaction_orders_lists.is_first_time_company AS is_first_time_order_company, + transaction_orders_lists.is_first_time_company_completed AS is_first_time_order_company_completed, + transaction_orders_lists.is_first_time_user AS is_first_time_order_user, + transaction_orders_lists.is_first_time_user_completed AS is_first_time_order_user_completed, + transaction_orders_lists.base_to_quote_currency_exchange_rate AS order_base_to_quote_currency_exchange_rate , + transaction_orders_lists.base_value AS order_base_value , + transaction_orders_lists.base_service_charge AS order_base_service_charge , + transaction_orders_lists.base_tax AS order_base_tax , + transaction_orders_lists.quote_value AS order_quote_value , + transaction_orders_lists.value_rm AS order_value_rm , + transaction_orders_lists.service_charge_rm AS order_service_charge_rm , + transaction_orders_lists.tax_rm AS order_tax_rm , + transaction_orders_lists.total_value_rm AS total_order_value_rm, + transaction_orders_lists.order_created_datetime, + transaction_orders_lists.order_pending_submission_datetime, + transaction_orders_lists.order_pending_verification_datetime, + transaction_orders_lists.order_approved_datetime, + transaction_orders_lists.order_completed_datetime, + transaction_orders_lists.order_rejected_datetime, + transaction_orders_lists.order_suspended_datetime, + transaction_orders_lists.order_expired_datetime, + transaction_orders_lists.next_order_created_datetime, + transaction_orders_lists.order_expected_expired_datetime, + + transaction_costs_lists.transaction_cost_id AS cost_id, + transaction_costs_lists.bank_id AS currency_vendor_bank_id, + transaction_costs_lists.supplier_company_id AS supplier_company_id, + transaction_costs_lists.transaction_type AS cost_transaction_type, + transaction_costs_lists.payment_method AS cost_payment_method, + transaction_costs_lists.status AS cost_status, + transaction_costs_lists.payment_reference AS cost_payment_reference, + transaction_costs_lists.bill_number AS cost_bill_number, + transaction_costs_lists.base_value AS cost_base_value, + transaction_costs_lists.quote_value AS cost_quote_value, + transaction_costs_lists.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate, + transaction_costs_lists.base_tax AS cost_base_tax, + transaction_costs_lists.base_service_charge AS cost_base_service_charge, + transaction_costs_lists.value_rm AS cost_value_rm, + transaction_costs_lists.tax_rm AS cost_tax_rm, + transaction_costs_lists.service_charge_rm AS cost_service_charge_rm, + transaction_costs_lists.total_value_rm AS total_cost_value_rm, + transaction_costs_lists.cost_created_datetime, + transaction_costs_lists.cost_pending_submission_datetime, + transaction_costs_lists.cost_pending_verification_datetime, + transaction_costs_lists.cost_approved_datetime, + transaction_costs_lists.cost_completed_datetime, + transaction_costs_lists.cost_rejected_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_orders_lists + + LEFT JOIN transaction_costs_lists + ON transaction_orders_lists.transaction_order_id = transaction_costs_lists.transaction_order_id + + LEFT JOIN bookings_lists + ON transaction_orders_lists.booking_id = bookings_lists.booking_id + ), -- FINAL -final__fct_exchange__transaction_orders AS ( - SELECT +final_fct_exchange__new_orders AS ( + + SELECT -- ids - transaction_order_id, + order_id, booking_id, + booking_marking_id, company_id, user_id, bank_id, - base_currency_id, + cost_id, + supplier_company_id, + currency_vendor_bank_id, + fix_currency_id, quote_currency_id, + base_currency_id, -- dimensions - transaction_type, - payment_method, + service_type, + booking_status, + is_first_time_booking_company, + is_first_time_booking_company_completed, + is_first_time_booking_user, + is_first_time_booking_user_completed, + is_first_time_order_company, + is_first_time_order_company_completed, + is_first_time_order_user, + is_first_time_order_user_completed, + order_transaction_type, + order_payment_method, order_status, - payment_reference, - bill_number, - is_first_time_company, - is_first_time_company_completed, - is_first_time_user, - is_first_time_user_completed, - + order_payment_reference, + order_bill_number, + cost_transaction_type, + cost_payment_method, + cost_status, + cost_payment_reference, + cost_bill_number, + -- measures - base_to_quote_currency_exchange_rate, - base_value, - base_service_charge, - base_tax, - - quote_value, - - value_rm, - service_charge_rm, - tax_rm, - total_value_rm, + estimate_booking_base_to_quote_currency_exchange_rate, + estimate_booking_quote_value, + estimate_booking_base_value, + estimate_booking_value_rm, + order_base_to_quote_currency_exchange_rate, + order_base_value, + order_base_service_charge, + order_base_tax, + order_quote_value, + order_value_rm, + order_service_charge_rm, + order_tax_rm, + total_order_value_rm, + cost_base_value, + cost_quote_value, + cost_base_to_quote_currency_exchange_rate, + cost_base_tax, + cost_base_service_charge, + cost_value_rm, + cost_tax_rm, + cost_service_charge_rm, + total_cost_value_rm, -- date/times - order_expired_datetime, - order_deleted_datetime, - order_updated_datetime, + booking_created_datetime, + booking_approved_datetime, + booking_completed_datetime, + booking_suspended_datetime, + order_created_datetime, + order_expected_expired_datetime, order_pending_submission_datetime, order_pending_verification_datetime, order_approved_datetime, order_completed_datetime, order_rejected_datetime, order_suspended_datetime, + order_expired_datetime, + + next_order_created_datetime, + cost_created_datetime, + cost_pending_submission_datetime, + cost_pending_verification_datetime, + cost_approved_datetime, + cost_completed_datetime, + cost_rejected_datetime, + -- metadata - _dbt_ran_datetime - - FROM transaction_orders_lists + _dbt_ran_datetime + + FROM bookings_orders_costs_join ) -SELECT * FROM final__fct_exchange__transaction_orders \ No newline at end of file +SELECT * FROM final_fct_exchange__new_orders \ No newline at end of file