dbt_cloud/models/marts/warehouse/fct_exchange__transaction_orders.sql

-- VARIABLES
{% set order_status = dbt_utils.get_column_values(
    table=ref('stg_exchange__transaction_order_logs'),
    column='status') 
%}

{% set booking_status = dbt_utils.get_column_values(
    table=ref('stg_exchange__booking_logs'),
    column='status') 
%}

-- IMPORTS
WITH transaction_orders AS (
    SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} 
),

transaction_order_logs AS ( 
    SELECT * FROM {{ ref('stg_exchange__transaction_order_logs') }}
),

transaction_costs AS (
    SELECT * FROM {{ ref('stg_exchange__transaction_costs') }}
),

transaction_cost_logs AS (
    SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }}
),

bookings AS (
    SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_booking_values') }}
),

booking_logs AS (
    SELECT * FROM {{ ref('stg_exchange__booking_logs') }}
),

-- LOGIC
order_logs_join_bookings AS (
    SELECT
        transaction_order_logs.*,
        bookings.service_type,
        ROW_NUMBER() OVER (PARTITION BY transaction_order_id 
            ORDER BY transaction_order_logs.updated_datetime DESC, 
            transaction_order_logs.transaction_order_id DESC, 
            transaction_order_logs.transaction_order_log_id DESC)   AS rank_index

    FROM transaction_order_logs

    LEFT JOIN bookings
        ON (transaction_order_logs.booking_id = bookings.booking_id)
),

transaction_order_order_status_datetime AS (

    SELECT
        transaction_order_id,
        payment_method,
        service_type,
        MAX(rank_index)                                                             AS status_count,
        MAX(CASE WHEN rank_index = 1 THEN status END)                               AS final_order_status,
        MIN(created_datetime)                                                       AS order_created_datetime,

        {%- for status in order_status %}
            {% if status != 'APPROVED' %}
                MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END)      AS order_{{status.lower()}}_datetime 
            {% else %}
                MIN(CASE WHEN status = '{{status}}' THEN updated_datetime END)      AS order_{{status.lower()}}_datetime
            {% endif %}
            {%- if not loop.last %},{% endif -%}
        {% endfor %}

    FROM order_logs_join_bookings

    GROUP BY
        transaction_order_id,
        payment_method,
        service_type
),

transaction_order_datetime_imputation AS (

    SELECT
        transaction_order_id,
        payment_method,
        service_type,
        final_order_status,

        order_created_datetime                                                  AS new_order_created_datetime,
        order_rejected_datetime                                                 AS new_order_rejected_datetime,
        order_suspended_datetime                                                AS new_order_suspended_datetime,
        order_expired_datetime                                                  AS new_order_expired_datetime,   

        -- imputation for ORDER_PENDING_SUBMISSION DATETIME
        COALESCE(order_pending_submission_datetime, order_created_datetime)     AS new_order_pending_submission_datetime,

        /*
        Data manipulation for ORDER_PENDING_VERIFICATION_DATETIME
            - To improve data consistency
        */
        CASE
            -- When final order status is not pending verification, return null
            WHEN final_order_status NOT IN ('PENDING_VERIFICATION', 'APPROVED', 'COMPLETED') THEN null
                ELSE COALESCE(order_pending_verification_datetime, new_order_pending_submission_datetime)            
        END     AS new_order_pending_verification_datetime,


        /*
        Data manipulation for ORDER_APPROVED_DATETIME
            - To improve data consistency
            - All orders from year 2023 has order_approved datetime
        */
        CASE
            -- When final order status is not approved, return null
            WHEN order_created_datetime >= '2023-01-01'AND final_order_status NOT IN ('APPROVED', 'COMPLETED') THEN null
                ELSE COALESCE(order_approved_datetime, new_order_pending_verification_datetime)
        END     AS new_order_approved_datetime,


        /*
        Missing value imputation for order_completed_datetime
            - Missing value caused by inconsistency and backlog from exchange system
            - Imputed dataset used for estimation of on-time delivery
        */
        CASE
            -- When final order status is not completed, return null, imputation will be done in next code block
            WHEN order_created_datetime >= '2023-01-01'AND final_order_status NOT IN ('COMPLETED') THEN null
            WHEN order_created_datetime < '2023-01-01' THEN order_completed_datetime

            /*
            Imputation for order_approved_datetime for 1688 payment
                - If payment_method is 'cash', add median of 53 minute to order_approved datetime
                - If payment_method is 'wallet', add median of 569 minute to order_approved datetime
                - If payment_method is 'cheque', reuse order_completed datetime, no orders using cheque in 1688 payment
                - If payment_method is 'payment_gateway', add median of 433 minute to order_approved datetime
            */
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'CASH'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 53, new_order_approved_datetime))
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'WALLET'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 569, new_order_approved_datetime))
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'CHEQUE'  
                THEN order_completed_datetime
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1688 PAYMENT' AND payment_method = 'PAYMENT_GATEWAY'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 433, new_order_approved_datetime))

            /*
            Imputation for order_approved_datetime for 1 day transfer
                - If payment_method is 'cash', add median of 59 minute to order_approved datetime
                - If payment_method is 'wallet', add median of 206 minute to order_approved datetime
                - If payment_method is 'cheque', reuse order_completed datetime, all orders have completed datetime
                - If payment_method is 'payment_gateway', add median of 214.5 minute to order_approved datetime
            */
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'CASH'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 59, new_order_approved_datetime))
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'WALLET'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 206, new_order_approved_datetime))
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'CHEQUE'  
                THEN order_completed_datetime
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '1 DAY TRANSFER' AND payment_method = 'PAYMENT_GATEWAY'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 214.5, new_order_approved_datetime))

            /*
            Imputation for order_approved_datetime for 3 days transfer
                - If payment_method is 'cash', add median of 2702 minute to order_approved datetime
                - If payment_method is 'wallet', add median of 4502.92 minute to order_approved datetime
                - If payment_method is 'cheque', reuse order_completed datetime, all orders have completed datetime
                - If payment_method is 'payment_gateway', add median of 5253 minute to order_approved datetime
            */
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'CASH'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 2702, new_order_approved_datetime))
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'WALLET'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 4502.92, new_order_approved_datetime))
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'CHEQUE'  
                THEN order_completed_datetime
            WHEN order_created_datetime >= '2023-01-01' AND service_type = '3 DAYS TRANSFER' AND payment_method = 'PAYMENT_GATEWAY'  
                THEN COALESCE(order_completed_datetime, DATEADD(minute, 5253, new_order_approved_datetime))
                
        END     AS new_order_completed_datetime
  
    FROM transaction_order_order_status_datetime
),

transaction_cost_status_datetime AS (

    SELECT
        transaction_cost_id,
        MIN(created_datetime) AS cost_created_datetime,

        {%- for status in order_status %}
            MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
        {% endfor %}    

    FROM transaction_cost_logs

    GROUP BY
        transaction_cost_id
),

booking_status_datetime AS (

    SELECT
        booking_id,
        MIN(created_datetime) AS booking_created_datetime,

        {%- for status in booking_status %}
            MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS booking_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
        {% endfor %}    

    FROM booking_logs

    GROUP BY
        booking_id
),

transaction_orders_lists AS (

    SELECT 
        transaction_orders.transaction_order_id,
        transaction_orders.booking_id,
        transaction_orders.bank_id,
        transaction_orders.company_id,
        transaction_orders.user_id,
        transaction_orders.status,
        transaction_orders.transaction_type,
        transaction_orders.payment_method,
        transaction_orders.base_currency_id,
        transaction_orders.quote_currency_id,
        transaction_orders.base_to_quote_currency_exchange_rate,
        transaction_orders.payment_reference,
        transaction_orders.bill_number,
        transaction_orders.base_value,
        transaction_orders.quote_value,
        -- use for future if base currency id other from '1'
        IFF(transaction_orders.base_currency_id = '1', 
           transaction_orders.base_value,
           -9999999999)                                             AS value_rm, 
            
        transaction_orders.base_tax                                 AS base_tax,
        -- use for future if base currency id other from '1'
        IFF(transaction_orders.base_currency_id = '1', 
           transaction_orders.base_tax,
           -9999999999)                                             AS tax_rm, 
                   
        transaction_orders.base_service_charge,
        -- use for future if base currency id other from '1'
        IFF(transaction_orders.base_currency_id = '1', 
           transaction_orders.base_service_charge,
           -9999999999)                                             AS service_charge_rm, 

        (value_rm + service_charge_rm + tax_rm)                     AS total_value_rm,

        transaction_orders.expired_datetime                         AS order_expected_expired_datetime,
        transaction_orders.deleted_datetime                         AS order_deleted_datetime,
        transaction_orders.updated_datetime                         AS order_updated_datetime,

        {% for column_name in ["company", "user"] %}
        CASE 
            WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.{{column_name}}_id ORDER BY transaction_orders.created_datetime, transaction_orders.transaction_order_id) = 1 THEN 1 
            ELSE 0 
        END             AS is_first_time_{{column_name}},

        CASE 
            WHEN transaction_orders.status = 'COMPLETED' 
                AND 
                MIN(IFF(transaction_orders.status = 'COMPLETED', transaction_orders.created_datetime, NULL))
                    OVER (PARTITION BY transaction_orders.{{column_name}}_id) 
                    = transaction_orders.created_datetime
            THEN 1
            ELSE 0 
        END             AS is_first_time_{{column_name}}_completed,
        {% endfor %}
        
        transaction_order_datetime_imputation.new_order_created_datetime                    AS order_created_datetime,
        -- transaction_order_datetime_imputation.order_expired_datetime,

        {%- for status in order_status %}
            transaction_order_datetime_imputation.new_order_{{status.lower()}}_datetime     AS order_{{status.lower()}}_datetime,
        {% endfor %}  
        
        LEAD(transaction_order_datetime_imputation.new_order_created_datetime) OVER 
            (PARTITION BY company_id ORDER BY transaction_order_datetime_imputation.new_order_created_datetime, 
            transaction_order_datetime_imputation.transaction_order_id)                     AS next_order_created_datetime
    
    FROM transaction_orders 

    LEFT JOIN transaction_order_datetime_imputation
        ON (transaction_orders.transaction_order_id = transaction_order_datetime_imputation.transaction_order_id)
),

transaction_costs_lists AS (

    SELECT 
        transaction_costs.transaction_cost_id,
        transaction_costs.transaction_order_id,
        transaction_costs.supplier_company_id,
        transaction_costs.bank_id,
        transaction_costs.base_currency_id,
        transaction_costs.quote_currency_id,
        transaction_costs.transaction_type,
        transaction_costs.payment_method,
        transaction_costs.status,
        transaction_costs.payment_reference,
        transaction_costs.bill_number,
        transaction_costs.base_value,
        transaction_costs.quote_value,
        transaction_costs.base_to_quote_currency_exchange_rate,
        transaction_costs.base_tax,
        transaction_costs.base_service_charge,

        -- use for future if base currency id other from '1'
        IFF(transaction_costs.base_currency_id = '1', 
           transaction_costs.base_value,
           -9999999999)         AS value_rm, 
            
        -- use for future if base currency id other from '1'
        IFF(transaction_costs.base_currency_id = '1', 
           transaction_costs.base_tax,
           -9999999999)         AS tax_rm, 
                   
        -- use for future if base currency id other from '1'
        IFF(transaction_costs.base_currency_id = '1', 
           transaction_costs.base_service_charge,
           -9999999999)         AS service_charge_rm, 

        (value_rm + service_charge_rm + tax_rm)     AS total_value_rm,

        transaction_costs.expired_datetime,
        transaction_costs.deleted_datetime,
        transaction_costs.created_datetime,
        transaction_costs.updated_datetime,

        {%- for status in order_status %}
            transaction_cost_status_datetime.cost_{{status.lower()}}_datetime,
        {% endfor %}
        transaction_cost_status_datetime.cost_created_datetime

    FROM transaction_costs

    LEFT JOIN transaction_cost_status_datetime
        ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id)

), 

bookings_lists AS (
    SELECT 
        bookings.*,

        {% for column_name in ["company", "user"] %}
        CASE 
            WHEN ROW_NUMBER() OVER (PARTITION BY bookings.{{column_name}}_id ORDER BY bookings.created_datetime, bookings.booking_id) = 1 THEN 1 
            ELSE 0 
        END         AS is_first_time_{{column_name}},

        CASE 
            WHEN bookings.status = 'COMPLETED' 
                AND 
                MIN(IFF(bookings.status = 'COMPLETED', bookings.created_datetime, NULL))
                    OVER (PARTITION BY bookings.{{column_name}}_id) 
                    = bookings.created_datetime
            THEN 1
            ELSE 0 
        END         AS is_first_time_{{column_name}}_completed,
        {% endfor %}

        booking_status_datetime.booking_created_datetime,
        {%- for status in booking_status %}
            booking_status_datetime.booking_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
        {% endfor %}  
    
    FROM bookings

    LEFT JOIN booking_status_datetime
        ON (bookings.booking_id = booking_status_datetime.booking_id)
),

bookings_orders_costs_join AS (

    SELECT
        bookings_lists.booking_id,
        bookings_lists.booking_marking_id,
        bookings_lists.user_id,
        bookings_lists.fix_currency_id,
        bookings_lists.quote_currency_id,
        bookings_lists.base_currency_id,
        bookings_lists.service_type,
        bookings_lists.status                                             AS booking_status,
        bookings_lists.is_first_time_company                              AS is_first_time_booking_company,
        bookings_lists.is_first_time_company_completed                    AS is_first_time_booking_company_completed,
        bookings_lists.is_first_time_user                                 AS is_first_time_booking_user,
        bookings_lists.is_first_time_user_completed                       AS is_first_time_booking_user_completed,
        bookings_lists.estimate_base_to_quote_currency_exchange_rate      AS estimate_booking_base_to_quote_currency_exchange_rate,
        bookings_lists.estimate_quote_value                               AS estimate_booking_quote_value,
        bookings_lists.estimate_base_value                                AS estimate_booking_base_value,
        bookings_lists.estimate_value_rm                                  AS estimate_booking_value_rm,
        bookings_lists.booking_created_datetime,
        bookings_lists.booking_approved_datetime,
        bookings_lists.booking_completed_datetime,
        bookings_lists.booking_suspended_datetime,
                
        transaction_orders_lists.transaction_order_id                     AS order_id,
        transaction_orders_lists.company_id,
        transaction_orders_lists.bank_id,
        transaction_orders_lists.transaction_type                         AS order_transaction_type,
        transaction_orders_lists.payment_method                           AS order_payment_method,
        transaction_orders_lists.status                                   AS order_status,                             
        transaction_orders_lists.payment_reference                        AS order_payment_reference,
        transaction_orders_lists.bill_number                              AS order_bill_number,
        transaction_orders_lists.is_first_time_company                    AS is_first_time_order_company,
        transaction_orders_lists.is_first_time_company_completed          AS is_first_time_order_company_completed,
        transaction_orders_lists.is_first_time_user                       AS is_first_time_order_user,
        transaction_orders_lists.is_first_time_user_completed             AS is_first_time_order_user_completed,
        transaction_orders_lists.base_to_quote_currency_exchange_rate     AS order_base_to_quote_currency_exchange_rate ,
        transaction_orders_lists.base_value                               AS order_base_value ,
        transaction_orders_lists.base_service_charge                      AS order_base_service_charge ,
        transaction_orders_lists.base_tax                                 AS order_base_tax ,
        transaction_orders_lists.quote_value                              AS order_quote_value ,
        transaction_orders_lists.value_rm                                 AS order_value_rm ,
        transaction_orders_lists.service_charge_rm                        AS order_service_charge_rm ,
        transaction_orders_lists.tax_rm                                   AS order_tax_rm ,
        transaction_orders_lists.total_value_rm                           AS total_order_value_rm,
        transaction_orders_lists.order_created_datetime,
        transaction_orders_lists.order_pending_submission_datetime,
        transaction_orders_lists.order_pending_verification_datetime,
        transaction_orders_lists.order_approved_datetime,
        transaction_orders_lists.order_completed_datetime,
        transaction_orders_lists.order_rejected_datetime,
        transaction_orders_lists.order_suspended_datetime,
        transaction_orders_lists.order_expired_datetime,
        transaction_orders_lists.next_order_created_datetime,
        transaction_orders_lists.order_expected_expired_datetime,

        transaction_costs_lists.transaction_cost_id                     AS cost_id,
        transaction_costs_lists.bank_id                                 AS currency_vendor_bank_id,
        transaction_costs_lists.supplier_company_id                     AS supplier_company_id,
        transaction_costs_lists.transaction_type                        AS cost_transaction_type,
        transaction_costs_lists.payment_method                          AS cost_payment_method,
        transaction_costs_lists.status                                  AS cost_status,
        transaction_costs_lists.payment_reference                       AS cost_payment_reference,
        transaction_costs_lists.bill_number                             AS cost_bill_number,
        transaction_costs_lists.base_value                              AS cost_base_value,
        transaction_costs_lists.quote_value                             AS cost_quote_value,
        transaction_costs_lists.base_to_quote_currency_exchange_rate    AS cost_base_to_quote_currency_exchange_rate,
        transaction_costs_lists.base_tax                                AS cost_base_tax,
        transaction_costs_lists.base_service_charge                     AS cost_base_service_charge,
        transaction_costs_lists.value_rm                                AS cost_value_rm,
        transaction_costs_lists.tax_rm                                  AS cost_tax_rm,
        transaction_costs_lists.service_charge_rm                       AS cost_service_charge_rm,
        transaction_costs_lists.total_value_rm                          AS total_cost_value_rm,
        transaction_costs_lists.cost_created_datetime,
        transaction_costs_lists.cost_pending_submission_datetime,
        transaction_costs_lists.cost_pending_verification_datetime,
        transaction_costs_lists.cost_approved_datetime,
        transaction_costs_lists.cost_completed_datetime,
        transaction_costs_lists.cost_rejected_datetime,

        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime

    FROM transaction_orders_lists

    LEFT JOIN transaction_costs_lists
        ON transaction_orders_lists.transaction_order_id = transaction_costs_lists.transaction_order_id

    LEFT JOIN bookings_lists
        ON transaction_orders_lists.booking_id = bookings_lists.booking_id

),

-- FINAL
final_fct_exchange__new_orders AS (

    SELECT
        -- ids
        order_id,
        booking_id,
        booking_marking_id,
        company_id,
        user_id,
        bank_id,
        cost_id,
        supplier_company_id,
        currency_vendor_bank_id,
        fix_currency_id,
        quote_currency_id,
        base_currency_id,

        -- dimensions
        service_type,
        booking_status,
        is_first_time_booking_company,
        is_first_time_booking_company_completed,
        is_first_time_booking_user,
        is_first_time_booking_user_completed,
        is_first_time_order_company,
        is_first_time_order_company_completed,
        is_first_time_order_user,
        is_first_time_order_user_completed,
        order_transaction_type,
        order_payment_method,
        order_status,
        order_payment_reference,
        order_bill_number,
        cost_transaction_type,
        cost_payment_method,
        cost_status,
        cost_payment_reference,
        cost_bill_number,
        
        -- measures
        estimate_booking_base_to_quote_currency_exchange_rate,
        estimate_booking_quote_value,
        estimate_booking_base_value,
        estimate_booking_value_rm,
        order_base_to_quote_currency_exchange_rate,
        order_base_value,
        order_base_service_charge,
        order_base_tax,
        order_quote_value,
        order_value_rm,
        order_service_charge_rm,
        order_tax_rm,
        total_order_value_rm,
        cost_base_value,
        cost_quote_value,
        cost_base_to_quote_currency_exchange_rate,
        cost_base_tax,
        cost_base_service_charge,
        cost_value_rm,
        cost_tax_rm,
        cost_service_charge_rm,
        total_cost_value_rm,

        -- date/times
        booking_created_datetime,
        booking_approved_datetime,
        booking_completed_datetime,
        booking_suspended_datetime,
        
        order_deleted_datetime,
        order_updated_datetime,
        order_created_datetime,
        order_expected_expired_datetime,
        order_pending_submission_datetime,
        order_pending_verification_datetime,
        order_approved_datetime,
        order_completed_datetime,
        order_rejected_datetime,
        order_suspended_datetime,
        order_expired_datetime,
        
        next_order_created_datetime,
        cost_created_datetime,
        cost_pending_submission_datetime,
        cost_pending_verification_datetime,
        cost_approved_datetime,
        cost_completed_datetime,
        cost_rejected_datetime, 

        -- metadata
        _dbt_ran_datetime

    FROM bookings_orders_costs_join
)

SELECT * FROM final_fct_exchange__new_orders