Merge branch 'lee_build_exchange_rate_fact' into 'main'

Redefine order/cost value formual + Redefine on-time boolean logic

See merge request cief-data/dbt_cloud!35

models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql

@@ -30,7 +30,7 @@ temp_orders AS (
 -- LOGIC
 /*
 Some order may have multiple same cost status. This is due to the accounting department adjusting the exchange rate.
-This is particularly true for the 1688 payment, as exchange rate is amended at a later stage instead of during the initial order placement stage.
+This is particularly true for the 1688 payment, as the cost rate is amended at a later stage instead of during the initial order placement stage.
 Thus, it is to be assumed that the datetime from first repeat occurence as the true datetime.
 */
 transaction_cost_status_datetime AS (
@@ -71,6 +71,7 @@ join_cost_and_logs AS (
         transaction_costs.base_to_quote_currency_exchange_rate,
         transaction_costs.base_tax,
         transaction_costs.base_service_charge,
+        transaction_costs.total_base_value,
         transaction_costs.expired_datetime,
         transaction_costs.deleted_datetime,
         transaction_costs.created_datetime,
@@ -100,6 +101,7 @@ join_cost_and_logs AS (
 
     QUALIFY
         row_number_index = 1
+        
 ),
 
 remove_deleted_expired_costs AS (
@@ -157,6 +159,7 @@ temp_cte_1688_cleaning AS (
         remove_system_error_duplicate_row.quote_value,
         remove_system_error_duplicate_row.base_to_quote_currency_exchange_rate,
         remove_system_error_duplicate_row.base_tax,
+        remove_system_error_duplicate_row.total_base_value,
         remove_system_error_duplicate_row.expired_datetime,
         remove_system_error_duplicate_row.deleted_datetime,
         remove_system_error_duplicate_row.created_datetime,
@@ -231,6 +234,7 @@ final__int_exchange__transaction_cost_get_latest_cost_ids AS (
         base_to_quote_currency_exchange_rate,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         expired_datetime,

models/intermediate/int_exchange__transaction_orders_get_user_ids.sql

@@ -7,8 +7,10 @@ bookings AS (
     SELECT * FROM {{ ref('stg_exchange__bookings') }}
 ),
 
+
 -- LOGICS
 transaction_orders_join_bookings AS (
+
     SELECT 
         transaction_orders.*,
         bookings.user_id
@@ -17,11 +19,13 @@ transaction_orders_join_bookings AS (
 
     LEFT JOIN bookings 
         ON (transaction_orders.booking_id = bookings.booking_id)
+        
 ),
 
 
 -- FINAL
 final__int_exchange__transaction_orders_get_user_ids AS (
+
     SELECT 
         -- ids
         transaction_order_id,
@@ -45,6 +49,7 @@ final__int_exchange__transaction_orders_get_user_ids AS (
         base_to_quote_currency_exchange_rate,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         expired_datetime,
@@ -56,6 +61,7 @@ final__int_exchange__transaction_orders_get_user_ids AS (
         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
 
     FROM transaction_orders_join_bookings
+
 )
 
 SELECT * FROM final__int_exchange__transaction_orders_get_user_ids

models/marts/reporting/rep_exchange__daily_orders.sql

@@ -27,7 +27,7 @@
 
 -- IMPORT
 WITH orders AS (
-    SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} 
+    SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
 ),
 
 companies AS (
@@ -190,10 +190,16 @@ fct_and_dim_joins AS (
         CASE
             {% for (service_type, working_day) in service_type_work_day_dict.items() %}
                 
-                WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 
+                WHEN 
-                    THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
+                    orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' 
-                WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 
+                    AND 
-                    THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
+                    DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 
+                        THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
+                WHEN 
+                    orders.order_created_datetime >= '2023-01-01' 
+                    AND 
+                    orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 
+                        THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
                     
             {% endfor %}
 
@@ -203,19 +209,21 @@ fct_and_dim_joins AS (
         -- On time delivery boolean (CIEF internal SLA)
         CASE
             -- On-time boolean is null when order is not complete
-            WHEN account_verified_payment_datetime IS NULL THEN NULL
+            WHEN service_type IN ('BA') THEN NULL
-            WHEN estimated_order_delivery_datetime_website_sla IS NULL THEN NULL
+            WHEN transaction_status NOT IN ('UPLOADED_BANK_SLIP') THEN NULL
             WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1
-            ELSE 0
+            WHEN estimated_order_delivery_datetime_website_sla < orders.operation_uploaded_bank_slip_datetime THEN 0
+            ELSE -999
         END     AS is_on_time_delivery_website_sla,
 
         -- On time delivery boolean (Customer Expectation)
         CASE
             -- On-time boolean is null when order is not complete
-            WHEN account_verified_payment_datetime IS NULL THEN NULL
+            WHEN service_type IN ('BA') THEN NULL
-            WHEN estimated_order_delivery_datetime_customer_expectation IS NULL THEN NULL
+            WHEN transaction_status NOT IN ('UPLOADED_BANK_SLIP') THEN NULL
             WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1
-            ELSE 0
+            WHEN estimated_order_delivery_datetime_customer_expectation < orders.operation_uploaded_bank_slip_datetime THEN 0
+            ELSE -999
         END     AS is_on_time_delivery_customer_expectation,
 
         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'       AS _dbt_ran_datetime
@@ -242,6 +250,7 @@ fct_and_dim_joins AS (
 
 ),
 
+
 --FINAL
 final__rep_exchange__daily_orders AS (
 

models/marts/warehouse/dim_exchange__companies.sql

@@ -38,6 +38,7 @@ transaction_orders AS (
 
 -- LOGIC
 count_company_user AS (
+
     SELECT 
         company_id,
         COUNT(user_id)      AS count_user
@@ -45,22 +46,26 @@ count_company_user AS (
     FROM companies_brg_users
 
     GROUP BY company_id
+
 ),
 
 company_billing_addresses AS (
+
     SELECT
         *
 
     FROM company_addresses
 
     WHERE 
-        is_billing_address = '1'
+        is_billing_address = 1
+
 ),
 
 latest_company_upload_identity_documents AS (
+
     SELECT
         *,
-        row_number() OVER 
+        ROW_NUMBER() OVER 
                     (PARTITION BY company_id 
                     ORDER BY created_datetime DESC) AS latest_row_number
 
@@ -69,14 +74,15 @@ latest_company_upload_identity_documents AS (
     WHERE 
         document_type IN ('IDENTITY_CARD','SSM_REGISTRATION')
         AND
-        status = 'APPROVED'
+        status IN ('APPROVED')
 
     QUALIFY
-        latest_row_number = '1'
+        latest_row_number = 1
 
 ),
 
 companies_rename_and_join_table AS (
+
     SELECT
         companies.company_id,
         companies.name,
@@ -161,19 +167,23 @@ companies_rename_and_join_table AS (
 
     LEFT JOIN currencies
         ON (company_wallets.currency_id = currencies.currency_id)
+
 ),
 
 lifetime_value AS (
+
     SELECT
         companies_rename_and_join_table.*,
 
         ROUND(
             ZEROIFNULL(SUM(
-            CASE 
+                CASE 
-                WHEN transaction_orders.status in ('COMPLETED', 'APPROVED') 
+                    WHEN transaction_orders.status IN ('COMPLETED', 'APPROVED') 
-                THEN ( transaction_orders.base_value + transaction_orders.base_tax + transaction_orders.base_service_charge )  
+                        THEN transaction_orders.total_base_value
-            END))
+                END
-        ,2)         AS company_lifetime_value,
+            ))
+        ,2
+        )         AS company_lifetime_value,
 
         CASE
             WHEN company_lifetime_value <= {{monetary_list_lifetime[0]}} THEN 1
@@ -256,6 +266,7 @@ lifetime_value AS (
 
 -- FINAL
 final__dim_exchange__companies AS (
+
     SELECT
         -- ids
         company_id,

models/marts/warehouse/fct_exchange__transaction_costs.sql

@@ -11,6 +11,7 @@ transaction_cost_logs AS (
     SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }}
 ),
 
+
 -- LOGICS
 transaction_cost_status_datetime AS (
 
@@ -29,6 +30,7 @@ transaction_cost_status_datetime AS (
 ),
 
 transaction_costs_lists AS (
+
     SELECT 
         transaction_costs.transaction_cost_id,
         transaction_costs.transaction_order_id,
@@ -46,23 +48,24 @@ transaction_costs_lists AS (
         transaction_costs.base_to_quote_currency_exchange_rate,
         transaction_costs.base_tax,
         transaction_costs.base_service_charge,
+        transaction_costs.total_base_value,
 
         -- use for future if base currency id other from '1'
         IFF(transaction_costs.base_currency_id = '1', 
            transaction_costs.base_value,
-           -9999999999) AS value_rm, 
+           -9999999999)                                         AS value_rm, 
             
         -- use for future if base currency id other from '1'
         IFF(transaction_costs.base_currency_id = '1', 
            transaction_costs.base_tax,
-           -9999999999) AS tax_rm, 
+           -9999999999)                                         AS tax_rm, 
                    
         -- use for future if base currency id other from '1'
         IFF(transaction_costs.base_currency_id = '1', 
            transaction_costs.base_service_charge,
-           -9999999999) AS service_charge_rm, 
+           -9999999999)                                         AS service_charge_rm, 
 
-        (value_rm - service_charge_rm - tax_rm) AS total_value_rm,
+        (value_rm + service_charge_rm + tax_rm)                 AS total_value_rm,
 
         transaction_costs.expired_datetime,
         transaction_costs.deleted_datetime,
@@ -70,6 +73,7 @@ transaction_costs_lists AS (
         transaction_costs.updated_datetime,
 
         transaction_cost_status_datetime.cost_created_datetime,
+        
         {%- for status in cost_status %}
             transaction_cost_status_datetime.cost_{{status.lower()}}_datetime,
         {% endfor %}
@@ -80,11 +84,13 @@ transaction_costs_lists AS (
 
     LEFT JOIN transaction_cost_status_datetime
         ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id)
+
 ),
 
 
 -- FINAL
 final__fct_exchange__transaction_costs AS (
+    
     SELECT 
         -- ids
         transaction_cost_id,
@@ -110,6 +116,7 @@ final__fct_exchange__transaction_costs AS (
         value_rm,
         tax_rm,
         service_charge_rm,
+        total_base_value,
         total_value_rm,
 
         -- date/times
@@ -129,6 +136,7 @@ final__fct_exchange__transaction_costs AS (
         _dbt_ran_datetime
 
     FROM transaction_costs_lists
+
 )
 
 SELECT * FROM final__fct_exchange__transaction_costs

models/marts/warehouse/fct_exchange__transaction_orders.sql

@@ -46,16 +46,19 @@ order_logs_join_bookings AS (
     SELECT
         transaction_order_logs.*,
         bookings.service_type,
-        ROW_NUMBER() OVER (PARTITION BY transaction_order_id 
+        ROW_NUMBER() OVER (
+            PARTITION BY transaction_order_id 
             ORDER BY 
-            transaction_order_logs.updated_datetime DESC, 
+                transaction_order_logs.updated_datetime DESC, 
-            transaction_order_logs.transaction_order_id DESC, 
+                transaction_order_logs.transaction_order_id DESC, 
-            transaction_order_logs.transaction_order_log_id DESC)   AS rank_index
+                transaction_order_logs.transaction_order_log_id DESC
+        )   AS rank_index
 
     FROM transaction_order_logs
 
     LEFT JOIN bookings
         ON (transaction_order_logs.booking_id = bookings.booking_id)
+
 ),
 
 transaction_order_order_status_datetime AS (
@@ -69,12 +72,15 @@ transaction_order_order_status_datetime AS (
         MIN(created_datetime)                                                       AS order_created_datetime,
 
         {%- for status in order_status %}
+
             {% if status != 'APPROVED' %}
                 MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END)      AS order_{{status.lower()}}_datetime 
             {% else %}
                 MIN(CASE WHEN status = '{{status}}' THEN updated_datetime END)      AS order_{{status.lower()}}_datetime
             {% endif %}
+
             {%- if not loop.last %},{% endif -%}
+
         {% endfor %}
 
     FROM order_logs_join_bookings
@@ -83,6 +89,7 @@ transaction_order_order_status_datetime AS (
         transaction_order_id,
         payment_method,
         service_type
+
 ),
 
 transaction_order_datetime_imputation AS (
@@ -96,7 +103,7 @@ transaction_order_datetime_imputation AS (
         order_created_datetime                                                  AS new_order_created_datetime,
         order_rejected_datetime                                                 AS new_order_rejected_datetime,
         order_suspended_datetime                                                AS new_order_suspended_datetime,
-        order_expired_datetime                                                  AS new_order_expired_datetime,   
+        order_expired_datetime                                                  AS new_order_expired_datetime,
         order_refunded_datetime                                                 AS new_order_refunded_datetime,  
 
         -- imputation for ORDER_PENDING_SUBMISSION DATETIME
@@ -224,44 +231,56 @@ transaction_orders_lists AS (
         transaction_orders.bill_number,
         transaction_orders.base_value,
         transaction_orders.quote_value,
+        transaction_orders.total_base_value,
+
         -- use for future if base currency id other from '1'
         IFF(transaction_orders.base_currency_id = '1', 
            transaction_orders.base_value,
-           -9999999999)                                             AS value_rm, 
+           -9999999999)                                             AS base_value_rm,
             
         transaction_orders.base_tax                                 AS base_tax,
+
         -- use for future if base currency id other from '1'
         IFF(transaction_orders.base_currency_id = '1', 
            transaction_orders.base_tax,
            -9999999999)                                             AS tax_rm, 
                    
         transaction_orders.base_service_charge,
+
         -- use for future if base currency id other from '1'
         IFF(transaction_orders.base_currency_id = '1', 
            transaction_orders.base_service_charge,
            -9999999999)                                             AS service_charge_rm, 
 
-        (value_rm - service_charge_rm - tax_rm)                     AS total_value_rm,
+        (base_value_rm + service_charge_rm + tax_rm)                AS total_base_value_rm,
 
         transaction_orders.expired_datetime                         AS order_expected_expired_datetime,
         transaction_orders.deleted_datetime                         AS order_deleted_datetime,
         transaction_orders.updated_datetime                         AS order_updated_datetime,
 
         {% for column_name in ["company", "user"] %}
-        CASE 
+            CASE 
-            WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.{{column_name}}_id ORDER BY transaction_orders.created_datetime, transaction_orders.transaction_order_id) = 1 THEN 1 
+                WHEN 
-            ELSE 0 
+                    ROW_NUMBER() OVER (
-        END             AS is_first_time_{{column_name}},
+                        PARTITION BY transaction_orders.{{column_name}}_id 
+                        ORDER BY 
+                            transaction_orders.created_datetime, 
+                            transaction_orders.transaction_order_id
+                    ) = 1 
+                THEN 1 
+                ELSE 0 
+            END             AS is_first_time_{{column_name}},
 
-        CASE 
+            CASE 
-            WHEN transaction_orders.status = 'COMPLETED' 
+                WHEN 
-                AND 
+                    transaction_orders.status = 'COMPLETED' 
-                MIN(IFF(transaction_orders.status = 'COMPLETED', transaction_orders.created_datetime, NULL))
+                    AND 
-                    OVER (PARTITION BY transaction_orders.{{column_name}}_id) 
+                    MIN(IFF(transaction_orders.status = 'COMPLETED', transaction_orders.created_datetime, NULL)) OVER (
-                    = transaction_orders.created_datetime
+                        PARTITION BY transaction_orders.{{column_name}}_id
-            THEN 1
+                    ) = transaction_orders.created_datetime
-            ELSE 0 
+                THEN 1
-        END             AS is_first_time_{{column_name}}_completed,
+                ELSE 0 
+            END             AS is_first_time_{{column_name}}_completed,
         {% endfor %}
         
         transaction_order_datetime_imputation.new_order_created_datetime                    AS order_created_datetime,
@@ -271,9 +290,12 @@ transaction_orders_lists AS (
             transaction_order_datetime_imputation.new_order_{{status.lower()}}_datetime     AS order_{{status.lower()}}_datetime,
         {% endfor %}  
         
-        LEAD(transaction_order_datetime_imputation.new_order_created_datetime) OVER 
+        LEAD(transaction_order_datetime_imputation.new_order_created_datetime) OVER (
-            (PARTITION BY company_id ORDER BY transaction_order_datetime_imputation.new_order_created_datetime, 
+            PARTITION BY company_id 
-            transaction_order_datetime_imputation.transaction_order_id)                     AS next_order_created_datetime
+            ORDER BY 
+                transaction_order_datetime_imputation.new_order_created_datetime, 
+                transaction_order_datetime_imputation.transaction_order_id
+            )               AS next_order_created_datetime
     
     FROM transaction_orders 
 
@@ -319,7 +341,7 @@ transaction_costs_lists AS (
            base_service_charge,
            -9999999999)         AS service_charge_rm, 
 
-        (value_rm - service_charge_rm - tax_rm)     AS total_value_rm,
+        (value_rm + service_charge_rm + tax_rm)     AS total_value_rm,
 
         expired_datetime,
         deleted_datetime,
@@ -407,15 +429,17 @@ bookings_orders_costs_join AS (
         transaction_orders_lists.is_first_time_company_completed          AS is_first_time_order_company_completed,
         transaction_orders_lists.is_first_time_user                       AS is_first_time_order_user,
         transaction_orders_lists.is_first_time_user_completed             AS is_first_time_order_user_completed,
-        transaction_orders_lists.base_to_quote_currency_exchange_rate     AS order_base_to_quote_currency_exchange_rate ,
+
-        transaction_orders_lists.base_value                               AS order_base_value ,
+        transaction_orders_lists.base_to_quote_currency_exchange_rate     AS order_base_to_quote_currency_exchange_rate,
-        transaction_orders_lists.base_service_charge                      AS order_base_service_charge ,
+        transaction_orders_lists.base_value                               AS order_base_value,
-        transaction_orders_lists.base_tax                                 AS order_base_tax ,
+        transaction_orders_lists.base_service_charge                      AS order_base_service_charge,
-        transaction_orders_lists.quote_value                              AS order_quote_value ,
+        transaction_orders_lists.base_tax                                 AS order_base_tax,
-        transaction_orders_lists.value_rm                                 AS order_value_rm ,
+        transaction_orders_lists.quote_value                              AS order_quote_value,
-        transaction_orders_lists.service_charge_rm                        AS order_service_charge_rm ,
+        transaction_orders_lists.base_value_rm                            AS order_value_rm,
-        transaction_orders_lists.tax_rm                                   AS order_tax_rm ,
+        transaction_orders_lists.service_charge_rm                        AS order_service_charge_rm,
-        transaction_orders_lists.total_value_rm                           AS total_order_value_rm,
+        transaction_orders_lists.tax_rm                                   AS order_tax_rm,
+        transaction_orders_lists.total_base_value_rm                      AS total_order_value_rm,
+
         transaction_orders_lists.order_created_datetime,
         transaction_orders_lists.order_deleted_datetime,
         transaction_orders_lists.order_pending_submission_datetime,
@@ -478,6 +502,7 @@ purchase_orders_invoices_join AS (
         CASE 
             WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.purchase_order_number
         END     AS purchase_order_number,
+
         CASE 
             WHEN order_status NOT IN ('REJECTED', 'SUSPENDED', 'EXPIRED') THEN purchase_orders.status
         END     AS purchase_order_status,
@@ -485,6 +510,7 @@ purchase_orders_invoices_join AS (
         CASE
             WHEN cost_status IN ('APPROVED') THEN invoices.invoice_number
         END     AS invoice_number,
+        
         CASE
             WHEN cost_status IN ('APPROVED') THEN invoices.status
         END     AS invoice_status

models/staging/exchange/stg_exchange__transaction_cost_logs.sql

@@ -23,32 +23,36 @@ status AS (
 
 -- LOGIC
 transactions_generate_transaction_log_id AS (
+
     SELECT
         MD5_NUMBER_LOWER64(CONCAT(transaction_id, created_datetime))  AS transaction_log_id,
         *
 
     FROM transactions
+
 ),
 
 transaction_arch_logs_union_transactions AS (
+
     SELECT * FROM transactions_generate_transaction_log_id
     UNION
     SELECT * FROM transaction_arch_logs
+
 ),
 
 union_transaction_rename_column_and_column_value AS (
 
     SELECT
-        transaction_arch_logs_union_transactions.transaction_log_id AS transaction_cost_log_id,
+        transaction_arch_logs_union_transactions.transaction_log_id     AS transaction_cost_log_id,
-        transaction_arch_logs_union_transactions.transaction_id AS transaction_cost_id,
+        transaction_arch_logs_union_transactions.transaction_id         AS transaction_cost_id,
 
         IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Transaction', 
             transaction_arch_logs_union_transactions.owner_id,
             null)   AS transaction_order_id,
 
-        COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type,
+        COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string)     AS transaction_type,
 
-        transaction_arch_logs_union_transactions.issuer_user_id AS supplier_company_id,
+        transaction_arch_logs_union_transactions.issuer_user_id     AS supplier_company_id,
         transaction_arch_logs_union_transactions.receiver_user_id   AS company_id,
 
         transaction_arch_logs_union_transactions.recipient_bank_id,
@@ -58,23 +62,22 @@ union_transaction_rename_column_and_column_value AS (
         transaction_arch_logs_union_transactions.payment_reference,
         transaction_arch_logs_union_transactions.bill_number,
 
+        transaction_arch_logs_union_transactions.base_currency_id,
+        transaction_arch_logs_union_transactions.quote_currency_id,
         transaction_arch_logs_union_transactions.base_value,
         transaction_arch_logs_union_transactions.quote_value,
-
-        transaction_arch_logs_union_transactions.base_currency_id,
-       
-        transaction_arch_logs_union_transactions.quote_currency_id,
-     
         transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate,
         transaction_arch_logs_union_transactions.base_tax,
         transaction_arch_logs_union_transactions.base_service_charge,
-        transaction_arch_logs_union_transactions.expired_datetime,
+        ( transaction_arch_logs_union_transactions.base_value + transaction_arch_logs_union_transactions.base_service_charge )    AS total_base_value,
         
         COALESCE(status.name, transaction_arch_logs_union_transactions.status::string)   AS status,
 
+        transaction_arch_logs_union_transactions.expired_datetime,
         transaction_arch_logs_union_transactions.deleted_datetime,
         transaction_arch_logs_union_transactions.created_datetime,
         transaction_arch_logs_union_transactions.updated_datetime,
+
         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
 
     FROM transaction_arch_logs_union_transactions
@@ -87,12 +90,12 @@ union_transaction_rename_column_and_column_value AS (
 
     LEFT JOIN status
         ON (transaction_arch_logs_union_transactions.status = status.id)
-
     
     WHERE 
         transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Transaction' 
         AND 
         transaction_arch_logs_union_transactions.transaction_type = '3' --BILL
+
 ),
 
 
@@ -123,6 +126,7 @@ final__stg_exchange__transaction_log_costs AS (
         base_to_quote_currency_exchange_rate,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         expired_datetime,

models/staging/exchange/stg_exchange__transaction_costs.sql

@@ -21,37 +21,32 @@ status AS (
 transactions_join_transaction_types_payment_methods_currencies_status AS (
 
     SELECT
-        transactions.transaction_id AS transaction_cost_id,
+        transactions.transaction_id     AS transaction_cost_id,
-        IFF(transactions.owner_type = 'App\\Models\\Transaction', transactions.owner_id, null)  AS transaction_order_id,
+        transactions.issuer_user_id     AS supplier_company_id,
-
-        COALESCE(transaction_types.name, transactions.transaction_type::string)   AS transaction_type,
-
-        transactions.issuer_user_id             AS supplier_company_id,
-
         transactions.recipient_bank_id  AS bank_id,
-
+        COALESCE(payment_methods.name, transactions.payment_method::string)                     AS payment_method,
-        COALESCE(payment_methods.name, transactions.payment_method::string)   AS payment_method,
+        COALESCE(transaction_types.name, transactions.transaction_type::string)                 AS transaction_type,
+        IFF(transactions.owner_type = 'App\\Models\\Transaction', transactions.owner_id, null)  AS transaction_order_id,
 
         transactions.payment_reference,
         transactions.bill_number,
-
-        transactions.base_value,
-        transactions.quote_value,
-
         transactions.base_currency_id,
         transactions.quote_currency_id,
-
         transactions.base_to_quote_currency_exchange_rate,
+
         transactions.base_tax,
         transactions.base_service_charge,
-
+        transactions.base_value,                                   
-        transactions.expired_datetime,
+        transactions.quote_value,
+        ( transactions.base_value + transactions.base_service_charge )  AS total_base_value,
         
-        COALESCE(status.name, transactions.status::string)   AS status,        
+        COALESCE(status.name, transactions.status::string)              AS status,    
-
+            
+        transactions.expired_datetime,
         transactions.deleted_datetime,
         transactions.created_datetime,
         transactions.updated_datetime,
+
         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
 
     FROM transactions
@@ -65,7 +60,6 @@ transactions_join_transaction_types_payment_methods_currencies_status AS (
     LEFT JOIN status
         ON (transactions.status = status.id)
 
-    
     WHERE 
         transactions.owner_type = 'App\\Models\\Transaction' 
         AND 
@@ -95,10 +89,11 @@ final__stg_exchange__transaction_costs AS (
 
         -- measures
         base_value,
-        quote_value,
         base_to_quote_currency_exchange_rate,
+        quote_value,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         expired_datetime,
@@ -110,6 +105,7 @@ final__stg_exchange__transaction_costs AS (
         _dbt_ran_datetime
 
     FROM transactions_join_transaction_types_payment_methods_currencies_status
+
 ) 
 
 SELECT * FROM final__stg_exchange__transaction_costs

models/staging/exchange/stg_exchange__transaction_invoices.sql

@@ -47,11 +47,12 @@ transactions_join_transaction_types_payment_methods_bookings_status AS (
         COALESCE(payment_methods.name, transactions.payment_method::string)                 AS payment_method,
         COALESCE(status.name, transactions.status::string)                                  AS status,
         transactions.bill_number                                                            AS invoice_number,
-        transactions.base_value,
+        ( transactions.quote_value / transactions.base_to_quote_currency_exchange_rate )    AS base_value,
         transactions.quote_value,
         transactions.base_to_quote_currency_exchange_rate,
         transactions.base_tax,
         transactions.base_service_charge,
+        (( transactions.quote_value / transactions.base_to_quote_currency_exchange_rate ) + transactions.base_service_charge )   AS total_base_value,
 
         transactions.expired_datetime,
         transactions.deleted_datetime,
@@ -80,6 +81,7 @@ transactions_join_transaction_types_payment_methods_bookings_status AS (
 ),
 
 remove_deleted_records AS (
+    
     SELECT 
         *
     FROM transactions_join_transaction_types_payment_methods_bookings_status
@@ -88,9 +90,10 @@ remove_deleted_records AS (
 ),
 
 remove_duplicated_records AS (
+
     SELECT
         *,
-        ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC)   AS row_index,
+        ROW_NUMBER() OVER (PARTITION BY booking_id ORDER BY updated_datetime DESC)          AS row_index,
 
         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'   AS _dbt_ran_datetime
 
@@ -98,6 +101,7 @@ remove_duplicated_records AS (
 
     QUALIFY
         row_index = 1
+
 ),
 
 
@@ -128,6 +132,7 @@ final__stg_exchange__invoices AS (
         base_to_quote_currency_exchange_rate,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         deleted_datetime,

models/staging/exchange/stg_exchange__transaction_order_logs.sql

@@ -29,19 +29,17 @@ transactions_generate_transaction_log_id AS (
     FROM transactions
 ),
 
-
 transaction_arch_logs_union_transactions AS (
     SELECT * FROM transactions_generate_transaction_log_id
     UNION
     SELECT * FROM transaction_arch_logs
 ),
 
-
 union_transaction_rename_column_and_column_value AS (
 
     SELECT
-        transaction_arch_logs_union_transactions.transaction_log_id AS transaction_order_log_id,
+        transaction_arch_logs_union_transactions.transaction_log_id     AS transaction_order_log_id,
-        transaction_arch_logs_union_transactions.transaction_id AS transaction_order_id,
+        transaction_arch_logs_union_transactions.transaction_id         AS transaction_order_id,
 
         IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Booking', 
             transaction_arch_logs_union_transactions.owner_id,
@@ -49,16 +47,15 @@ union_transaction_rename_column_and_column_value AS (
 
         COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type,
 
-        transaction_arch_logs_union_transactions.receiver_user_id   AS company_id,
+        transaction_arch_logs_union_transactions.receiver_user_id                                           AS company_id,
-
         transaction_arch_logs_union_transactions.recipient_bank_id,
 
-        COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string) AS payment_method,
+        COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string)     AS payment_method,
 
         transaction_arch_logs_union_transactions.payment_reference,
         transaction_arch_logs_union_transactions.bill_number,
 
-        transaction_arch_logs_union_transactions.base_value,
+        ( transaction_arch_logs_union_transactions.base_value - transaction_arch_logs_union_transactions.base_service_charge )  AS base_value,
         transaction_arch_logs_union_transactions.quote_value,
 
         transaction_arch_logs_union_transactions.base_currency_id,
@@ -68,14 +65,16 @@ union_transaction_rename_column_and_column_value AS (
         transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate,
         transaction_arch_logs_union_transactions.base_tax,
         transaction_arch_logs_union_transactions.base_service_charge,
-        transaction_arch_logs_union_transactions.expired_datetime,
+        transaction_arch_logs_union_transactions.base_value                                 AS total_base_value,
         
-        COALESCE(status.name, transaction_arch_logs_union_transactions.status::string)   AS status,
+        COALESCE(status.name, transaction_arch_logs_union_transactions.status::string)      AS status,
 
+        transaction_arch_logs_union_transactions.expired_datetime,
         transaction_arch_logs_union_transactions.deleted_datetime,
         transaction_arch_logs_union_transactions.created_datetime,
         transaction_arch_logs_union_transactions.updated_datetime,
-        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
+
+        '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}'   AS _dbt_ran_datetime
 
     FROM transaction_arch_logs_union_transactions
 
@@ -88,11 +87,11 @@ union_transaction_rename_column_and_column_value AS (
     LEFT JOIN status
         ON (transaction_arch_logs_union_transactions.status = status.id)
 
-    
     WHERE 
         transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Booking' 
         AND 
         transaction_arch_logs_union_transactions.transaction_type = '1' --PAYMENT
+        
 ),
 
 -- FINAL
@@ -121,6 +120,7 @@ final__stg_exchange__transaction_order_logs AS (
         base_to_quote_currency_exchange_rate,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         expired_datetime,

models/staging/exchange/stg_exchange__transaction_orders.sql

@@ -1,7 +1,3 @@
--- change name for final__stg_exchange__transaction_costs
--- @todo
--- @yam
-
 -- IMPORTS
 WITH transactions AS (
     SELECT * FROM {{ ref('base_exchange__transactions') }}
@@ -25,37 +21,32 @@ status AS (
 transactions_join_transaction_types_payment_methods_status AS (
 
     SELECT
-        transactions.transaction_id AS transaction_order_id,
+        transactions.transaction_id     AS transaction_order_id,
-        IFF(transactions.owner_type = 'App\\Models\\Booking', transactions.owner_id, null)  AS booking_id,
+        transactions.receiver_user_id   AS company_id,
-
-        COALESCE(transaction_types.name, transactions.transaction_type::string)   AS transaction_type,
-
-        transactions.receiver_user_id           AS company_id,
-
         transactions.recipient_bank_id  AS bank_id,
-
+        COALESCE(payment_methods.name, transactions.payment_method::string)                 AS payment_method,
-        COALESCE(payment_methods.name, transactions.payment_method::string)   AS payment_method,
+        COALESCE(transaction_types.name, transactions.transaction_type::string)             AS transaction_type,
+        IFF(transactions.owner_type = 'App\\Models\\Booking', transactions.owner_id, null)  AS booking_id,
 
         transactions.payment_reference,
         transactions.bill_number,
-
-        transactions.base_value,
-        transactions.quote_value,
-
         transactions.base_currency_id,
         transactions.quote_currency_id,
-
         transactions.base_to_quote_currency_exchange_rate,
+
         transactions.base_tax,
         transactions.base_service_charge,
+        ( transactions.base_value - transactions.base_service_charge )      AS base_value, -- Tax not implemented in CIEF yet.
+        transactions.base_value                                             AS total_base_value,
+        transactions.quote_value,
+
+        COALESCE(status.name, transactions.status::string)                  AS status,
 
         transactions.expired_datetime,
-        
-        COALESCE(status.name, transactions.status::string)   AS status,        
-
         transactions.deleted_datetime,
         transactions.created_datetime,
         transactions.updated_datetime,
+
         '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
 
     FROM transactions
@@ -69,17 +60,16 @@ transactions_join_transaction_types_payment_methods_status AS (
     LEFT JOIN status
         ON (transactions.status = status.id)
 
-    
     WHERE 
         transactions.owner_type = 'App\\Models\\Booking' 
-        AND 
+        AND
         transactions.transaction_type = '1' --PAYMENT
 
 ),
 
 
 -- FINAL
-final__stg_exchange__transaction_costs AS (
+final__stg_exchange__transaction_orders AS (
 
     SELECT
         -- ids
@@ -99,10 +89,11 @@ final__stg_exchange__transaction_costs AS (
 
         -- measures
         base_value,
-        quote_value,
         base_to_quote_currency_exchange_rate,
+        quote_value,
         base_tax,
         base_service_charge,
+        total_base_value,
 
         -- date/times
         expired_datetime,
@@ -115,6 +106,6 @@ final__stg_exchange__transaction_costs AS (
 
     FROM transactions_join_transaction_types_payment_methods_status
 
-) 
+)
 
-SELECT * FROM final__stg_exchange__transaction_costs
+SELECT * FROM final__stg_exchange__transaction_orders