diff --git a/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql b/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql index 02a701d..d1eccd1 100644 --- a/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql +++ b/models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql @@ -1,21 +1,38 @@ -- VARIABLES -{% set order_status = dbt_utils.get_column_values( - table=ref('stg_exchange__transaction_order_logs'), - column='status') +{% set cost_status = dbt_utils.get_column_values( + table=ref('stg_exchange__transaction_cost_logs'), + column='status') %} -- IMPORTS WITH transaction_cost_logs AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} + SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} ), transaction_costs AS ( SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} ), +currency_vendor_payment_proof_documents AS ( + SELECT * FROM {{ ref('stg_exchange__currency_vendor_payment_proof') }} +), + +temp_bookings AS ( + SELECT * FROM {{ ref('stg_exchange__bookings') }} +), + +temp_orders AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_orders') }} +), + -- LOGIC +/* +Some order may have multiple same cost status. This is due to the accounting department adjusting the exchange rate. +This is particularly true for the 1688 payment, as exchange rate is amended at a later stage instead of during the initial order placement stage. +Thus, it is to be assumed that the datetime from first repeat occurence as the true datetime. +*/ transaction_cost_status_datetime AS ( SELECT @@ -23,8 +40,8 @@ transaction_cost_status_datetime AS ( company_id, MIN(created_datetime) AS cost_created_datetime, - {%- for status in order_status %} - MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {%- for status in cost_status %} + MIN(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} {% endfor %} FROM transaction_cost_logs @@ -61,19 +78,28 @@ join_cost_and_logs AS ( transaction_cost_status_datetime.company_id, transaction_cost_status_datetime.cost_created_datetime, - transaction_cost_status_datetime.cost_completed_datetime, transaction_cost_status_datetime.cost_pending_submission_datetime, - transaction_cost_status_datetime.cost_approved_datetime, transaction_cost_status_datetime.cost_pending_verification_datetime, - transaction_cost_status_datetime.cost_rejected_datetime, - transaction_cost_status_datetime.cost_suspended_datetime, - transaction_cost_status_datetime.cost_expired_datetime + transaction_cost_status_datetime.cost_approved_datetime, + transaction_cost_status_datetime.cost_completed_datetime, + + currency_vendor_payment_proof_documents.document_type AS cost_document_type, + currency_vendor_payment_proof_documents.status AS cost_document_status, + currency_vendor_payment_proof_documents.created_datetime AS cost_document_created_datetime, + currency_vendor_payment_proof_documents.updated_datetime AS cost_document_updated_datetime, + + ROW_NUMBER() OVER (PARTITION BY transaction_costs.transaction_cost_id ORDER BY cost_document_created_datetime, cost_document_updated_datetime) AS row_number_index FROM transaction_costs LEFT JOIN transaction_cost_status_datetime ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id) + LEFT JOIN currency_vendor_payment_proof_documents + ON (transaction_costs.transaction_cost_id = currency_vendor_payment_proof_documents.transaction_cost_id) + + QUALIFY + row_number_index = 1 ), remove_deleted_expired_costs AS ( @@ -95,9 +121,7 @@ remove_system_error_duplicate_row AS ( SELECT *, - ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index, - - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index FROM remove_deleted_expired_costs @@ -106,6 +130,78 @@ remove_system_error_duplicate_row AS ( ), +/* +Temporary CTE to address the issue of 0.4% service charge for 1688 payments +Affected supplier_company_id from 2729 and 4548 +Affected orders are from 3rd-Apr-2023 onwards +Service charge is only applied onto the CNY +*/ +temp_cte_1688_cleaning AS ( + + SELECT + remove_system_error_duplicate_row.transaction_cost_id, + remove_system_error_duplicate_row.transaction_order_id, + remove_system_error_duplicate_row.supplier_company_id, + remove_system_error_duplicate_row.company_id, + remove_system_error_duplicate_row.bank_id, + remove_system_error_duplicate_row.base_currency_id, + remove_system_error_duplicate_row.quote_currency_id, + remove_system_error_duplicate_row.transaction_type, + remove_system_error_duplicate_row.payment_method, + remove_system_error_duplicate_row.status, + remove_system_error_duplicate_row.payment_reference, + remove_system_error_duplicate_row.bill_number, + remove_system_error_duplicate_row.cost_document_type, + remove_system_error_duplicate_row.cost_document_status, + remove_system_error_duplicate_row.base_value, + remove_system_error_duplicate_row.quote_value, + remove_system_error_duplicate_row.base_to_quote_currency_exchange_rate, + remove_system_error_duplicate_row.base_tax, + remove_system_error_duplicate_row.expired_datetime, + remove_system_error_duplicate_row.deleted_datetime, + remove_system_error_duplicate_row.created_datetime, + remove_system_error_duplicate_row.updated_datetime, + remove_system_error_duplicate_row.cost_created_datetime, + remove_system_error_duplicate_row.cost_pending_submission_datetime, + remove_system_error_duplicate_row.cost_pending_verification_datetime, + remove_system_error_duplicate_row.cost_approved_datetime, + remove_system_error_duplicate_row.cost_completed_datetime, + remove_system_error_duplicate_row.cost_document_created_datetime, + remove_system_error_duplicate_row.cost_document_updated_datetime, + + temp_bookings.service_type, + + CASE + WHEN + remove_system_error_duplicate_row.supplier_company_id IN (2729, 4548) + -- 2729 = HCK Global + -- 4548 = Power Progress + AND + temp_bookings.service_type = '1688 PAYMENT' + AND + cost_created_datetime >= '2023-04-03' + THEN + -- Formula used for service charge in MYR: CNY * 0.4% / exchange rate + ROUND(DIV0( + remove_system_error_duplicate_row.quote_value * ( 0.4 / 100 ), + remove_system_error_duplicate_row.base_to_quote_currency_exchange_rate + ),4) + ELSE + remove_system_error_duplicate_row.base_service_charge + END AS base_service_charge, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM remove_system_error_duplicate_row + + LEFT JOIN temp_orders + ON (remove_system_error_duplicate_row.transaction_order_id = temp_orders.transaction_order_id) + + LEFT JOIN temp_bookings + ON (temp_orders.booking_id = temp_bookings.booking_id) + +), + -- FINAL final__int_exchange__transaction_cost_get_latest_cost_ids AS ( @@ -126,6 +222,8 @@ final__int_exchange__transaction_cost_get_latest_cost_ids AS ( status, payment_reference, bill_number, + cost_document_type, + cost_document_status, -- measures base_value, @@ -140,19 +238,18 @@ final__int_exchange__transaction_cost_get_latest_cost_ids AS ( created_datetime, updated_datetime, cost_created_datetime, - cost_completed_datetime, cost_pending_submission_datetime, - cost_approved_datetime, cost_pending_verification_datetime, - cost_rejected_datetime, - cost_suspended_datetime, - cost_expired_datetime, - + cost_approved_datetime, + cost_completed_datetime, + cost_document_created_datetime, + cost_document_updated_datetime, + -- metadata _dbt_ran_datetime - FROM remove_system_error_duplicate_row - + FROM temp_cte_1688_cleaning + ) SELECT * FROM final__int_exchange__transaction_cost_get_latest_cost_ids \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__company_conversion_funnels.sql b/models/marts/reporting/rep_exchange__company_conversion_funnels.sql index 3150e13..7864b94 100644 --- a/models/marts/reporting/rep_exchange__company_conversion_funnels.sql +++ b/models/marts/reporting/rep_exchange__company_conversion_funnels.sql @@ -347,13 +347,32 @@ generate_cumulative_count AS ( FROM union_all_activities_flag_migrated_company ORDER BY activity_datetime + ), +-- Include additional details for companies +enhance_company_details AS ( + + SELECT + generate_cumulative_count.*, + + companies.company_marking_id, + companies.company_created_datetime AS company_register_datetime + + FROM generate_cumulative_count + + LEFT JOIN companies + ON (generate_cumulative_count.company_id = companies.company_id) + +), + + -- FINAL final__rep_exchange__company_conversion_funnels AS ( SELECT -- ids company_id, + company_marking_id, -- dimensions is_data_cleansing_generate_row, @@ -372,6 +391,7 @@ final__rep_exchange__company_conversion_funnels AS ( cumulative_first_time_order_completed_activity, cumulative_repeat_completed_order_after_30days_activity, cumulative_repeat_completed_order_after_60days_activity, + include_migrated_cumulative_register_activity, include_migrated_cumulative_email_verified_activity, include_migrated_cumulative_identity_document_uploaded_activity, @@ -383,13 +403,13 @@ final__rep_exchange__company_conversion_funnels AS ( include_migrated_cumulative_repeat_completed_order_after_60days_activity, -- date/times + company_register_datetime, activity_datetime, -- metadata _dbt_ran_datetime - FROM generate_cumulative_count + FROM enhance_company_details ) - SELECT * FROM final__rep_exchange__company_conversion_funnels \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql index 62836db..95326d1 100644 --- a/models/marts/reporting/rep_exchange__daily_orders.sql +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -1,26 +1,33 @@ -- VARIABLES -{% set var_service_type = [ - '1688 PAYMENT', - '1 DAY TRANSFER', - '3 DAYS TRANSFER', - 'Enterprise to Enterprise 公打公'] %} +{% set service_types = dbt_utils.get_column_values( + table=ref('fct_exchange__transaction_orders'), + column='service_type') +%} -{% set var_working_days_including_base_date = [ - 'add_1_working_day_included_base_date', - 'add_1_working_day_included_base_date', - 'add_3_working_day_included_base_date', - 'add_7_working_day_included_base_date'] %} +-- working days mapping for each service type +{% set service_type_working_day_mapping = { + '1 DAY TRANSFER': 1, + '1688 PAYMENT': 1, + '3 DAYS TRANSFER': 3, + 'ENTERPRISE TO ENTERPRISE 公打公': 7 +} %} -{% set var_working_days_excluding_base_date = [ - 'add_1_working_day_excluded_base_date', - 'add_1_working_day_excluded_base_date', - 'add_3_working_day_excluded_base_date', - 'add_7_working_day_excluded_base_date'] %} +{% set service_type_work_day_dict={} %} + +{% for service_type, working_day in service_type_working_day_mapping.items() %} + + {% set service_type_work_day_dict = service_type_work_day_dict.update( + {service_type: { + 'include_base': 'add_' ~ working_day ~ '_working_day_included_base_date', + 'exclude_base': 'add_' ~ working_day ~ '_working_day_excluded_base_date' + }}) %} + +{% endfor %} -- IMPORT WITH orders AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} ), companies AS ( @@ -39,67 +46,47 @@ users AS ( SELECT * FROM {{ ref('dim_exchange__users') }} ), ---LOGIC + +-- LOGIC fct_and_dim_joins AS ( + SELECT orders.booking_id, orders.booking_marking_id, + orders.order_id, + orders.cost_id, orders.company_id, - orders.transaction_status, - orders.document_status, - - companies.name AS company_name, - companies.company_marking_id AS company_marking_id, - companies.autocount_id AS company_autocount_id, - companies.company_type AS company_type, - companies.business_type AS company_business_type, - companies.exchange_rate_segment AS company_exchange_rate_segment, - companies.is_migrated_company AS is_migrated_company, - companies.country_name AS company_country_name, - companies.state_name AS company_state_name, - companies.district_name AS company_district_name, - companies.postcode AS company_postcode, - companies.has_wallet AS company_has_wallet, - companies.latitude AS company_latitude, - companies.longitude AS company_longitude, - companies.company_created_datetime, - companies.company_lifetime_value, - companies.m_score_lifetime AS company_m_score_lifetime, - + orders.supplier_company_id, orders.user_id, orders.bank_id, - - fix_currencies.currency_name AS fix_currency_name, - - quote_currencies.currency_name AS quote_currency_name, - - base_currencies.currency_name AS base_currency_name, - + orders.currency_vendor_bank_id, + orders.transaction_status, + orders.document_status, orders.service_type, orders.booking_status, + orders.order_status, + orders.cost_status, + orders.order_payment_method, + orders.order_payment_reference, + orders.order_bill_number, + orders.cost_transaction_type, + orders.cost_payment_method, + orders.cost_payment_reference, + orders.cost_bill_number, + orders.purchase_order_number, + orders.invoice_number, orders.is_first_time_booking_company, orders.is_first_time_booking_company_completed, orders.is_first_time_booking_user, orders.is_first_time_booking_user_completed, - orders.estimate_booking_base_to_quote_currency_exchange_rate, - orders.estimate_booking_quote_value, - orders.estimate_booking_base_value, - orders.estimate_booking_value_rm, - orders.booking_created_datetime, - orders.booking_approved_datetime, - orders.booking_completed_datetime, - orders.booking_suspended_datetime, - - orders.order_id, - orders.order_payment_method, - orders.order_status, - orders.order_payment_reference, - orders.order_bill_number, orders.is_first_time_order_company, orders.is_first_time_order_company_completed, orders.is_first_time_order_user, orders.is_first_time_order_user_completed, - orders.order_base_to_quote_currency_exchange_rate, + orders.estimate_booking_quote_value, + orders.estimate_booking_base_value, + orders.estimate_booking_value_rm, + orders.estimate_booking_base_to_quote_currency_exchange_rate, orders.order_base_value, orders.order_base_service_charge, orders.order_base_tax, @@ -108,100 +95,36 @@ fct_and_dim_joins AS ( orders.order_service_charge_rm, orders.order_tax_rm, orders.total_order_value_rm, - order_created_dates.day_of_year AS order_created_day_number, - order_created_dates.first_day_of_week AS order_created_week, - order_created_dates.week_of_year AS order_created_week_number, - order_created_dates.first_day_of_month AS order_created_month, - order_created_dates.day_of_month AS order_created_month_day_number, - order_created_dates.month_actual AS order_created_month_number, - order_created_dates.year_actual AS order_created_year, + orders.order_base_to_quote_currency_exchange_rate, - orders.order_created_datetime, - orders.order_pending_submission_datetime, - orders.order_pending_verification_datetime, - - -- Estimated delivery datetime based on CIEF internal SLA (cutoff time 4pm) - CASE - {% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %} - - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' 23:59:59 +08:00') - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' 23:59:59 +08:00') - - {% endfor %} - - ELSE NULL - - END AS estimated_order_delivery_datetime_website_sla, - - --- Estimated delivery datetime based on customer expectation - CASE - {% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %} - - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 - THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 - THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) - - {% endfor %} - - ELSE NULL - - END AS estimated_order_delivery_datetime_customer_expectation, - - -- On time delivery boolean (CIEF internal SLA) - CASE - -- On-time boolean is null when order is not complete - WHEN account_verified_payment_datetime IS NULL THEN NULL - WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1 - ELSE 0 - - END AS is_on_time_delivery_website_sla, - - -- On time delivery boolean (Customer Expectation) - CASE - -- On-time boolean is null when order is not complete - WHEN account_verified_payment_datetime IS NULL THEN NULL - WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1 - ELSE 0 - - END AS is_on_time_delivery_customer_expectation, - - orders.order_approved_datetime, - orders.order_completed_datetime, - orders.order_rejected_datetime, - orders.order_suspended_datetime, - orders.order_expected_expired_datetime, - orders.order_expired_datetime, - - orders.cost_id, - orders.supplier_company_id, - supplier_companies.name AS supplier_company_name, - orders.currency_vendor_bank_id, - orders.cost_transaction_type, - orders.cost_payment_method, - orders.cost_status, - orders.cost_payment_reference, - orders.cost_bill_number, orders.cost_base_value, orders.cost_quote_value, - orders.cost_base_to_quote_currency_exchange_rate, orders.cost_base_tax, orders.cost_base_service_charge, orders.cost_value_rm, orders.cost_tax_rm, orders.cost_service_charge_rm, orders.total_cost_value_rm, - orders.purchase_order_number, - orders.invoice_number, + orders.cost_base_to_quote_currency_exchange_rate, + orders.booking_created_datetime, + orders.booking_approved_datetime, + orders.booking_completed_datetime, + orders.booking_suspended_datetime, + orders.order_created_datetime, + orders.order_pending_submission_datetime, + orders.order_pending_verification_datetime, + orders.order_approved_datetime, + orders.order_completed_datetime, + orders.order_rejected_datetime, + orders.order_suspended_datetime, + orders.order_expected_expired_datetime, + orders.order_expired_datetime, + orders.order_deleted_datetime, orders.cost_created_datetime, orders.cost_pending_submission_datetime, orders.cost_pending_verification_datetime, orders.cost_approved_datetime, orders.cost_completed_datetime, - orders.cost_rejected_datetime, - orders.customer_made_booking_datetime, orders.customer_placed_order_datetime, orders.customer_made_payment_datetime, @@ -215,6 +138,84 @@ fct_and_dim_joins AS ( orders.account_rejected_payment_datetime, orders.next_order_created_datetime, + companies.name AS company_name, + companies.company_marking_id AS company_marking_id, + companies.autocount_id AS company_autocount_id, + companies.company_type AS company_type, + companies.business_type AS company_business_type, + companies.exchange_rate_segment AS company_exchange_rate_segment, + companies.is_migrated_company AS is_migrated_company, + companies.country_name AS company_country_name, + companies.state_name AS company_state_name, + companies.district_name AS company_district_name, + companies.postcode AS company_postcode, + companies.has_wallet AS company_has_wallet, + companies.latitude AS company_latitude, + companies.longitude AS company_longitude, + companies.m_score_lifetime AS company_m_score_lifetime, + companies.company_created_datetime, + companies.company_lifetime_value, + + fix_currencies.currency_name AS fix_currency_name, + + quote_currencies.currency_name AS quote_currency_name, + + base_currencies.currency_name AS base_currency_name, + + order_created_dates.day_of_year AS order_created_day_number, + order_created_dates.first_day_of_week AS order_created_week, + order_created_dates.week_of_year AS order_created_week_number, + order_created_dates.first_day_of_month AS order_created_month, + order_created_dates.day_of_month AS order_created_month_day_number, + order_created_dates.month_actual AS order_created_month_number, + order_created_dates.year_actual AS order_created_year, + + supplier_companies.name AS supplier_company_name, + + -- Estimated delivery datetime based on CIEF internal SLA (cutoff time 4pm) + CASE + {% for (service_type, working_day) in service_type_work_day_dict.items() %} + + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' 23:59:59 +08:00') + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' 23:59:59 +08:00') + + {% endfor %} + + ELSE NULL + END AS estimated_order_delivery_datetime_website_sla, + + --- Estimated delivery datetime based on customer expectation + CASE + {% for (service_type, working_day) in service_type_work_day_dict.items() %} + + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16 + THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING ) + + {% endfor %} + + ELSE NULL + END AS estimated_order_delivery_datetime_customer_expectation, + + -- On time delivery boolean (CIEF internal SLA) + CASE + -- On-time boolean is null when order is not complete + WHEN account_verified_payment_datetime IS NULL THEN NULL + WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1 + ELSE 0 + END AS is_on_time_delivery_website_sla, + + -- On time delivery boolean (Customer Expectation) + CASE + -- On-time boolean is null when order is not complete + WHEN account_verified_payment_datetime IS NULL THEN NULL + WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1 + ELSE 0 + END AS is_on_time_delivery_customer_expectation, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM orders @@ -236,10 +237,12 @@ fct_and_dim_joins AS ( LEFT JOIN currencies AS base_currencies ON (orders.base_currency_id = base_currencies.currency_id) + ), --FINAL final__rep_exchange__daily_orders AS ( + SELECT -- ids order_id, @@ -341,12 +344,12 @@ final__rep_exchange__daily_orders AS ( order_rejected_datetime, order_suspended_datetime, order_expired_datetime, + order_deleted_datetime, cost_created_datetime, cost_pending_submission_datetime, cost_pending_verification_datetime, cost_approved_datetime, cost_completed_datetime, - cost_rejected_datetime, booking_created_datetime, booking_approved_datetime, booking_completed_datetime, diff --git a/models/marts/reporting/rep_exchange__sales_lead_conversions.sql b/models/marts/reporting/rep_exchange__sales_lead_conversions.sql index e5bf4e5..0c6e426 100644 --- a/models/marts/reporting/rep_exchange__sales_lead_conversions.sql +++ b/models/marts/reporting/rep_exchange__sales_lead_conversions.sql @@ -15,6 +15,7 @@ transactions AS ( SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} ), + -- LOGIC join_booking_transaction AS ( @@ -49,8 +50,8 @@ join_booking_transaction AS ( transactions.order_id AS transaction_order_id, transactions.order_transaction_type, transactions.order_payment_method, + transactions.transaction_status, transactions.total_order_value_rm AS total_value_rm, - transactions.order_status AS transaction_order_status, transactions.order_created_datetime AS transaction_created_datetime, transactions.order_approved_datetime AS transaction_approved_datetime, transactions.order_completed_datetime AS transaction_completed_datetime @@ -88,7 +89,7 @@ data_enrich_grouped AS ( -- Count of orders placed after contacted by sales team SUM( CASE - WHEN transaction_order_status = 'COMPLETED' AND contact_status = 'Contacted' AND transaction_created_datetime > sales_team_contacted_datetime + WHEN transaction_status IN ('UPLOADED_BANK_SLIP') AND contact_status IN ('Contacted') AND transaction_created_datetime > sales_team_contacted_datetime THEN 1 ELSE 0 END @@ -110,6 +111,7 @@ data_enrich_grouped AS ( -- Boolean - Bookings and orders made AFTER contacted datetime and successful contact CASE WHEN MAX( booking_created_datetime ) > sales_team_contacted_datetime AND contact_status = 'Contacted' THEN '1' + WHEN MAX( transaction_created_datetime ) > sales_team_contacted_datetime AND contact_status = 'Contacted' THEN '1' ELSE '0' END AS has_booked_after_contacted, CASE @@ -127,7 +129,7 @@ data_enrich_grouped AS ( MIN( CASE WHEN transaction_created_datetime < sales_team_contacted_datetime - AND transaction_order_status NOT IN ('SUSPENDED', 'REJECTED') + AND transaction_status NOT IN ('ORDER_REJECTED', 'ORDER_SUSPENDED') THEN TIMEDIFF(minute, transaction_created_datetime, sales_team_contacted_datetime) / 60 END ) AS hours_last_ordered_before_contacted, @@ -142,7 +144,7 @@ data_enrich_grouped AS ( MIN( CASE WHEN ( sales_team_contacted_datetime < transaction_created_datetime ) AND contact_status = 'Contacted' - AND transaction_order_status NOT IN ('SUSPENDED', 'REJECTED') + AND transaction_status NOT IN ('ORDER_REJECTED', 'ORDER_SUSPENDED') THEN TIMEDIFF(minute, sales_team_contacted_datetime, transaction_created_datetime) / 60 END ) AS hours_first_ordered_after_contacted, @@ -157,7 +159,7 @@ data_enrich_grouped AS ( CASE WHEN sales_team_contacted_datetime > transaction_created_datetime THEN total_value_rm ELSE 0 END ) AS total_value_rm_before_contact, SUM( - CASE WHEN transaction_order_status IN ('APPROVED', 'COMPLETED') + CASE WHEN transaction_status IN ('UPLOADED_BANK_SLIP') AND sales_team_contacted_datetime > transaction_created_datetime THEN total_value_rm ELSE 0 END ) AS total_value_rm_approved_completed_before_contact, @@ -167,7 +169,7 @@ data_enrich_grouped AS ( AND contact_status = 'Contacted' THEN total_value_rm ELSE 0 END ) AS total_value_rm_after_contact, SUM( - CASE WHEN transaction_order_status IN ('APPROVED', 'COMPLETED') + CASE WHEN transaction_status IN ('UPLOADED_BANK_SLIP') AND sales_team_contacted_datetime < transaction_created_datetime AND contact_status = 'Contacted' THEN total_value_rm ELSE 0 END ) AS total_value_rm_approved_completed_after_contact, @@ -241,6 +243,7 @@ data_enrich_grouped AS ( exchange_registered_datetime ), + -- FINAL final__rep_exchange__sales_lead_conversions AS ( diff --git a/models/marts/warehouse/dim__dates.py b/models/marts/warehouse/dim__dates.py index d9d0a55..91a7dad 100644 --- a/models/marts/warehouse/dim__dates.py +++ b/models/marts/warehouse/dim__dates.py @@ -17,7 +17,7 @@ from datetime import datetime as dt, timedelta # Function to check if a given date is a weekend [Saturday(5) or Sunday(6)] def is_weekend(date): - return (date.weekday() == 5) | (date.weekday() == 6) + return date.weekday() in (5, 6) # Function to compute the in advance working days @@ -28,14 +28,14 @@ def compute_working_days(start_date, num_working_days, holiday_list, after_cut_o counter = 0 # For dates on weekend or holiday or after cut off time, additional 1 working day to the loop - if (after_cut_off == 1) | is_weekend(start_date) | (start_date in holiday_list): + if (after_cut_off == 1) or is_weekend(start_date) or (start_date in holiday_list): counter -= 1 # Loop to increase n number of working days, if weekend/holiday, skip counter while counter < num_working_days: start_date = start_date + timedelta(days=1) - if is_weekend(start_date) | (start_date in holiday_list): + if is_weekend(start_date) or (start_date in holiday_list): continue counter += 1 @@ -45,36 +45,37 @@ def compute_working_days(start_date, num_working_days, holiday_list, after_cut_o # Main function def model(dbt, session): - + # Setting configuration dbt.config(materialized="table", packages = ["pandas"]) # Import data from upstream dbt model - df_date = dbt.ref("int__dates") + sp_df_date = dbt.ref("int__dates") - # Extract holiday dates into a list - df_work = df_date.to_pandas() - df_new = df_work[df_work["IS_COMPANY_HOLIDAY"] == 1] - df_holiday_list = df_new["DATE_DAY"].tolist() + # Filter holiday dates into a dataframe + sp_df_filter = sp_df_date.filter(sp_df_date['IS_COMPANY_HOLIDAY'] == 1) + sp_df_holiday_date = sp_df_filter.select('DATE_DAY') - # Apply function to df - df_work["ADD_1_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 0), axis=1) - df_work["ADD_2_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 0), axis=1) - df_work["ADD_3_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 0), axis=1) - df_work["ADD_7_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 0), axis=1) - df_work["ADD_30_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 0), axis=1) - df_work["ADD_90_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 0), axis=1) - df_work["ADD_365_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 0), axis=1) + # Convert snowpark dataframe to pandas dataframe + pd_df_date = sp_df_date.to_pandas() + pd_df_holiday_date = sp_df_holiday_date.to_pandas() + + # Store holiday dates in a list + holiday_list = pd_df_holiday_date['DATE_DAY'].tolist() - # For orders after cut off time 4pm - # Affects only the weekdays - df_work["ADD_1_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 1), axis=1) - df_work["ADD_2_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 1), axis=1) - df_work["ADD_3_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 1), axis=1) - df_work["ADD_7_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 1), axis=1) - df_work["ADD_30_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 1), axis=1) - df_work["ADD_90_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 1), axis=1) - df_work["ADD_365_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 1), axis=1) + # Applying add working days function to df + add_working_days = [1, 2, 3, 7, 30, 90 ,365] - return df_work \ No newline at end of file + for working_day in add_working_days: + for after_cut_off in (0, 1): + + if after_cut_off == 0: + col_name = f'ADD_{working_day}_WORKING_DAY_INCLUDED_BASE_DATE' + else: + col_name = f'ADD_{working_day}_WORKING_DAY_EXCLUDED_BASE_DATE' + + # Apply function and append new calculated columns in the dataframe + pd_df_date[col_name] = pd_df_date['DATE_DAY'].apply(lambda start_date: compute_working_days(start_date, working_day, holiday_list, after_cut_off)) + + return pd_df_date \ No newline at end of file diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql index 2e59b19..f479946 100644 --- a/models/marts/warehouse/fct_exchange__transaction_orders.sql +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -293,6 +293,8 @@ transaction_costs_lists AS ( transaction_type, payment_method, status, + cost_document_type, + cost_document_status, payment_reference, bill_number, base_value, @@ -322,14 +324,13 @@ transaction_costs_lists AS ( deleted_datetime, created_datetime, updated_datetime, + cost_document_created_datetime, + cost_document_updated_datetime, cost_created_datetime, cost_completed_datetime, cost_pending_submission_datetime, cost_approved_datetime, - cost_pending_verification_datetime, - cost_rejected_datetime, - cost_suspended_datetime, - cost_expired_datetime + cost_pending_verification_datetime FROM transaction_costs @@ -444,12 +445,15 @@ bookings_orders_costs_join AS ( transaction_costs_lists.tax_rm AS cost_tax_rm, transaction_costs_lists.service_charge_rm AS cost_service_charge_rm, transaction_costs_lists.total_value_rm AS total_cost_value_rm, + transaction_costs_lists.cost_document_type, + transaction_costs_lists.cost_document_status, transaction_costs_lists.cost_created_datetime, transaction_costs_lists.cost_pending_submission_datetime, transaction_costs_lists.cost_pending_verification_datetime, transaction_costs_lists.cost_approved_datetime, transaction_costs_lists.cost_completed_datetime, - transaction_costs_lists.cost_rejected_datetime + transaction_costs_lists.cost_document_created_datetime, + transaction_costs_lists.cost_document_updated_datetime FROM transaction_orders_lists @@ -554,7 +558,10 @@ renaming_datetime_columns AS ( order_approved_datetime AS account_verified_payment_datetime, order_completed_datetime AS first_white_form_generated_datetime, cost_pending_verification_datetime AS last_white_form_generated_datetime, - COALESCE(cost_approved_datetime, cost_completed_datetime) AS operation_uploaded_bank_slip_datetime, + COALESCE( + cost_document_created_datetime, + cost_approved_datetime, + cost_completed_datetime) AS operation_uploaded_bank_slip_datetime, purchase_order_created_datetime AS first_purchase_order_uploaded_datetime, purchase_order_updated_datetime AS last_purchase_order_uploaded_datetime, invoice_created_datetime AS operation_generated_invoice_datetime, @@ -608,6 +615,8 @@ final_fct_exchange__new_orders AS ( cost_status, cost_payment_reference, cost_bill_number, + cost_document_type, + cost_document_status, purchase_order_status, purchase_order_number, invoice_status, @@ -656,12 +665,13 @@ final_fct_exchange__new_orders AS ( order_suspended_datetime, order_expired_datetime, + cost_document_created_datetime, + cost_document_updated_datetime, cost_created_datetime, cost_pending_submission_datetime, cost_pending_verification_datetime, cost_approved_datetime, cost_completed_datetime, - cost_rejected_datetime, customer_made_booking_datetime, customer_placed_order_datetime, diff --git a/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql b/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql index d6ec6c6..2a9dbf5 100644 --- a/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql +++ b/models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql @@ -6,15 +6,20 @@ WITH lead_activations AS ( -- FINAL final__fct_googlesheet__sales_exchange_lead_activations AS ( + SELECT + -- id sales_team_lead_activation_id, company_marking_id, + + -- dimension agent_name, contact_name, contact_number, contact_email, contact_method, contact_status, + sales_team_remark, -- measures @@ -24,7 +29,6 @@ final__fct_googlesheet__sales_exchange_lead_activations AS ( exchange_registered_datetime, -- metadata - sales_team_remark, '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM lead_activations diff --git a/models/staging/exchange/base/base_exchange__booking_arch_logs.sql b/models/staging/exchange/base/base_exchange__booking_arch_logs.sql index fce4089..dc517f0 100644 --- a/models/staging/exchange/base/base_exchange__booking_arch_logs.sql +++ b/models/staging/exchange/base/base_exchange__booking_arch_logs.sql @@ -5,29 +5,43 @@ WITH booking_arch_logs AS ( -- LOGIC -booking_arch_logs_rename AS ( - +-- Exclude service type 'Internal Purchase' +exclude_service_type AS ( + SELECT - booking_arch_logs.id AS booking_log_id, - booking_arch_logs.booking_id, - booking_arch_logs.company_id, - booking_arch_logs.marking AS booking_marking_id, - booking_arch_logs.service_id AS service_type, - booking_arch_logs.bank_id, - booking_arch_logs.fix_amount AS fix_value, - booking_arch_logs.fix_currency_id, - booking_arch_logs.convertible_currency_id AS quote_currency_id, - booking_arch_logs.conversion_currency_id AS base_currency_id, - booking_arch_logs.status, - booking_arch_logs.deleted_at AS deleted_datetime, - booking_arch_logs.created_at AS created_datetime, - booking_arch_logs.updated_at As updated_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + * FROM booking_arch_logs + WHERE service_id NOT IN (7) -- 7 : INTERNAL PURCHASE + ), +booking_arch_logs_rename AS ( + + SELECT + exclude_service_type.id AS booking_log_id, + exclude_service_type.booking_id, + exclude_service_type.company_id, + exclude_service_type.marking AS booking_marking_id, + exclude_service_type.service_id AS service_type, + exclude_service_type.bank_id, + exclude_service_type.fix_amount AS fix_value, + exclude_service_type.fix_currency_id, + exclude_service_type.convertible_currency_id AS quote_currency_id, + exclude_service_type.conversion_currency_id AS base_currency_id, + exclude_service_type.status, + exclude_service_type.deleted_at AS deleted_datetime, + exclude_service_type.created_at AS created_datetime, + exclude_service_type.updated_at As updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM exclude_service_type + +), + + -- FINAL final_base_exchange__booking_arch_logs AS ( diff --git a/models/staging/exchange/base/base_exchange__bookings.sql b/models/staging/exchange/base/base_exchange__bookings.sql index 8e5115a..e9295d3 100644 --- a/models/staging/exchange/base/base_exchange__bookings.sql +++ b/models/staging/exchange/base/base_exchange__bookings.sql @@ -5,28 +5,42 @@ WITH bookings AS ( -- LOGIC -bookings_rename AS ( - +-- Exclude service type 'Internal Purchase' +exclude_service_type AS ( + SELECT - bookings.id AS booking_id, - bookings.company_id, - bookings.marking AS booking_marking_id, - bookings.service_id AS service_type, - bookings.bank_id, - bookings.fix_amount AS fix_value, - bookings.fix_currency_id, - bookings.convertible_currency_id AS quote_currency_id, - bookings.conversion_currency_id AS base_currency_id, - bookings.status, - bookings.deleted_at AS deleted_datetime, - bookings.created_at AS created_datetime, - bookings.updated_at As updated_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + * FROM bookings + WHERE service_id NOT IN (7) -- 7 : INTERNAL PURCHASE + ), +bookings_rename AS ( + + SELECT + exclude_service_type.id AS booking_id, + exclude_service_type.company_id, + exclude_service_type.marking AS booking_marking_id, + exclude_service_type.service_id AS service_type, + exclude_service_type.bank_id, + exclude_service_type.fix_amount AS fix_value, + exclude_service_type.fix_currency_id, + exclude_service_type.convertible_currency_id AS quote_currency_id, + exclude_service_type.conversion_currency_id AS base_currency_id, + exclude_service_type.status, + exclude_service_type.deleted_at AS deleted_datetime, + exclude_service_type.created_at AS created_datetime, + exclude_service_type.updated_at As updated_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM exclude_service_type + +), + + -- FINAL final_base_exchange__bookings AS ( diff --git a/models/staging/exchange/base/base_exchange__currency_rates.sql b/models/staging/exchange/base/base_exchange__currency_rates.sql index 35419c8..c38cde3 100644 --- a/models/staging/exchange/base/base_exchange__currency_rates.sql +++ b/models/staging/exchange/base/base_exchange__currency_rates.sql @@ -5,20 +5,34 @@ WITH currency_rates AS ( -- LOGIC +-- Exclude service type 'Internal Purchase' +exclude_service_type AS ( + + SELECT + * + + FROM currency_rates + + WHERE service_id NOT IN (7) -- 7 : INTERNAL PURCHASE + +), + currency_rates_rename AS ( SELECT - currency_rates.id AS currency_rate_id, - currency_rates.currency_id AS quote_currency_id, - currency_rates.selling AS base_to_quote_currency_exchange_rate, - currency_rates.payment_method_type AS payment_method, - currency_rates.service_id AS service_type, - currency_rates.deleted_at AS deleted_datetime, - currency_rates.created_at AS created_datetime, - currency_rates.updated_at AS updated_datetime, + exclude_service_type.id AS currency_rate_id, + exclude_service_type.currency_id AS quote_currency_id, + exclude_service_type.selling AS base_to_quote_currency_exchange_rate, + exclude_service_type.payment_method_type AS payment_method, + exclude_service_type.service_id AS service_type, + exclude_service_type.deleted_at AS deleted_datetime, + exclude_service_type.created_at AS created_datetime, + exclude_service_type.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - FROM currency_rates + FROM exclude_service_type + ), diff --git a/models/staging/exchange/base/base_exchange__transactions.sql b/models/staging/exchange/base/base_exchange__transactions.sql index d336dcd..910a19a 100644 --- a/models/staging/exchange/base/base_exchange__transactions.sql +++ b/models/staging/exchange/base/base_exchange__transactions.sql @@ -5,34 +5,65 @@ WITH transactions AS ( -- LOGIC +-- Filter transaction id for 'internal purchase' +internal_purchase_transaction_id AS ( + + SELECT + * + + FROM transactions + + WHERE owner_id IN ( + SELECT id FROM {{ source('src_exchange_mysql', 'bookings') }} + WHERE service_id IN (7) -- INTERNAL PURCHASE + ) + AND owner_type = 'App\\Models\\Booking' + +), + +exclude_internal_purchase_bookings AS ( + + SELECT + * + + FROM transactions + + WHERE id NOT IN ( + SELECT id FROM internal_purchase_transaction_id + ) + +), + transactions_rename AS ( SELECT - transactions.id AS transaction_id, - transactions.owner_type, - transactions.owner_id, - transactions.type AS transaction_type, - transactions.issuer AS issuer_user_id, - transactions.receiver AS receiver_user_id, - transactions.recipient_bank_account_id AS recipient_bank_id, - transactions.payment_method, - transactions.payment_reference, - transactions.bill_no AS bill_number, - transactions.amount AS base_value, - transactions.original_amount AS quote_value, - transactions.currency_id AS base_currency_id, - transactions.original_currency_id AS quote_currency_id, - transactions.currency_rate AS base_to_quote_currency_exchange_rate, - transactions.tax AS base_tax, - transactions.service_charge AS base_service_charge, - transactions.expires_on AS expired_datetime, - transactions.status, - transactions.deleted_at AS deleted_datetime, - transactions.created_at AS created_datetime, - transactions.updated_at AS updated_datetime, + exclude_internal_purchase_bookings.id AS transaction_id, + exclude_internal_purchase_bookings.owner_type, + exclude_internal_purchase_bookings.owner_id, + exclude_internal_purchase_bookings.type AS transaction_type, + exclude_internal_purchase_bookings.issuer AS issuer_user_id, + exclude_internal_purchase_bookings.receiver AS receiver_user_id, + exclude_internal_purchase_bookings.recipient_bank_account_id AS recipient_bank_id, + exclude_internal_purchase_bookings.payment_method, + exclude_internal_purchase_bookings.payment_reference, + exclude_internal_purchase_bookings.bill_no AS bill_number, + exclude_internal_purchase_bookings.amount AS base_value, + exclude_internal_purchase_bookings.original_amount AS quote_value, + exclude_internal_purchase_bookings.currency_id AS base_currency_id, + exclude_internal_purchase_bookings.original_currency_id AS quote_currency_id, + exclude_internal_purchase_bookings.currency_rate AS base_to_quote_currency_exchange_rate, + exclude_internal_purchase_bookings.tax AS base_tax, + exclude_internal_purchase_bookings.service_charge AS base_service_charge, + exclude_internal_purchase_bookings.expires_on AS expired_datetime, + exclude_internal_purchase_bookings.status, + exclude_internal_purchase_bookings.deleted_at AS deleted_datetime, + exclude_internal_purchase_bookings.created_at AS created_datetime, + exclude_internal_purchase_bookings.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - FROM transactions + FROM exclude_internal_purchase_bookings + ), @@ -74,6 +105,7 @@ final__base_exchange__transactions AS ( _dbt_ran_datetime FROM transactions_rename + ) SELECT * FROM final__base_exchange__transactions \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__currency_vendor_payment_proof.sql b/models/staging/exchange/stg_exchange__currency_vendor_payment_proof.sql new file mode 100644 index 0000000..fd944a3 --- /dev/null +++ b/models/staging/exchange/stg_exchange__currency_vendor_payment_proof.sql @@ -0,0 +1,71 @@ +/* +One order can have multiple bookings. +Therefore, multiple china bank slip can be uploaded to one order, which is associated to each booking. +Currency_vendor_payment_proof will always have pending_verification as their status. +*/ + +-- IMPORT +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +seed_status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + documents.owner_id AS transaction_cost_id, + documents.document_type, + seed_status.name AS status, + documents.created_datetime, + documents.updated_datetime, + documents.deleted_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents + + LEFT JOIN seed_status + ON (documents.status = seed_status.id) + + WHERE + documents.document_type = 'CURRENCY_VENDOR_PAYMENT_PROOF' + AND + deleted_datetime IS NULL + +), + + +-- FINAL +final__stg_exchange__currency_vendor_payment_proof AS ( + + SELECT + -- ids + document_id, + transaction_cost_id, + + -- dimensions + document_type, + status, + + -- measures + + -- date/times + created_datetime, + updated_datetime, + deleted_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_join_status + +) + +SELECT * FROM final__stg_exchange__currency_vendor_payment_proof \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__customer_payment_proof.sql b/models/staging/exchange/stg_exchange__customer_payment_proof.sql new file mode 100644 index 0000000..024bb83 --- /dev/null +++ b/models/staging/exchange/stg_exchange__customer_payment_proof.sql @@ -0,0 +1,86 @@ +-- IMPORT +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +seed_status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + documents.owner_id AS transaction_order_id, + documents.approver_user_id, + documents.document_type, + seed_status.name AS status, + IFF(seed_status.name = 'REJECTED', documents.updated_datetime, null) AS rejected_datetime, + IFF(seed_status.name = 'APPROVED', documents.updated_datetime, null) AS approved_datetime, + documents.created_datetime, + documents.updated_datetime, + documents.deleted_datetime + + FROM documents + + LEFT JOIN seed_status + ON (documents.status = seed_status.id) + + WHERE + documents.document_type = 'CUSTOMER_PAYMENT_PROOF' + +), + +/* +Few orders in year 2021 has separate rows for +when status = 'Pending Verfication' and when status = 'Approved' +*/ +filter_latest_documents AS ( + + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents_join_status + + QUALIFY + row_number_index = 1 + +), + + +-- FINAL +final__stg_exchange__customer_payment_proof AS ( + + SELECT + -- ids + document_id, + transaction_order_id, + approver_user_id, + + -- dimensions + document_type, + status, + + -- measures + + -- date/times + created_datetime, + approved_datetime, + updated_datetime, + rejected_datetime, + deleted_datetime, + + -- metadata + _dbt_ran_datetime + + FROM filter_latest_documents + +) + +SELECT * FROM final__stg_exchange__customer_payment_proof \ No newline at end of file diff --git a/models/staging/googlesheet/_googlesheet__sources.yml b/models/staging/googlesheet/_googlesheet__sources.yml index 800c970..ad34469 100644 --- a/models/staging/googlesheet/_googlesheet__sources.yml +++ b/models/staging/googlesheet/_googlesheet__sources.yml @@ -6,7 +6,7 @@ sources: database: DEV_CIEF_RAW_DB schema: GOOGLESHEET_AIRBYTE loader: Airbyte - loaded_at_field: _airbyte_emitted_at + loaded_at_field: _airbyte_extracted_at freshness: warn_after: {count: 26, period: hour} error_after: {count: 48, period: hour} @@ -19,32 +19,31 @@ sources: tables: - - name: _airbyte_raw_leads_activations + - name: leads_activations description: This data is get from sales gsheet, sales will call user once they register & not put any order columns: - - name: _airbyte_ab_id - description: Primary key for '_airbyte_raw_leads_activations' + - name: _airbyte_raw_id + description: Primary key for 'leads_activations' tests: - unique - not_null - - name: _airbyte_data - description: JSON data which come from gsheet - tests: - - not_null - - - name: _airbyte_raw_event_and_holiday_lists + - name: event_and_holiday_lists description: Data retrieved from gsheet, comprises dates for events and holidays that may or may not impact operation of company. columns: - - name: _airbyte_ab_id - description: Primary key for '_airbyte_raw_event_and_holiday_lists' + - name: _airbyte_raw_id + description: Primary key for 'event_and_holiday_lists' tests: - unique - not_null - - - name: _airbyte_data - description: JSON data which come from gsheet - tests: - - not_null + + - name: account_frozen_case + description: Data retrieved from gsheet, comprises frozen account cases from Exchange service. + columns: + - name: _airbyte_raw_id + description: Primary key for 'account_frozen_case' + tests: + - unique + - not_null \ No newline at end of file diff --git a/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql b/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql index 61fbfb3..df62257 100644 --- a/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql +++ b/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql @@ -1,44 +1,36 @@ -- IMPORT WITH holiday_lists AS ( - SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_event_and_holiday_lists') }} + SELECT * FROM {{ source('src_googlesheet_airbyte', 'event_and_holiday_lists') }} ), --- LOGIC -parse_jason_holidays AS ( - SELECT - _airbyte_ab_id AS airbyte_id, - parse_json(_airbyte_data) AS holiday_json, - _airbyte_emitted_at AS _airbyte_emitted_datetime +-- LOGIC +data_casting AS ( + + SELECT + id AS holiday_id, + _airbyte_raw_id AS _airbyte_id, + DATE(date) AS date, + event_cultural_origin AS religion, + event_desc AS event_description, + general_type AS holiday_general_type, + holiday_type AS holiday_specific_type, + special_remark AS remark, + day, + event_name, + country, + state, + is_malaysia_holiday, + is_company_holiday, + is_china_holiday, + is_china_warehouse_holiday, + + _airbyte_extracted_at AS _airbyte_extracted_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM holiday_lists ), -flatten_json AS ( - - SELECT - airbyte_id, - holiday_json['id']::INTEGER AS holiday_id, - holiday_json['date']::DATE AS date, - holiday_json['day']::STRING AS day, - holiday_json['event_name']::STRING AS event_name, - holiday_json['event_desc']::STRING AS event_description, - holiday_json['general_type']::STRING AS holiday_general_type, - holiday_json['holiday_type']::STRING AS holiday_specific_type, - holiday_json['country']::STRING AS country, - holiday_json['state']::STRING AS state, - holiday_json['event_cultural_origin']::STRING AS religion, - holiday_json['is_malaysia_holiday']::INTEGER AS is_malaysia_holiday, - holiday_json['is_company_holiday']::INTEGER AS is_company_holiday, - holiday_json['is_china_holiday']::INTEGER AS is_china_holiday, - holiday_json['is_china_warehouse_holiday']::INTEGER AS is_china_warehouse_holiday, - holiday_json['special_remark']::STRING AS remark, - - _airbyte_emitted_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime - - FROM parse_jason_holidays -), -- FINAL final__stg_googlesheet__event_and_holiday_lists AS ( @@ -46,7 +38,6 @@ final__stg_googlesheet__event_and_holiday_lists AS ( SELECT -- ids holiday_id, - airbyte_id, -- dimensions day, @@ -61,6 +52,7 @@ final__stg_googlesheet__event_and_holiday_lists AS ( is_company_holiday, is_china_holiday, is_china_warehouse_holiday, + remark, -- measures @@ -68,11 +60,11 @@ final__stg_googlesheet__event_and_holiday_lists AS ( date, -- metadata - remark, - _airbyte_emitted_datetime, + _airbyte_id, + _airbyte_extracted_datetime, _dbt_ran_datetime - FROM flatten_json + FROM data_casting ) -SELECT * FROM final__stg_googlesheet__event_and_holiday_lists +SELECT * FROM final__stg_googlesheet__event_and_holiday_lists \ No newline at end of file diff --git a/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql b/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql index 917d47f..c700cd3 100644 --- a/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql +++ b/models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql @@ -1,49 +1,57 @@ -- IMPORT WITH lead_activations AS ( - SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_leads_activations') }} + SELECT * FROM {{ source('src_googlesheet_airbyte', 'leads_activations') }} ), + -- LOGIC -parse_json_leads AS ( +data_casting AS ( + SELECT - _airbyte_ab_id AS sales_team_lead_activation_id, - parse_json(_airbyte_data) AS leads_json, - _airbyte_emitted_at AS _airbyte_emitted_datetime + _airbyte_raw_id AS _airbyte_id, + company_marking AS company_marking_id, + agent_name, + contact_number, + user_name AS contact_name, + email AS contact_email, + remark AS sales_team_remark, + sorce AS contact_method, + tag AS contact_status, + COALESCE( + TRY_TO_TIMESTAMP(register_date::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(register_date::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS exchange_registered_datetime, + + COALESCE( + TRY_TO_TIMESTAMP(user_first_reply_datetime::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(user_first_reply_datetime::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS first_reply_datetime, + + COALESCE( + TRY_TO_TIMESTAMP(contacted_datetime::STRING, 'DD-MM-YYYY'), + TRY_TO_TIMESTAMP(contacted_datetime::STRING, 'DD-MM-YYYY HH24:MI:SS') + ) AS sales_team_contacted_datetime, + + _airbyte_extracted_at AS _airbyte_extracted_datetime + FROM lead_activations + ), -flatten_json AS ( +generate_surrogate_key AS ( SELECT - sales_team_lead_activation_id, - leads_json['agent_name']::STRING AS agent_name, - leads_json['company_marking']::STRING AS company_marking_id, - leads_json['contact_number']::STRING AS contact_number, - leads_json['email']::STRING AS contact_email, - leads_json['sorce']::STRING AS contact_method, - leads_json['tag']::STRING AS contact_status, - leads_json['user_name']::STRING AS contact_name, - leads_json['remark']::STRING AS sales_team_remark, - COALESCE( - TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY'), - TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') - ) AS sales_team_contacted_datetime, - COALESCE( - TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY'), - TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY HH24:MI:SS') - ) AS exchange_registered_datetime, - COALESCE( - TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY'), - TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS') - ) AS first_reply_datetime, + *, + {{ dbt_utils.generate_surrogate_key(['sales_team_contacted_datetime', 'company_marking_id', 'agent_name']) }} AS sales_team_lead_activation_id, - _airbyte_emitted_datetime, - '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM data_casting - FROM parse_json_leads ), + -- FINAL final__stg_googlesheet__sales_exchange_lead_activations AS ( @@ -68,11 +76,13 @@ final__stg_googlesheet__sales_exchange_lead_activations AS ( first_reply_datetime, exchange_registered_datetime, - -- metadata - _airbyte_emitted_datetime, + -- metadata + _airbyte_id, + _airbyte_extracted_datetime, _dbt_ran_datetime - FROM flatten_json + FROM generate_surrogate_key + ) SELECT * FROM final__stg_googlesheet__sales_exchange_lead_activations \ No newline at end of file