cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-19 04:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'lee_build_rep_frozen_account_incidents' into 'main'

Browse Source 
Lee build rep frozen account incidents

See merge request cief-data/dbt_cloud!29
This commit is contained in:
Yam ZhengLim
2023-12-10 06:57:04 +00:00
parent 78b65467da 42202ccea1
commit 2712fb88db
16 changed files with 728 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/intermediate/int_exchange__transaction_cost_with_latest_cost_ids.sql
+119 -22
View File
@@ -1,21 +1,38 @@
-- VARIABLES
{% set order_status = dbt_utils.get_column_values(
table=ref('stg_exchange__transaction_order_logs'),
column='status')
{% set cost_status = dbt_utils.get_column_values(
table=ref('stg_exchange__transaction_cost_logs'),
column='status')
%}
-- IMPORTS
WITH transaction_cost_logs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }}
SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }}
),
transaction_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_costs') }}
),
currency_vendor_payment_proof_documents AS (
SELECT * FROM {{ ref('stg_exchange__currency_vendor_payment_proof') }}
),
temp_bookings AS (
SELECT * FROM {{ ref('stg_exchange__bookings') }}
),
temp_orders AS (
SELECT * FROM {{ ref('stg_exchange__transaction_orders') }}
),
-- LOGIC
/*
Some order may have multiple same cost status. This is due to the accounting department adjusting the exchange rate.
This is particularly true for the 1688 payment, as exchange rate is amended at a later stage instead of during the initial order placement stage.
Thus, it is to be assumed that the datetime from first repeat occurence as the true datetime.
*/
transaction_cost_status_datetime AS (
SELECT
@@ -23,8 +40,8 @@ transaction_cost_status_datetime AS (
company_id,
MIN(created_datetime) AS cost_created_datetime,
{%- for status in order_status %}
MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
{%- for status in cost_status %}
MIN(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%}
{% endfor %}
FROM transaction_cost_logs
@@ -61,19 +78,28 @@ join_cost_and_logs AS (
transaction_cost_status_datetime.company_id,
transaction_cost_status_datetime.cost_created_datetime,
transaction_cost_status_datetime.cost_completed_datetime,
transaction_cost_status_datetime.cost_pending_submission_datetime,
transaction_cost_status_datetime.cost_approved_datetime,
transaction_cost_status_datetime.cost_pending_verification_datetime,
transaction_cost_status_datetime.cost_rejected_datetime,
transaction_cost_status_datetime.cost_suspended_datetime,
transaction_cost_status_datetime.cost_expired_datetime
transaction_cost_status_datetime.cost_approved_datetime,
transaction_cost_status_datetime.cost_completed_datetime,
currency_vendor_payment_proof_documents.document_type AS cost_document_type,
currency_vendor_payment_proof_documents.status AS cost_document_status,
currency_vendor_payment_proof_documents.created_datetime AS cost_document_created_datetime,
currency_vendor_payment_proof_documents.updated_datetime AS cost_document_updated_datetime,
ROW_NUMBER() OVER (PARTITION BY transaction_costs.transaction_cost_id ORDER BY cost_document_created_datetime, cost_document_updated_datetime) AS row_number_index
FROM transaction_costs
LEFT JOIN transaction_cost_status_datetime
ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id)
LEFT JOIN currency_vendor_payment_proof_documents
ON (transaction_costs.transaction_cost_id = currency_vendor_payment_proof_documents.transaction_cost_id)
QUALIFY
row_number_index = 1
),
remove_deleted_expired_costs AS (
@@ -95,9 +121,7 @@ remove_system_error_duplicate_row AS (
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index
FROM remove_deleted_expired_costs
@@ -106,6 +130,78 @@ remove_system_error_duplicate_row AS (
),
/*
Temporary CTE to address the issue of 0.4% service charge for 1688 payments
Affected supplier_company_id from 2729 and 4548
Affected orders are from 3rd-Apr-2023 onwards
Service charge is only applied onto the CNY
*/
temp_cte_1688_cleaning AS (
SELECT
remove_system_error_duplicate_row.transaction_cost_id,
remove_system_error_duplicate_row.transaction_order_id,
remove_system_error_duplicate_row.supplier_company_id,
remove_system_error_duplicate_row.company_id,
remove_system_error_duplicate_row.bank_id,
remove_system_error_duplicate_row.base_currency_id,
remove_system_error_duplicate_row.quote_currency_id,
remove_system_error_duplicate_row.transaction_type,
remove_system_error_duplicate_row.payment_method,
remove_system_error_duplicate_row.status,
remove_system_error_duplicate_row.payment_reference,
remove_system_error_duplicate_row.bill_number,
remove_system_error_duplicate_row.cost_document_type,
remove_system_error_duplicate_row.cost_document_status,
remove_system_error_duplicate_row.base_value,
remove_system_error_duplicate_row.quote_value,
remove_system_error_duplicate_row.base_to_quote_currency_exchange_rate,
remove_system_error_duplicate_row.base_tax,
remove_system_error_duplicate_row.expired_datetime,
remove_system_error_duplicate_row.deleted_datetime,
remove_system_error_duplicate_row.created_datetime,
remove_system_error_duplicate_row.updated_datetime,
remove_system_error_duplicate_row.cost_created_datetime,
remove_system_error_duplicate_row.cost_pending_submission_datetime,
remove_system_error_duplicate_row.cost_pending_verification_datetime,
remove_system_error_duplicate_row.cost_approved_datetime,
remove_system_error_duplicate_row.cost_completed_datetime,
remove_system_error_duplicate_row.cost_document_created_datetime,
remove_system_error_duplicate_row.cost_document_updated_datetime,
temp_bookings.service_type,
CASE
WHEN
remove_system_error_duplicate_row.supplier_company_id IN (2729, 4548)
-- 2729 = HCK Global
-- 4548 = Power Progress
AND
temp_bookings.service_type = '1688 PAYMENT'
AND
cost_created_datetime >= '2023-04-03'
THEN
-- Formula used for service charge in MYR: CNY * 0.4% / exchange rate
ROUND(DIV0(
remove_system_error_duplicate_row.quote_value * ( 0.4 / 100 ),
remove_system_error_duplicate_row.base_to_quote_currency_exchange_rate
),4)
ELSE
remove_system_error_duplicate_row.base_service_charge
END AS base_service_charge,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM remove_system_error_duplicate_row
LEFT JOIN temp_orders
ON (remove_system_error_duplicate_row.transaction_order_id = temp_orders.transaction_order_id)
LEFT JOIN temp_bookings
ON (temp_orders.booking_id = temp_bookings.booking_id)
),
-- FINAL
final__int_exchange__transaction_cost_get_latest_cost_ids AS (
@@ -126,6 +222,8 @@ final__int_exchange__transaction_cost_get_latest_cost_ids AS (
status,
payment_reference,
bill_number,
cost_document_type,
cost_document_status,
-- measures
base_value,
@@ -140,19 +238,18 @@ final__int_exchange__transaction_cost_get_latest_cost_ids AS (
created_datetime,
updated_datetime,
cost_created_datetime,
cost_completed_datetime,
cost_pending_submission_datetime,
cost_approved_datetime,
cost_pending_verification_datetime,
cost_rejected_datetime,
cost_suspended_datetime,
cost_expired_datetime,
cost_approved_datetime,
cost_completed_datetime,
cost_document_created_datetime,
cost_document_updated_datetime,
-- metadata
_dbt_ran_datetime
FROM remove_system_error_duplicate_row
FROM temp_cte_1688_cleaning
)
SELECT * FROM final__int_exchange__transaction_cost_get_latest_cost_ids
models/marts/reporting/rep_exchange__company_conversion_funnels.sql
+22 -2
View File
@@ -347,13 +347,32 @@ generate_cumulative_count AS (
FROM union_all_activities_flag_migrated_company
ORDER BY activity_datetime
),
-- Include additional details for companies
enhance_company_details AS (
SELECT
generate_cumulative_count.*,
companies.company_marking_id,
companies.company_created_datetime AS company_register_datetime
FROM generate_cumulative_count
LEFT JOIN companies
ON (generate_cumulative_count.company_id = companies.company_id)
),
-- FINAL
final__rep_exchange__company_conversion_funnels AS (
SELECT
-- ids
company_id,
company_marking_id,
-- dimensions
is_data_cleansing_generate_row,
@@ -372,6 +391,7 @@ final__rep_exchange__company_conversion_funnels AS (
cumulative_first_time_order_completed_activity,
cumulative_repeat_completed_order_after_30days_activity,
cumulative_repeat_completed_order_after_60days_activity,
include_migrated_cumulative_register_activity,
include_migrated_cumulative_email_verified_activity,
include_migrated_cumulative_identity_document_uploaded_activity,
@@ -383,13 +403,13 @@ final__rep_exchange__company_conversion_funnels AS (
include_migrated_cumulative_repeat_completed_order_after_60days_activity,
-- date/times
company_register_datetime,
activity_datetime,
-- metadata
_dbt_ran_datetime
FROM generate_cumulative_count
FROM enhance_company_details
)
SELECT * FROM final__rep_exchange__company_conversion_funnels
models/marts/reporting/rep_exchange__daily_orders.sql
+144 -141
View File
@@ -1,26 +1,33 @@
-- VARIABLES
{% set var_service_type = [
'1688 PAYMENT',
'1 DAY TRANSFER',
'3 DAYS TRANSFER',
'Enterprise to Enterprise 公打公'] %}
{% set service_types = dbt_utils.get_column_values(
table=ref('fct_exchange__transaction_orders'),
column='service_type')
%}
{% set var_working_days_including_base_date = [
'add_1_working_day_included_base_date',
'add_1_working_day_included_base_date',
'add_3_working_day_included_base_date',
'add_7_working_day_included_base_date'] %}
-- working days mapping for each service type
{% set service_type_working_day_mapping = {
'1 DAY TRANSFER': 1,
'1688 PAYMENT': 1,
'3 DAYS TRANSFER': 3,
'ENTERPRISE TO ENTERPRISE 公打公': 7
} %}
{% set var_working_days_excluding_base_date = [
'add_1_working_day_excluded_base_date',
'add_1_working_day_excluded_base_date',
'add_3_working_day_excluded_base_date',
'add_7_working_day_excluded_base_date'] %}
{% set service_type_work_day_dict={} %}
{% for service_type, working_day in service_type_working_day_mapping.items() %}
{% set service_type_work_day_dict = service_type_work_day_dict.update(
{service_type: {
'include_base': 'add_' ~ working_day ~ '_working_day_included_base_date',
'exclude_base': 'add_' ~ working_day ~ '_working_day_excluded_base_date'
}}) %}
{% endfor %}
-- IMPORT
WITH orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
companies AS (
@@ -39,67 +46,47 @@ users AS (
SELECT * FROM {{ ref('dim_exchange__users') }}
),
--LOGIC
-- LOGIC
fct_and_dim_joins AS (
SELECT
orders.booking_id,
orders.booking_marking_id,
orders.order_id,
orders.cost_id,
orders.company_id,
orders.transaction_status,
orders.document_status,
companies.name AS company_name,
companies.company_marking_id AS company_marking_id,
companies.autocount_id AS company_autocount_id,
companies.company_type AS company_type,
companies.business_type AS company_business_type,
companies.exchange_rate_segment AS company_exchange_rate_segment,
companies.is_migrated_company AS is_migrated_company,
companies.country_name AS company_country_name,
companies.state_name AS company_state_name,
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.has_wallet AS company_has_wallet,
companies.latitude AS company_latitude,
companies.longitude AS company_longitude,
companies.company_created_datetime,
companies.company_lifetime_value,
companies.m_score_lifetime AS company_m_score_lifetime,
orders.supplier_company_id,
orders.user_id,
orders.bank_id,
fix_currencies.currency_name AS fix_currency_name,
quote_currencies.currency_name AS quote_currency_name,
base_currencies.currency_name AS base_currency_name,
orders.currency_vendor_bank_id,
orders.transaction_status,
orders.document_status,
orders.service_type,
orders.booking_status,
orders.order_status,
orders.cost_status,
orders.order_payment_method,
orders.order_payment_reference,
orders.order_bill_number,
orders.cost_transaction_type,
orders.cost_payment_method,
orders.cost_payment_reference,
orders.cost_bill_number,
orders.purchase_order_number,
orders.invoice_number,
orders.is_first_time_booking_company,
orders.is_first_time_booking_company_completed,
orders.is_first_time_booking_user,
orders.is_first_time_booking_user_completed,
orders.estimate_booking_base_to_quote_currency_exchange_rate,
orders.estimate_booking_quote_value,
orders.estimate_booking_base_value,
orders.estimate_booking_value_rm,
orders.booking_created_datetime,
orders.booking_approved_datetime,
orders.booking_completed_datetime,
orders.booking_suspended_datetime,
orders.order_id,
orders.order_payment_method,
orders.order_status,
orders.order_payment_reference,
orders.order_bill_number,
orders.is_first_time_order_company,
orders.is_first_time_order_company_completed,
orders.is_first_time_order_user,
orders.is_first_time_order_user_completed,
orders.order_base_to_quote_currency_exchange_rate,
orders.estimate_booking_quote_value,
orders.estimate_booking_base_value,
orders.estimate_booking_value_rm,
orders.estimate_booking_base_to_quote_currency_exchange_rate,
orders.order_base_value,
orders.order_base_service_charge,
orders.order_base_tax,
@@ -108,100 +95,36 @@ fct_and_dim_joins AS (
orders.order_service_charge_rm,
orders.order_tax_rm,
orders.total_order_value_rm,
order_created_dates.day_of_year AS order_created_day_number,
order_created_dates.first_day_of_week AS order_created_week,
order_created_dates.week_of_year AS order_created_week_number,
order_created_dates.first_day_of_month AS order_created_month,
order_created_dates.day_of_month AS order_created_month_day_number,
order_created_dates.month_actual AS order_created_month_number,
order_created_dates.year_actual AS order_created_year,
orders.order_base_to_quote_currency_exchange_rate,
orders.order_created_datetime,
orders.order_pending_submission_datetime,
orders.order_pending_verification_datetime,
-- Estimated delivery datetime based on CIEF internal SLA (cutoff time 4pm)
CASE
{% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %}
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' 23:59:59 +08:00')
{% endfor %}
ELSE NULL
END AS estimated_order_delivery_datetime_website_sla,
--- Estimated delivery datetime based on customer expectation
CASE
{% for (loop_service_type, working_day_with_base_date, working_day_without_base_date) in zip(var_service_type, var_working_days_including_base_date, var_working_days_excluding_base_date) %}
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_with_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{loop_service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day_without_base_date}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
{% endfor %}
ELSE NULL
END AS estimated_order_delivery_datetime_customer_expectation,
-- On time delivery boolean (CIEF internal SLA)
CASE
-- On-time boolean is null when order is not complete
WHEN account_verified_payment_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1
ELSE 0
END AS is_on_time_delivery_website_sla,
-- On time delivery boolean (Customer Expectation)
CASE
-- On-time boolean is null when order is not complete
WHEN account_verified_payment_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1
ELSE 0
END AS is_on_time_delivery_customer_expectation,
orders.order_approved_datetime,
orders.order_completed_datetime,
orders.order_rejected_datetime,
orders.order_suspended_datetime,
orders.order_expected_expired_datetime,
orders.order_expired_datetime,
orders.cost_id,
orders.supplier_company_id,
supplier_companies.name AS supplier_company_name,
orders.currency_vendor_bank_id,
orders.cost_transaction_type,
orders.cost_payment_method,
orders.cost_status,
orders.cost_payment_reference,
orders.cost_bill_number,
orders.cost_base_value,
orders.cost_quote_value,
orders.cost_base_to_quote_currency_exchange_rate,
orders.cost_base_tax,
orders.cost_base_service_charge,
orders.cost_value_rm,
orders.cost_tax_rm,
orders.cost_service_charge_rm,
orders.total_cost_value_rm,
orders.purchase_order_number,
orders.invoice_number,
orders.cost_base_to_quote_currency_exchange_rate,
orders.booking_created_datetime,
orders.booking_approved_datetime,
orders.booking_completed_datetime,
orders.booking_suspended_datetime,
orders.order_created_datetime,
orders.order_pending_submission_datetime,
orders.order_pending_verification_datetime,
orders.order_approved_datetime,
orders.order_completed_datetime,
orders.order_rejected_datetime,
orders.order_suspended_datetime,
orders.order_expected_expired_datetime,
orders.order_expired_datetime,
orders.order_deleted_datetime,
orders.cost_created_datetime,
orders.cost_pending_submission_datetime,
orders.cost_pending_verification_datetime,
orders.cost_approved_datetime,
orders.cost_completed_datetime,
orders.cost_rejected_datetime,
orders.customer_made_booking_datetime,
orders.customer_placed_order_datetime,
orders.customer_made_payment_datetime,
@@ -215,6 +138,84 @@ fct_and_dim_joins AS (
orders.account_rejected_payment_datetime,
orders.next_order_created_datetime,
companies.name AS company_name,
companies.company_marking_id AS company_marking_id,
companies.autocount_id AS company_autocount_id,
companies.company_type AS company_type,
companies.business_type AS company_business_type,
companies.exchange_rate_segment AS company_exchange_rate_segment,
companies.is_migrated_company AS is_migrated_company,
companies.country_name AS company_country_name,
companies.state_name AS company_state_name,
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.has_wallet AS company_has_wallet,
companies.latitude AS company_latitude,
companies.longitude AS company_longitude,
companies.m_score_lifetime AS company_m_score_lifetime,
companies.company_created_datetime,
companies.company_lifetime_value,
fix_currencies.currency_name AS fix_currency_name,
quote_currencies.currency_name AS quote_currency_name,
base_currencies.currency_name AS base_currency_name,
order_created_dates.day_of_year AS order_created_day_number,
order_created_dates.first_day_of_week AS order_created_week,
order_created_dates.week_of_year AS order_created_week_number,
order_created_dates.first_day_of_month AS order_created_month,
order_created_dates.day_of_month AS order_created_month_day_number,
order_created_dates.month_actual AS order_created_month_number,
order_created_dates.year_actual AS order_created_year,
supplier_companies.name AS supplier_company_name,
-- Estimated delivery datetime based on CIEF internal SLA (cutoff time 4pm)
CASE
{% for (service_type, working_day) in service_type_work_day_dict.items() %}
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' 23:59:59 +08:00')
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' 23:59:59 +08:00')
{% endfor %}
ELSE NULL
END AS estimated_order_delivery_datetime_website_sla,
--- Estimated delivery datetime based on customer expectation
CASE
{% for (service_type, working_day) in service_type_work_day_dict.items() %}
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) < 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day['include_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
WHEN orders.order_created_datetime >= '2023-01-01' AND orders.service_type = '{{service_type}}' AND DATE_PART(HOUR, orders.order_pending_verification_datetime) >= 16
THEN TO_TIMESTAMP(order_created_dates.{{working_day['exclude_base']}} || ' ' || (orders.order_pending_verification_datetime::TIME)::STRING )
{% endfor %}
ELSE NULL
END AS estimated_order_delivery_datetime_customer_expectation,
-- On time delivery boolean (CIEF internal SLA)
CASE
-- On-time boolean is null when order is not complete
WHEN account_verified_payment_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_website_sla > orders.operation_uploaded_bank_slip_datetime THEN 1
ELSE 0
END AS is_on_time_delivery_website_sla,
-- On time delivery boolean (Customer Expectation)
CASE
-- On-time boolean is null when order is not complete
WHEN account_verified_payment_datetime IS NULL THEN NULL
WHEN estimated_order_delivery_datetime_customer_expectation > orders.operation_uploaded_bank_slip_datetime THEN 1
ELSE 0
END AS is_on_time_delivery_customer_expectation,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM orders
@@ -236,10 +237,12 @@ fct_and_dim_joins AS (
LEFT JOIN currencies AS base_currencies
ON (orders.base_currency_id = base_currencies.currency_id)
),
--FINAL
final__rep_exchange__daily_orders AS (
SELECT
-- ids
order_id,
@@ -341,12 +344,12 @@ final__rep_exchange__daily_orders AS (
order_rejected_datetime,
order_suspended_datetime,
order_expired_datetime,
order_deleted_datetime,
cost_created_datetime,
cost_pending_submission_datetime,
cost_pending_verification_datetime,
cost_approved_datetime,
cost_completed_datetime,
cost_rejected_datetime,
booking_created_datetime,
booking_approved_datetime,
booking_completed_datetime,
models/marts/reporting/rep_exchange__sales_lead_conversions.sql
+9 -6
View File
@@ -15,6 +15,7 @@ transactions AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
-- LOGIC
join_booking_transaction AS (
@@ -49,8 +50,8 @@ join_booking_transaction AS (
transactions.order_id AS transaction_order_id,
transactions.order_transaction_type,
transactions.order_payment_method,
transactions.transaction_status,
transactions.total_order_value_rm AS total_value_rm,
transactions.order_status AS transaction_order_status,
transactions.order_created_datetime AS transaction_created_datetime,
transactions.order_approved_datetime AS transaction_approved_datetime,
transactions.order_completed_datetime AS transaction_completed_datetime
@@ -88,7 +89,7 @@ data_enrich_grouped AS (
-- Count of orders placed after contacted by sales team
SUM(
CASE
WHEN transaction_order_status = 'COMPLETED' AND contact_status = 'Contacted' AND transaction_created_datetime > sales_team_contacted_datetime
WHEN transaction_status IN ('UPLOADED_BANK_SLIP') AND contact_status IN ('Contacted') AND transaction_created_datetime > sales_team_contacted_datetime
THEN 1
ELSE 0
END
@@ -110,6 +111,7 @@ data_enrich_grouped AS (
-- Boolean - Bookings and orders made AFTER contacted datetime and successful contact
CASE
WHEN MAX( booking_created_datetime ) > sales_team_contacted_datetime AND contact_status = 'Contacted' THEN '1'
WHEN MAX( transaction_created_datetime ) > sales_team_contacted_datetime AND contact_status = 'Contacted' THEN '1'
ELSE '0'
END AS has_booked_after_contacted,
CASE
@@ -127,7 +129,7 @@ data_enrich_grouped AS (
MIN(
CASE
WHEN transaction_created_datetime < sales_team_contacted_datetime
AND transaction_order_status NOT IN ('SUSPENDED', 'REJECTED')
AND transaction_status NOT IN ('ORDER_REJECTED', 'ORDER_SUSPENDED')
THEN TIMEDIFF(minute, transaction_created_datetime, sales_team_contacted_datetime) / 60
END
) AS hours_last_ordered_before_contacted,
@@ -142,7 +144,7 @@ data_enrich_grouped AS (
MIN(
CASE
WHEN ( sales_team_contacted_datetime < transaction_created_datetime ) AND contact_status = 'Contacted'
AND transaction_order_status NOT IN ('SUSPENDED', 'REJECTED')
AND transaction_status NOT IN ('ORDER_REJECTED', 'ORDER_SUSPENDED')
THEN TIMEDIFF(minute, sales_team_contacted_datetime, transaction_created_datetime) / 60
END
) AS hours_first_ordered_after_contacted,
@@ -157,7 +159,7 @@ data_enrich_grouped AS (
CASE WHEN sales_team_contacted_datetime > transaction_created_datetime THEN total_value_rm ELSE 0 END
) AS total_value_rm_before_contact,
SUM(
CASE WHEN transaction_order_status IN ('APPROVED', 'COMPLETED')
CASE WHEN transaction_status IN ('UPLOADED_BANK_SLIP')
AND sales_team_contacted_datetime > transaction_created_datetime THEN total_value_rm ELSE 0 END
) AS total_value_rm_approved_completed_before_contact,
@@ -167,7 +169,7 @@ data_enrich_grouped AS (
AND contact_status = 'Contacted' THEN total_value_rm ELSE 0 END
) AS total_value_rm_after_contact,
SUM(
CASE WHEN transaction_order_status IN ('APPROVED', 'COMPLETED')
CASE WHEN transaction_status IN ('UPLOADED_BANK_SLIP')
AND sales_team_contacted_datetime < transaction_created_datetime AND contact_status = 'Contacted' THEN total_value_rm ELSE 0 END
) AS total_value_rm_approved_completed_after_contact,
@@ -241,6 +243,7 @@ data_enrich_grouped AS (
exchange_registered_datetime
),
-- FINAL
final__rep_exchange__sales_lead_conversions AS (
models/marts/warehouse/dim__dates.py
+28 -27
View File
@@ -17,7 +17,7 @@ from datetime import datetime as dt, timedelta
# Function to check if a given date is a weekend [Saturday(5) or Sunday(6)]
def is_weekend(date):
return (date.weekday() == 5) | (date.weekday() == 6)
return date.weekday() in (5, 6)
# Function to compute the in advance working days
@@ -28,14 +28,14 @@ def compute_working_days(start_date, num_working_days, holiday_list, after_cut_o
counter = 0
# For dates on weekend or holiday or after cut off time, additional 1 working day to the loop
if (after_cut_off == 1) | is_weekend(start_date) | (start_date in holiday_list):
if (after_cut_off == 1) or is_weekend(start_date) or (start_date in holiday_list):
counter -= 1
# Loop to increase n number of working days, if weekend/holiday, skip counter
while counter < num_working_days:
start_date = start_date + timedelta(days=1)
if is_weekend(start_date) | (start_date in holiday_list):
if is_weekend(start_date) or (start_date in holiday_list):
continue
counter += 1
@@ -45,36 +45,37 @@ def compute_working_days(start_date, num_working_days, holiday_list, after_cut_o
# Main function
def model(dbt, session):
# Setting configuration
dbt.config(materialized="table",
packages = ["pandas"])
# Import data from upstream dbt model
df_date = dbt.ref("int__dates")
sp_df_date = dbt.ref("int__dates")
# Extract holiday dates into a list
df_work = df_date.to_pandas()
df_new = df_work[df_work["IS_COMPANY_HOLIDAY"] == 1]
df_holiday_list = df_new["DATE_DAY"].tolist()
# Filter holiday dates into a dataframe
sp_df_filter = sp_df_date.filter(sp_df_date['IS_COMPANY_HOLIDAY'] == 1)
sp_df_holiday_date = sp_df_filter.select('DATE_DAY')
# Apply function to df
df_work["ADD_1_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 0), axis=1)
df_work["ADD_2_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 0), axis=1)
df_work["ADD_3_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 0), axis=1)
df_work["ADD_7_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 0), axis=1)
df_work["ADD_30_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 0), axis=1)
df_work["ADD_90_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 0), axis=1)
df_work["ADD_365_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 0), axis=1)
# Convert snowpark dataframe to pandas dataframe
pd_df_date = sp_df_date.to_pandas()
pd_df_holiday_date = sp_df_holiday_date.to_pandas()
# Store holiday dates in a list
holiday_list = pd_df_holiday_date['DATE_DAY'].tolist()
# For orders after cut off time 4pm
# Affects only the weekdays
df_work["ADD_1_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 1), axis=1)
df_work["ADD_2_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 1), axis=1)
df_work["ADD_3_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 1), axis=1)
df_work["ADD_7_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 7, df_holiday_list, 1), axis=1)
df_work["ADD_30_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 1), axis=1)
df_work["ADD_90_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 1), axis=1)
df_work["ADD_365_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 1), axis=1)
# Applying add working days function to df
add_working_days = [1, 2, 3, 7, 30, 90 ,365]
return df_work
for working_day in add_working_days:
for after_cut_off in (0, 1):
if after_cut_off == 0:
col_name = f'ADD_{working_day}_WORKING_DAY_INCLUDED_BASE_DATE'
else:
col_name = f'ADD_{working_day}_WORKING_DAY_EXCLUDED_BASE_DATE'
# Apply function and append new calculated columns in the dataframe
pd_df_date[col_name] = pd_df_date['DATE_DAY'].apply(lambda start_date: compute_working_days(start_date, working_day, holiday_list, after_cut_off))
return pd_df_date
models/marts/warehouse/fct_exchange__transaction_orders.sql
+17 -7
View File
@@ -293,6 +293,8 @@ transaction_costs_lists AS (
transaction_type,
payment_method,
status,
cost_document_type,
cost_document_status,
payment_reference,
bill_number,
base_value,
@@ -322,14 +324,13 @@ transaction_costs_lists AS (
deleted_datetime,
created_datetime,
updated_datetime,
cost_document_created_datetime,
cost_document_updated_datetime,
cost_created_datetime,
cost_completed_datetime,
cost_pending_submission_datetime,
cost_approved_datetime,
cost_pending_verification_datetime,
cost_rejected_datetime,
cost_suspended_datetime,
cost_expired_datetime
cost_pending_verification_datetime
FROM transaction_costs
@@ -444,12 +445,15 @@ bookings_orders_costs_join AS (
transaction_costs_lists.tax_rm AS cost_tax_rm,
transaction_costs_lists.service_charge_rm AS cost_service_charge_rm,
transaction_costs_lists.total_value_rm AS total_cost_value_rm,
transaction_costs_lists.cost_document_type,
transaction_costs_lists.cost_document_status,
transaction_costs_lists.cost_created_datetime,
transaction_costs_lists.cost_pending_submission_datetime,
transaction_costs_lists.cost_pending_verification_datetime,
transaction_costs_lists.cost_approved_datetime,
transaction_costs_lists.cost_completed_datetime,
transaction_costs_lists.cost_rejected_datetime
transaction_costs_lists.cost_document_created_datetime,
transaction_costs_lists.cost_document_updated_datetime
FROM transaction_orders_lists
@@ -554,7 +558,10 @@ renaming_datetime_columns AS (
order_approved_datetime AS account_verified_payment_datetime,
order_completed_datetime AS first_white_form_generated_datetime,
cost_pending_verification_datetime AS last_white_form_generated_datetime,
COALESCE(cost_approved_datetime, cost_completed_datetime) AS operation_uploaded_bank_slip_datetime,
COALESCE(
cost_document_created_datetime,
cost_approved_datetime,
cost_completed_datetime) AS operation_uploaded_bank_slip_datetime,
purchase_order_created_datetime AS first_purchase_order_uploaded_datetime,
purchase_order_updated_datetime AS last_purchase_order_uploaded_datetime,
invoice_created_datetime AS operation_generated_invoice_datetime,
@@ -608,6 +615,8 @@ final_fct_exchange__new_orders AS (
cost_status,
cost_payment_reference,
cost_bill_number,
cost_document_type,
cost_document_status,
purchase_order_status,
purchase_order_number,
invoice_status,
@@ -656,12 +665,13 @@ final_fct_exchange__new_orders AS (
order_suspended_datetime,
order_expired_datetime,
cost_document_created_datetime,
cost_document_updated_datetime,
cost_created_datetime,
cost_pending_submission_datetime,
cost_pending_verification_datetime,
cost_approved_datetime,
cost_completed_datetime,
cost_rejected_datetime,
customer_made_booking_datetime,
customer_placed_order_datetime,
models/marts/warehouse/fct_googlesheet__sales_exchange_lead_activations.sql
+5 -1
View File
@@ -6,15 +6,20 @@ WITH lead_activations AS (
-- FINAL
final__fct_googlesheet__sales_exchange_lead_activations AS (
SELECT
-- id
sales_team_lead_activation_id,
company_marking_id,
-- dimension
agent_name,
contact_name,
contact_number,
contact_email,
contact_method,
contact_status,
sales_team_remark,
-- measures
@@ -24,7 +29,6 @@ final__fct_googlesheet__sales_exchange_lead_activations AS (
exchange_registered_datetime,
-- metadata
sales_team_remark,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM lead_activations
models/staging/exchange/base/base_exchange__booking_arch_logs.sql
+31 -17
View File
@@ -5,29 +5,43 @@ WITH booking_arch_logs AS (
-- LOGIC
booking_arch_logs_rename AS (
-- Exclude service type 'Internal Purchase'
exclude_service_type AS (
SELECT
booking_arch_logs.id AS booking_log_id,
booking_arch_logs.booking_id,
booking_arch_logs.company_id,
booking_arch_logs.marking AS booking_marking_id,
booking_arch_logs.service_id AS service_type,
booking_arch_logs.bank_id,
booking_arch_logs.fix_amount AS fix_value,
booking_arch_logs.fix_currency_id,
booking_arch_logs.convertible_currency_id AS quote_currency_id,
booking_arch_logs.conversion_currency_id AS base_currency_id,
booking_arch_logs.status,
booking_arch_logs.deleted_at AS deleted_datetime,
booking_arch_logs.created_at AS created_datetime,
booking_arch_logs.updated_at As updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
*
FROM booking_arch_logs
WHERE service_id NOT IN (7) -- 7 : INTERNAL PURCHASE
),
booking_arch_logs_rename AS (
SELECT
exclude_service_type.id AS booking_log_id,
exclude_service_type.booking_id,
exclude_service_type.company_id,
exclude_service_type.marking AS booking_marking_id,
exclude_service_type.service_id AS service_type,
exclude_service_type.bank_id,
exclude_service_type.fix_amount AS fix_value,
exclude_service_type.fix_currency_id,
exclude_service_type.convertible_currency_id AS quote_currency_id,
exclude_service_type.conversion_currency_id AS base_currency_id,
exclude_service_type.status,
exclude_service_type.deleted_at AS deleted_datetime,
exclude_service_type.created_at AS created_datetime,
exclude_service_type.updated_at As updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM exclude_service_type
),
-- FINAL
final_base_exchange__booking_arch_logs AS (
models/staging/exchange/base/base_exchange__bookings.sql
+30 -16
View File
@@ -5,28 +5,42 @@ WITH bookings AS (
-- LOGIC
bookings_rename AS (
-- Exclude service type 'Internal Purchase'
exclude_service_type AS (
SELECT
bookings.id AS booking_id,
bookings.company_id,
bookings.marking AS booking_marking_id,
bookings.service_id AS service_type,
bookings.bank_id,
bookings.fix_amount AS fix_value,
bookings.fix_currency_id,
bookings.convertible_currency_id AS quote_currency_id,
bookings.conversion_currency_id AS base_currency_id,
bookings.status,
bookings.deleted_at AS deleted_datetime,
bookings.created_at AS created_datetime,
bookings.updated_at As updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
*
FROM bookings
WHERE service_id NOT IN (7) -- 7 : INTERNAL PURCHASE
),
bookings_rename AS (
SELECT
exclude_service_type.id AS booking_id,
exclude_service_type.company_id,
exclude_service_type.marking AS booking_marking_id,
exclude_service_type.service_id AS service_type,
exclude_service_type.bank_id,
exclude_service_type.fix_amount AS fix_value,
exclude_service_type.fix_currency_id,
exclude_service_type.convertible_currency_id AS quote_currency_id,
exclude_service_type.conversion_currency_id AS base_currency_id,
exclude_service_type.status,
exclude_service_type.deleted_at AS deleted_datetime,
exclude_service_type.created_at AS created_datetime,
exclude_service_type.updated_at As updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM exclude_service_type
),
-- FINAL
final_base_exchange__bookings AS (
models/staging/exchange/base/base_exchange__currency_rates.sql
+23 -9
View File
@@ -5,20 +5,34 @@ WITH currency_rates AS (
-- LOGIC
-- Exclude service type 'Internal Purchase'
exclude_service_type AS (
SELECT
*
FROM currency_rates
WHERE service_id NOT IN (7) -- 7 : INTERNAL PURCHASE
),
currency_rates_rename AS (
SELECT
currency_rates.id AS currency_rate_id,
currency_rates.currency_id AS quote_currency_id,
currency_rates.selling AS base_to_quote_currency_exchange_rate,
currency_rates.payment_method_type AS payment_method,
currency_rates.service_id AS service_type,
currency_rates.deleted_at AS deleted_datetime,
currency_rates.created_at AS created_datetime,
currency_rates.updated_at AS updated_datetime,
exclude_service_type.id AS currency_rate_id,
exclude_service_type.currency_id AS quote_currency_id,
exclude_service_type.selling AS base_to_quote_currency_exchange_rate,
exclude_service_type.payment_method_type AS payment_method,
exclude_service_type.service_id AS service_type,
exclude_service_type.deleted_at AS deleted_datetime,
exclude_service_type.created_at AS created_datetime,
exclude_service_type.updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM currency_rates
FROM exclude_service_type
),
models/staging/exchange/base/base_exchange__transactions.sql
+55 -23
View File
@@ -5,34 +5,65 @@ WITH transactions AS (
-- LOGIC
-- Filter transaction id for 'internal purchase'
internal_purchase_transaction_id AS (
SELECT
*
FROM transactions
WHERE owner_id IN (
SELECT id FROM {{ source('src_exchange_mysql', 'bookings') }}
WHERE service_id IN (7) -- INTERNAL PURCHASE
)
AND owner_type = 'App\\Models\\Booking'
),
exclude_internal_purchase_bookings AS (
SELECT
*
FROM transactions
WHERE id NOT IN (
SELECT id FROM internal_purchase_transaction_id
)
),
transactions_rename AS (
SELECT
transactions.id AS transaction_id,
transactions.owner_type,
transactions.owner_id,
transactions.type AS transaction_type,
transactions.issuer AS issuer_user_id,
transactions.receiver AS receiver_user_id,
transactions.recipient_bank_account_id AS recipient_bank_id,
transactions.payment_method,
transactions.payment_reference,
transactions.bill_no AS bill_number,
transactions.amount AS base_value,
transactions.original_amount AS quote_value,
transactions.currency_id AS base_currency_id,
transactions.original_currency_id AS quote_currency_id,
transactions.currency_rate AS base_to_quote_currency_exchange_rate,
transactions.tax AS base_tax,
transactions.service_charge AS base_service_charge,
transactions.expires_on AS expired_datetime,
transactions.status,
transactions.deleted_at AS deleted_datetime,
transactions.created_at AS created_datetime,
transactions.updated_at AS updated_datetime,
exclude_internal_purchase_bookings.id AS transaction_id,
exclude_internal_purchase_bookings.owner_type,
exclude_internal_purchase_bookings.owner_id,
exclude_internal_purchase_bookings.type AS transaction_type,
exclude_internal_purchase_bookings.issuer AS issuer_user_id,
exclude_internal_purchase_bookings.receiver AS receiver_user_id,
exclude_internal_purchase_bookings.recipient_bank_account_id AS recipient_bank_id,
exclude_internal_purchase_bookings.payment_method,
exclude_internal_purchase_bookings.payment_reference,
exclude_internal_purchase_bookings.bill_no AS bill_number,
exclude_internal_purchase_bookings.amount AS base_value,
exclude_internal_purchase_bookings.original_amount AS quote_value,
exclude_internal_purchase_bookings.currency_id AS base_currency_id,
exclude_internal_purchase_bookings.original_currency_id AS quote_currency_id,
exclude_internal_purchase_bookings.currency_rate AS base_to_quote_currency_exchange_rate,
exclude_internal_purchase_bookings.tax AS base_tax,
exclude_internal_purchase_bookings.service_charge AS base_service_charge,
exclude_internal_purchase_bookings.expires_on AS expired_datetime,
exclude_internal_purchase_bookings.status,
exclude_internal_purchase_bookings.deleted_at AS deleted_datetime,
exclude_internal_purchase_bookings.created_at AS created_datetime,
exclude_internal_purchase_bookings.updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM transactions
FROM exclude_internal_purchase_bookings
),
@@ -74,6 +105,7 @@ final__base_exchange__transactions AS (
_dbt_ran_datetime
FROM transactions_rename
)
SELECT * FROM final__base_exchange__transactions
models/staging/exchange/stg_exchange__currency_vendor_payment_proof.sql
+71
View File
@@ -0,0 +1,71 @@
/*
One order can have multiple bookings.
Therefore, multiple china bank slip can be uploaded to one order, which is associated to each booking.
Currency_vendor_payment_proof will always have pending_verification as their status.
*/
-- IMPORT
WITH documents AS (
SELECT * FROM {{ ref('base_exchange__documents') }}
),
seed_status AS (
SELECT * FROM {{ ref('seed_exchange__status') }}
WHERE category = 'DEFAULT'
),
-- LOGIC
documents_join_status AS (
SELECT
documents.document_id,
documents.owner_id AS transaction_cost_id,
documents.document_type,
seed_status.name AS status,
documents.created_datetime,
documents.updated_datetime,
documents.deleted_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM documents
LEFT JOIN seed_status
ON (documents.status = seed_status.id)
WHERE
documents.document_type = 'CURRENCY_VENDOR_PAYMENT_PROOF'
AND
deleted_datetime IS NULL
),
-- FINAL
final__stg_exchange__currency_vendor_payment_proof AS (
SELECT
-- ids
document_id,
transaction_cost_id,
-- dimensions
document_type,
status,
-- measures
-- date/times
created_datetime,
updated_datetime,
deleted_datetime,
-- metadata
_dbt_ran_datetime
FROM documents_join_status
)
SELECT * FROM final__stg_exchange__currency_vendor_payment_proof
models/staging/exchange/stg_exchange__customer_payment_proof.sql
+86
View File
@@ -0,0 +1,86 @@
-- IMPORT
WITH documents AS (
SELECT * FROM {{ ref('base_exchange__documents') }}
),
seed_status AS (
SELECT * FROM {{ ref('seed_exchange__status') }}
WHERE category = 'DEFAULT'
),
-- LOGIC
documents_join_status AS (
SELECT
documents.document_id,
documents.owner_id AS transaction_order_id,
documents.approver_user_id,
documents.document_type,
seed_status.name AS status,
IFF(seed_status.name = 'REJECTED', documents.updated_datetime, null) AS rejected_datetime,
IFF(seed_status.name = 'APPROVED', documents.updated_datetime, null) AS approved_datetime,
documents.created_datetime,
documents.updated_datetime,
documents.deleted_datetime
FROM documents
LEFT JOIN seed_status
ON (documents.status = seed_status.id)
WHERE
documents.document_type = 'CUSTOMER_PAYMENT_PROOF'
),
/*
Few orders in year 2021 has separate rows for
when status = 'Pending Verfication' and when status = 'Approved'
*/
filter_latest_documents AS (
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM documents_join_status
QUALIFY
row_number_index = 1
),
-- FINAL
final__stg_exchange__customer_payment_proof AS (
SELECT
-- ids
document_id,
transaction_order_id,
approver_user_id,
-- dimensions
document_type,
status,
-- measures
-- date/times
created_datetime,
approved_datetime,
updated_datetime,
rejected_datetime,
deleted_datetime,
-- metadata
_dbt_ran_datetime
FROM filter_latest_documents
)
SELECT * FROM final__stg_exchange__customer_payment_proof
models/staging/googlesheet/_googlesheet__sources.yml
+16 -17
View File
@@ -6,7 +6,7 @@ sources:
database: DEV_CIEF_RAW_DB
schema: GOOGLESHEET_AIRBYTE
loader: Airbyte
loaded_at_field: _airbyte_emitted_at
loaded_at_field: _airbyte_extracted_at
freshness:
warn_after: {count: 26, period: hour}
error_after: {count: 48, period: hour}
@@ -19,32 +19,31 @@ sources:
tables:
- name: _airbyte_raw_leads_activations
- name: leads_activations
description: This data is get from sales gsheet, sales will call user once they register & not put any order
columns:
- name: _airbyte_ab_id
description: Primary key for '_airbyte_raw_leads_activations'
- name: _airbyte_raw_id
description: Primary key for 'leads_activations'
tests:
- unique
- not_null
- name: _airbyte_data
description: JSON data which come from gsheet
tests:
- not_null
- name: _airbyte_raw_event_and_holiday_lists
- name: event_and_holiday_lists
description: Data retrieved from gsheet, comprises dates for events and holidays that may or may not impact operation of company.
columns:
- name: _airbyte_ab_id
description: Primary key for '_airbyte_raw_event_and_holiday_lists'
- name: _airbyte_raw_id
description: Primary key for 'event_and_holiday_lists'
tests:
- unique
- not_null
- name: _airbyte_data
description: JSON data which come from gsheet
tests:
- not_null
- name: account_frozen_case
description: Data retrieved from gsheet, comprises frozen account cases from Exchange service.
columns:
- name: _airbyte_raw_id
description: Primary key for 'account_frozen_case'
tests:
- unique
- not_null
models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql
+29 -37
View File
@@ -1,44 +1,36 @@
-- IMPORT
WITH holiday_lists AS (
SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_event_and_holiday_lists') }}
SELECT * FROM {{ source('src_googlesheet_airbyte', 'event_and_holiday_lists') }}
),
-- LOGIC
parse_jason_holidays AS (
SELECT
_airbyte_ab_id AS airbyte_id,
parse_json(_airbyte_data) AS holiday_json,
_airbyte_emitted_at AS _airbyte_emitted_datetime
-- LOGIC
data_casting AS (
SELECT
id AS holiday_id,
_airbyte_raw_id AS _airbyte_id,
DATE(date) AS date,
event_cultural_origin AS religion,
event_desc AS event_description,
general_type AS holiday_general_type,
holiday_type AS holiday_specific_type,
special_remark AS remark,
day,
event_name,
country,
state,
is_malaysia_holiday,
is_company_holiday,
is_china_holiday,
is_china_warehouse_holiday,
_airbyte_extracted_at AS _airbyte_extracted_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM holiday_lists
),
flatten_json AS (
SELECT
airbyte_id,
holiday_json['id']::INTEGER AS holiday_id,
holiday_json['date']::DATE AS date,
holiday_json['day']::STRING AS day,
holiday_json['event_name']::STRING AS event_name,
holiday_json['event_desc']::STRING AS event_description,
holiday_json['general_type']::STRING AS holiday_general_type,
holiday_json['holiday_type']::STRING AS holiday_specific_type,
holiday_json['country']::STRING AS country,
holiday_json['state']::STRING AS state,
holiday_json['event_cultural_origin']::STRING AS religion,
holiday_json['is_malaysia_holiday']::INTEGER AS is_malaysia_holiday,
holiday_json['is_company_holiday']::INTEGER AS is_company_holiday,
holiday_json['is_china_holiday']::INTEGER AS is_china_holiday,
holiday_json['is_china_warehouse_holiday']::INTEGER AS is_china_warehouse_holiday,
holiday_json['special_remark']::STRING AS remark,
_airbyte_emitted_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM parse_jason_holidays
),
-- FINAL
final__stg_googlesheet__event_and_holiday_lists AS (
@@ -46,7 +38,6 @@ final__stg_googlesheet__event_and_holiday_lists AS (
SELECT
-- ids
holiday_id,
airbyte_id,
-- dimensions
day,
@@ -61,6 +52,7 @@ final__stg_googlesheet__event_and_holiday_lists AS (
is_company_holiday,
is_china_holiday,
is_china_warehouse_holiday,
remark,
-- measures
@@ -68,11 +60,11 @@ final__stg_googlesheet__event_and_holiday_lists AS (
date,
-- metadata
remark,
_airbyte_emitted_datetime,
_airbyte_id,
_airbyte_extracted_datetime,
_dbt_ran_datetime
FROM flatten_json
FROM data_casting
)
SELECT * FROM final__stg_googlesheet__event_and_holiday_lists
SELECT * FROM final__stg_googlesheet__event_and_holiday_lists
models/staging/googlesheet/stg_googlesheet__sales_exchange_lead_activations.sql
+43 -33
View File
@@ -1,49 +1,57 @@
-- IMPORT
WITH lead_activations AS (
SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_leads_activations') }}
SELECT * FROM {{ source('src_googlesheet_airbyte', 'leads_activations') }}
),
-- LOGIC
parse_json_leads AS (
data_casting AS (
SELECT
_airbyte_ab_id AS sales_team_lead_activation_id,
parse_json(_airbyte_data) AS leads_json,
_airbyte_emitted_at AS _airbyte_emitted_datetime
_airbyte_raw_id AS _airbyte_id,
company_marking AS company_marking_id,
agent_name,
contact_number,
user_name AS contact_name,
email AS contact_email,
remark AS sales_team_remark,
sorce AS contact_method,
tag AS contact_status,
COALESCE(
TRY_TO_TIMESTAMP(register_date::STRING, 'DD-MM-YYYY'),
TRY_TO_TIMESTAMP(register_date::STRING, 'DD-MM-YYYY HH24:MI:SS')
) AS exchange_registered_datetime,
COALESCE(
TRY_TO_TIMESTAMP(user_first_reply_datetime::STRING, 'DD-MM-YYYY'),
TRY_TO_TIMESTAMP(user_first_reply_datetime::STRING, 'DD-MM-YYYY HH24:MI:SS')
) AS first_reply_datetime,
COALESCE(
TRY_TO_TIMESTAMP(contacted_datetime::STRING, 'DD-MM-YYYY'),
TRY_TO_TIMESTAMP(contacted_datetime::STRING, 'DD-MM-YYYY HH24:MI:SS')
) AS sales_team_contacted_datetime,
_airbyte_extracted_at AS _airbyte_extracted_datetime
FROM lead_activations
),
flatten_json AS (
generate_surrogate_key AS (
SELECT
sales_team_lead_activation_id,
leads_json['agent_name']::STRING AS agent_name,
leads_json['company_marking']::STRING AS company_marking_id,
leads_json['contact_number']::STRING AS contact_number,
leads_json['email']::STRING AS contact_email,
leads_json['sorce']::STRING AS contact_method,
leads_json['tag']::STRING AS contact_status,
leads_json['user_name']::STRING AS contact_name,
leads_json['remark']::STRING AS sales_team_remark,
COALESCE(
TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY'),
TRY_TO_TIMESTAMP(leads_json['contacted_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS')
) AS sales_team_contacted_datetime,
COALESCE(
TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY'),
TRY_TO_TIMESTAMP(leads_json['register_date']::STRING, 'DD-MM-YYYY HH24:MI:SS')
) AS exchange_registered_datetime,
COALESCE(
TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY'),
TRY_TO_TIMESTAMP(leads_json['user_first_reply_datetime']::STRING, 'DD-MM-YYYY HH24:MI:SS')
) AS first_reply_datetime,
*,
{{ dbt_utils.generate_surrogate_key(['sales_team_contacted_datetime', 'company_marking_id', 'agent_name']) }} AS sales_team_lead_activation_id,
_airbyte_emitted_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM data_casting
FROM parse_json_leads
),
-- FINAL
final__stg_googlesheet__sales_exchange_lead_activations AS (
@@ -68,11 +76,13 @@ final__stg_googlesheet__sales_exchange_lead_activations AS (
first_reply_datetime,
exchange_registered_datetime,
-- metadata
_airbyte_emitted_datetime,
-- metadata
_airbyte_id,
_airbyte_extracted_datetime,
_dbt_ran_datetime
FROM flatten_json
FROM generate_surrogate_key
)
SELECT * FROM final__stg_googlesheet__sales_exchange_lead_activations