cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-19 04:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'compile_company_dimension_for_dedupe' into 'main'

Browse Source 
Compile company dimension for dedupe

See merge request cief-data/dbt_cloud!20
This commit is contained in:
Yam ZhengLim
2023-08-08 03:15:40 +00:00
parent a365b87988 52173653c0
commit 4d5d88ac89
11 changed files with 484 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
analyses/company_latest_order_datetime.sql
+7
View File
File diff suppressed because one or more lines are too long
models/marts/warehouse/dim__dates.sql → analyses/dim__dates_backup.sql
View File
models/marts/reporting/rep_exchange__daily_orders.sql
+64 -64
View File
@@ -28,7 +28,7 @@ dates AS (
transaction_orders_get_next_order_created_datetime AS (
SELECT
*,
LEAD(order_created_datetime) OVER (PARTITION BY company_id ORDER BY order_created_datetime) AS next_order_created_datetime
LEAD(order_created_datetime) OVER (PARTITION BY company_id ORDER BY order_created_datetime, transaction_order_id) AS next_order_created_datetime
FROM
transaction_orders
@@ -40,66 +40,66 @@ fct_and_dim_joins AS (
bookings.booking_marking_id,
bookings.company_id,
companies.company_marking_id AS company_marking_id,
companies.autocount_id AS company_autocount_id,
companies.company_type AS company_type,
companies.business_type AS company_business_type,
companies.exchange_rate_segment AS company_exchange_rate_segment,
companies.is_migrated_company AS is_migrated_company,
companies.country_name AS company_country_name,
companies.state_name AS company_state_name,
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.has_wallet AS company_has_wallet,
companies.latitude AS company_latitude,
companies.longitude AS company_longitude,
companies.company_marking_id AS company_marking_id,
companies.autocount_id AS company_autocount_id,
companies.company_type AS company_type,
companies.business_type AS company_business_type,
companies.exchange_rate_segment AS company_exchange_rate_segment,
companies.is_migrated_company AS is_migrated_company,
companies.country_name AS company_country_name,
companies.state_name AS company_state_name,
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.has_wallet AS company_has_wallet,
companies.latitude AS company_latitude,
companies.longitude AS company_longitude,
bookings.user_id,
bookings.bank_id,
fix_currencies.currency_name AS fix_currency_name,
fix_currencies.currency_name AS fix_currency_name,
quote_currencies.currency_name AS quote_currency_name,
base_currencies.currency_name AS base_currency_name,
bookings.service_type,
bookings.status AS booking_status,
bookings.is_first_time_company AS is_first_time_booking_company,
bookings.is_first_time_company_completed AS is_first_time_booking_company_completed,
bookings.is_first_time_user AS is_first_time_booking_user,
bookings.is_first_time_user_completed AS is_first_time_booking_user_completed,
bookings.estimate_base_to_quote_currency_exchange_rate AS estimate_booking_base_to_quote_currency_exchange_rate,
bookings.estimate_quote_value AS estimate_booking_quote_value,
bookings.estimate_base_value AS estimate_booking_base_value,
bookings.estimate_value_rm AS estimate_booking_value_rm,
bookings.status AS booking_status,
bookings.is_first_time_company AS is_first_time_booking_company,
bookings.is_first_time_company_completed AS is_first_time_booking_company_completed,
bookings.is_first_time_user AS is_first_time_booking_user,
bookings.is_first_time_user_completed AS is_first_time_booking_user_completed,
bookings.estimate_base_to_quote_currency_exchange_rate AS estimate_booking_base_to_quote_currency_exchange_rate,
bookings.estimate_quote_value AS estimate_booking_quote_value,
bookings.estimate_base_value AS estimate_booking_base_value,
bookings.estimate_value_rm AS estimate_booking_value_rm,
bookings.booking_created_datetime,
bookings.booking_approved_datetime,
bookings.booking_completed_datetime,
bookings.booking_suspended_datetime,
transaction_orders_get_next_order_created_datetime.transaction_order_id AS order_id,
transaction_orders_get_next_order_created_datetime.payment_method AS order_payment_method,
transaction_orders_get_next_order_created_datetime.transaction_order_id AS order_id,
transaction_orders_get_next_order_created_datetime.payment_method AS order_payment_method,
transaction_orders_get_next_order_created_datetime.order_status,
transaction_orders_get_next_order_created_datetime.payment_reference AS order_payment_reference,
transaction_orders_get_next_order_created_datetime.bill_number AS order_bill_number,
transaction_orders_get_next_order_created_datetime.is_first_time_company AS is_first_time_order_company,
transaction_orders_get_next_order_created_datetime.is_first_time_company_completed AS is_first_time_order_company_completed,
transaction_orders_get_next_order_created_datetime.is_first_time_user AS is_first_time_order_user,
transaction_orders_get_next_order_created_datetime.is_first_time_user_completed AS is_first_time_order_user_completed,
transaction_orders_get_next_order_created_datetime.base_to_quote_currency_exchange_rate AS order_base_to_quote_currency_exchange_rate ,
transaction_orders_get_next_order_created_datetime.base_value AS order_base_value ,
transaction_orders_get_next_order_created_datetime.base_service_charge AS order_base_service_charge ,
transaction_orders_get_next_order_created_datetime.base_tax AS order_base_tax ,
transaction_orders_get_next_order_created_datetime.quote_value AS order_quote_value ,
transaction_orders_get_next_order_created_datetime.value_rm AS order_value_rm ,
transaction_orders_get_next_order_created_datetime.service_charge_rm AS order_service_charge_rm ,
transaction_orders_get_next_order_created_datetime.tax_rm AS order_tax_rm ,
transaction_orders_get_next_order_created_datetime.total_value_rm AS total_order_value_rm,
transaction_orders_get_next_order_created_datetime.payment_reference AS order_payment_reference,
transaction_orders_get_next_order_created_datetime.bill_number AS order_bill_number,
transaction_orders_get_next_order_created_datetime.is_first_time_company AS is_first_time_order_company,
transaction_orders_get_next_order_created_datetime.is_first_time_company_completed AS is_first_time_order_company_completed,
transaction_orders_get_next_order_created_datetime.is_first_time_user AS is_first_time_order_user,
transaction_orders_get_next_order_created_datetime.is_first_time_user_completed AS is_first_time_order_user_completed,
transaction_orders_get_next_order_created_datetime.base_to_quote_currency_exchange_rate AS order_base_to_quote_currency_exchange_rate ,
transaction_orders_get_next_order_created_datetime.base_value AS order_base_value ,
transaction_orders_get_next_order_created_datetime.base_service_charge AS order_base_service_charge ,
transaction_orders_get_next_order_created_datetime.base_tax AS order_base_tax ,
transaction_orders_get_next_order_created_datetime.quote_value AS order_quote_value ,
transaction_orders_get_next_order_created_datetime.value_rm AS order_value_rm ,
transaction_orders_get_next_order_created_datetime.service_charge_rm AS order_service_charge_rm ,
transaction_orders_get_next_order_created_datetime.tax_rm AS order_tax_rm ,
transaction_orders_get_next_order_created_datetime.total_value_rm AS total_order_value_rm,
transaction_orders_get_next_order_created_datetime.order_created_datetime,
order_created_dates.first_day_of_week AS order_created_week,
order_created_dates.first_day_of_week AS order_created_week,
transaction_orders_get_next_order_created_datetime.order_pending_submission_datetime,
transaction_orders_get_next_order_created_datetime.order_pending_verification_datetime,
transaction_orders_get_next_order_created_datetime.order_approved_datetime,
@@ -108,22 +108,22 @@ fct_and_dim_joins AS (
transaction_orders_get_next_order_created_datetime.order_suspended_datetime,
transaction_orders_get_next_order_created_datetime.next_order_created_datetime,
transaction_costs.transaction_cost_id AS cost_id,
transaction_costs.bank_id AS currency_vendor_bank_id,
transaction_costs.transaction_type AS cost_transaction_type,
transaction_costs.payment_method AS cost_payment_method,
transaction_costs.status AS cost_status,
transaction_costs.payment_reference AS cost_payment_reference,
transaction_costs.bill_number AS cost_bill_number,
transaction_costs.base_value AS cost_base_value,
transaction_costs.quote_value AS cost_quote_value,
transaction_costs.transaction_cost_id AS cost_id,
transaction_costs.bank_id AS currency_vendor_bank_id,
transaction_costs.transaction_type AS cost_transaction_type,
transaction_costs.payment_method AS cost_payment_method,
transaction_costs.status AS cost_status,
transaction_costs.payment_reference AS cost_payment_reference,
transaction_costs.bill_number AS cost_bill_number,
transaction_costs.base_value AS cost_base_value,
transaction_costs.quote_value AS cost_quote_value,
transaction_costs.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate,
transaction_costs.base_tax AS cost_base_tax,
transaction_costs.base_service_charge AS cost_base_service_charge,
transaction_costs.value_rm AS cost_value_rm,
transaction_costs.tax_rm AS cost_tax_rm,
transaction_costs.service_charge_rm AS cost_service_charge_rm,
transaction_costs.total_value_rm AS total_cost_value_rm,
transaction_costs.base_tax AS cost_base_tax,
transaction_costs.base_service_charge AS cost_base_service_charge,
transaction_costs.value_rm AS cost_value_rm,
transaction_costs.tax_rm AS cost_tax_rm,
transaction_costs.service_charge_rm AS cost_service_charge_rm,
transaction_costs.total_value_rm AS total_cost_value_rm,
transaction_costs.cost_created_datetime,
transaction_costs.cost_pending_submission_datetime,
transaction_costs.cost_pending_verification_datetime,
@@ -144,16 +144,16 @@ fct_and_dim_joins AS (
LEFT JOIN companies
ON (bookings.company_id = companies.company_id)
LEFT JOIN dates AS order_created_dates
ON (transaction_orders_get_next_order_created_datetime.order_created_datetime = order_created_dates.date_day)
LEFT JOIN dates AS order_created_dates
ON (DATE(transaction_orders_get_next_order_created_datetime.order_created_datetime) = order_created_dates.date_day)
LEFT JOIN currencies AS fix_currencies
LEFT JOIN currencies AS fix_currencies
ON (bookings.fix_currency_id = fix_currencies.currency_id)
LEFT JOIN currencies AS quote_currencies
LEFT JOIN currencies AS quote_currencies
ON (bookings.quote_currency_id = quote_currencies.currency_id)
LEFT JOIN currencies AS base_currencies
LEFT JOIN currencies AS base_currencies
ON (bookings.base_currency_id = base_currencies.currency_id)
),
@@ -258,4 +258,4 @@ final__rep_exchange__daily_orders AS (
FROM fct_and_dim_joins
)
SELECT * FROM final__rep_exchange__daily_orders
SELECT * FROM final__rep_exchange__daily_orders
models/marts/warehouse/dim__dates.py
+80
View File
@@ -0,0 +1,80 @@
# Python script to compute advance working dates
# Weekend Example: using Saturday or Sunday and add 2 working days:
# Monday = 1 working day, Tuesday = 1 working day, Wednesday = Delivery Date
# Assumption for weekends, base date is NOT considered as one working day
# Weekday Example: using Monday and adding 2 working days:
# Monday = 1 working day, Tuesday = 1 working day, Wednesday = Delivery Date
# Assumption for weekdays, base date is considered as one working day
# Forecasting on future dates without information on holidays will not be accurate.
import pandas as pd
from datetime import datetime as dt, timedelta
# Function to check if a given date is a weekend [Saturday(5) or Sunday(6)]
def is_weekend(date):
return (date.weekday() == 5) | (date.weekday() == 6)
# Function to compute the in advance working days
def compute_working_days(start_date, num_working_days, holiday_list, after_cut_off = 0):
# after_cut_off = 0 for orders before 4pm
# after_cut_off = 1 for orders after 4pm
counter = 0
# For dates on weekend or holiday or after cut off time, additional 1 working day to the loop
if (after_cut_off == 1) | is_weekend(start_date) | (start_date in holiday_list):
counter -= 1
# Loop to increase n number of working days, if weekend/holiday, skip counter
while counter < num_working_days:
start_date = start_date + timedelta(days=1)
if is_weekend(start_date) | (start_date in holiday_list):
continue
counter += 1
return start_date
# Main function
def model(dbt, session):
# Setting configuration
dbt.config(materialized="table",
packages = ["pandas"])
# Import data from upstream dbt model
df_date = dbt.ref("int__dates")
# Extract holiday dates into a list
df_work = df_date.to_pandas()
df_new = df_work[df_work["IS_COMPANY_HOLIDAY"] == 1]
df_holiday_list = df_new["DATE_DAY"].tolist()
# Apply function to df
df_work["ADD_1_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 0), axis=1)
df_work["ADD_2_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 0), axis=1)
df_work["ADD_3_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 0), axis=1)
df_work["ADD_5_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 5, df_holiday_list, 0), axis=1)
df_work["ADD_30_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 0), axis=1)
df_work["ADD_90_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 0), axis=1)
df_work["ADD_365_WORKING_DAY_INCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 0), axis=1)
# For orders after cut off time 4pm
# Affects only the weekdays
df_work["ADD_1_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 1, df_holiday_list, 1), axis=1)
df_work["ADD_2_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 2, df_holiday_list, 1), axis=1)
df_work["ADD_3_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 3, df_holiday_list, 1), axis=1)
df_work["ADD_5_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 5, df_holiday_list, 1), axis=1)
df_work["ADD_30_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 30, df_holiday_list, 1), axis=1)
df_work["ADD_90_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 90, df_holiday_list, 1), axis=1)
df_work["ADD_365_WORKING_DAY_EXCLUDED_BASE_DATE"] = df_work.apply(lambda row: compute_working_days(row["DATE_DAY"], 365, df_holiday_list, 1), axis=1)
return df_work
models/staging/exchange/base/base_exchange__account_statements.sql
+52
View File
@@ -0,0 +1,52 @@
-- IMPORTS
WITH source AS (
SELECT * FROM {{ source('src_exchange_mysql', 'account_statements') }}
),
-- LOGIC
account_statements_rename as (
SELECT
id AS account_statement_id,
statement_account_id,
date(date_from) AS account_statement_from_date,
date(date_to) AS account_statement_to_date,
total_amount AS total_value,
begin_balance AS starting_balance,
end_balance AS closing_balance,
created_at AS created_datetime,
updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
),
-- FINAL
final__base_exchange__account_statements AS (
SELECT
-- ids
account_statement_id,
statement_account_id,
-- dimensions
starting_balance,
closing_balance,
-- measures
total_value,
-- date/times
account_statement_from_date,
account_statement_to_date,
created_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM account_statements_rename
)
SELECT * FROM final__base_exchange__account_statements
models/staging/exchange/base/base_exchange__statement_accounts.sql
+50
View File
@@ -0,0 +1,50 @@
-- IMPORTS
WITH source AS (
SELECT * FROM {{ source('src_exchange_mysql', 'statement_accounts') }}
),
-- LOGIC
statement_accounts_rename as (
SELECT
id AS statement_account_id,
number AS account_number,
type AS account_type,
name AS company_name,
currency AS currency_name,
created_at AS created_datetime,
updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
),
-- FINAL
final__base_exchange__statement_accounts AS (
SELECT
-- ids
statement_account_id,
-- dimensions
account_number,
account_type,
company_name,
currency_name,
-- measures
-- date/times
created_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM statement_accounts_rename
)
SELECT * FROM final__base_exchange__statement_accounts
models/staging/exchange/base/base_exchange__statement_transaction_owners.sql
+62
View File
@@ -0,0 +1,62 @@
-- IMPORTS
WITH source AS (
SELECT * FROM {{ source('src_exchange_mysql', 'statement_transaction_owners') }}
),
-- LOGIC
statement_transaction_owners_rename as (
SELECT
id AS statement_transaction_owner_id,
statement_transaction_id,
type,
system,
owner_type,
owner_id,
owner_reference,
invoice_reference,
receipt_reference,
is_auto_mapped,
status,
created_at AS created_datetime,
updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
),
-- FINAL
final__base_exchange__statement_transaction_owners AS (
SELECT
-- ids
statement_transaction_owner_id,
statement_transaction_id,
owner_id,
-- dimensions
system,
type,
owner_type,
owner_reference,
invoice_reference,
receipt_reference,
is_auto_mapped,
status,
-- measures
-- date/times
created_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM statement_transaction_owners_rename
)
SELECT * FROM final__base_exchange__statement_transaction_owners
models/staging/exchange/base/base_exchange__statement_transactions.sql
+68
View File
@@ -0,0 +1,68 @@
-- IMPORTS
WITH source AS (
SELECT * FROM {{ source('src_exchange_mysql', 'statement_transactions') }}
),
-- LOGIC
statement_transactions_rename as (
SELECT
id AS statement_transaction_id,
account_statement_id,
transaction_date AS transaction_datetime,
posting_date AS posting_datetime,
transaction_description,
transaction_description_2,
transaction_description_3,
transaction_description_4,
transaction_description_5,
transaction_ref AS transaction_reference,
amount,
teller_id,
branch_channel AS branch_channel_id,
transaction_code,
end_balance AS closing_balance,
created_at AS created_datetime,
updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
),
-- FINAL
final__base_exchange__statement_transactions AS (
SELECT
-- ids
statement_transaction_id,
account_statement_id,
branch_channel_id,
-- dimensions
transaction_description,
transaction_description_2,
transaction_description_3,
transaction_description_4,
transaction_description_5,
transaction_reference,
transaction_code,
amount,
closing_balance,
-- measures
-- date/times
transaction_datetime,
posting_datetime,
created_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM statement_transactions_rename
)
SELECT * FROM final__base_exchange__statement_transactions
models/staging/exchange/base/base_exchange__voucher_redemptions.sql
+50
View File
@@ -0,0 +1,50 @@
-- IMPORTS
WITH source AS (
SELECT * FROM {{ source('src_exchange_mysql', 'voucher_redemptions') }}
),
-- LOGIC
voucher_redemptions_rename AS (
SELECT
id AS voucher_redemption_id,
voucher_id,
transaction_id,
redemption_id,
value AS voucher_credit,
created_at AS created_datetime,
updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
),
-- FINAL
final__base_exchange__voucher_redemptions AS (
SELECT
-- ids
voucher_redemption_id,
voucher_id,
transaction_id,
redemption_id,
-- dimensions
voucher_credit,
-- measures
-- date/times
created_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM voucher_redemptions_rename
)
SELECT * FROM final__base_exchange__voucher_redemptions
models/staging/exchange/base/base_exchange__vouchers.sql
+50
View File
@@ -0,0 +1,50 @@
-- IMPORTS
WITH source AS (
SELECT * FROM {{ source('src_exchange_mysql', 'vouchers') }}
),
-- LOGIC
vouchers_rename AS (
SELECT
id AS voucher_id,
name,
code,
type,
value AS voucher_credit,
created_at AS created_datetime,
updated_at AS updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
),
-- FINAL
final__base_exchange__vouchers AS (
SELECT
-- ids
voucher_id,
-- dimensions
name,
code,
type,
voucher_credit,
-- measures
-- date/times
created_datetime,
updated_datetime,
-- metadata
_dbt_ran_datetime
FROM vouchers_rename
)
SELECT * FROM final__base_exchange__vouchers
models/staging/exchange/stg_exchange__company_addresses.sql
+1 -1
View File
@@ -1,7 +1,7 @@
-- A flag indicator to determine whether to run a full_load or an incremental_load
-- If user is a developer, a full load will be executed.
-- If the target schema is set to 'dbt_prod', only an incremental load will be performed.
{% if target.name == 'dbt_prod' %}{% set one_month_incremental_flag = true %}{% else %}{% set one_month_incremental_flag = false %}{% endif %}
{% if target.name == 'dbt_prod' %}{% set one_month_incremental_flag = true %}{% else %}{% set one_month_incremental_flag = true %}{% endif %}
-- MODEL CONFIGURATION BLOCK
{{