cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-20 12:54:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sat & Ger Develop Files

Browse Source
This commit is contained in:
Yam ZhengLim
2023-04-09 15:39:46 +08:00
parent 3db90f2ac6
commit 6e225c7e50
13 changed files with 799 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/exchange/intermediate/wallet_log_creadit_amount/int_exchange__wallet_last_transaction_sequence.sql
+8
View File
@@ -0,0 +1,8 @@
with
wallet_log_sequence as (
select * from {{ ref("int_exchange__wallet_transaction_sequence") }}
)
select max(operation_sequence) as last_transaction_id, wallet_id
from wallet_log_sequence
group by wallet_id
models/exchange/intermediate/wallet_log_creadit_amount/int_exchange__wallet_log_credit_amount.sql
+56
View File
@@ -0,0 +1,56 @@
-- IMPORTS
with
wallet_logs as (select * from {{ ref("stg_exchange__wallet_logs") }}),
-- LOGIT
wallet_credit_amount as (
select
*,
lag(amount) over (
partition by wallet_id order by updated_at
) as previous_wallet_balance,
amount - lag(amount) over (
partition by wallet_id order by updated_at
) as credited_amount
from wallet_logs
),
wallet_credit_amount_reformed as (
select
id,
wallet_id,
previous_wallet_balance,
amount as current_wallet_balance,
iff(
equal_null(previous_wallet_balance, null), amount, credited_amount
) as credited_amount,
owner_type,
owner_id,
code,
currency_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
from wallet_credit_amount
),
final_wallet_credit_amount as (
select
id,
wallet_id,
previous_wallet_balance,
current_wallet_balance,
credited_amount,
iff(credited_amount > 0, 'CREDIT', 'DEBIT') as credit_status,
owner_type,
owner_id,
code,
currency_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
from wallet_credit_amount_reformed
)
select *
from final_wallet_credit_amount
models/exchange/intermediate/wallet_log_creadit_amount/int_exchange__wallet_transaction_sequence.sql
+13
View File
@@ -0,0 +1,13 @@
with
wallet_credit_log as (
select * from {{ ref("int_exchange__wallet_log_credit_amount") }}
),
wallet_credit_with_sequence as (
select
dense_rank() over (
partition by wallet_id order by updated_at
) as operation_sequence,
*
from wallet_credit_log
)
select * from wallet_credit_with_sequence
models/exchange/intermediate/wallet_log_creadit_amount/int_exchange_wallet_final_log_sequence.sql
+24
View File
@@ -0,0 +1,24 @@
with
wallet_log_sequence as (
select * from {{ ref("int_exchange__wallet_transaction_sequence") }}
),
wallet_last_log as (
select * from {{ ref("int_exchange__wallet_last_transaction_sequence") }}
)
select
s.id,
s.wallet_id,
s.current_wallet_balance,
s.credit_status,
s.owner_type,
s.owner_id,
s.code,
s.currency_id,
s.deleted_at,
s.created_at,
s.updated_at,
s._dbt_ran_at
from wallet_log_sequence s
inner join wallet_last_log l
where s.wallet_id = l.wallet_id and s.operation_sequence = l.last_transaction_id
models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml
+71 -58
View File
@@ -2,27 +2,51 @@ version: 2
models:
- name: rep_exchange__daily_orders
# @YAM: This is the test code that I write
# @YAM: I already do some space in the YML, but not all the stuff is clean, please clean the code, and makesure it is easy readable.
# columns:
# - name: base_value
# description: test base value combine with dbt_utils
# tests:
# - not_null
# - dbt_utils.expression_is_true:
# expression: " >= 0"
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: order_completed_datetime
column_B: order_created_datetime
or_equal: True
- dbt_utils.expression_is_true:
expression: "COST_TRANSACTION_VALUE_RM + COST_TRANSACTION_TAX_RM + COST_TRANSACTION_SERVICE_CHARGE_RM < 2*(TRANSACTION_VALUE_RM + TRANSACTION_BASE_TAX + TRANSACTION_TAX_RM)"
severity: warn
columns:
- name: id
- name: ID
description: transaction orders id
tests:
- unique
- not_null
- name: booking_id
- name: BOOKING_ID
description: transaction orders id
tests:
- not_null
- relationships:
to: ref('base_exchange__bookings')
field: id
- name: company_marking_id
description: transaction orders id
tests:
- not_null
- name: service_type
- name: SERVICE_TYPE
description: type of service
tests:
- not_null
- name: service_type_name
- name: SERVICE_TYPE_NAME
description: name of type of service
tests:
- not_null
@@ -35,55 +59,55 @@ models:
- name: seller_company_id
description: seller company id
tests:
- not_null
- not_null
- name: bank_id
description: bank id
tests:
- not_null
- not_null
- name: company_id
description: user id
tests:
- not_null
- name: company_marking_id
description: company marking id
- not_null
- name: marking_id
description: marking id
tests:
- relationships:
to: ref('base_exchange__companies')
field: marking_id
to: ref('base_exchange_companies')
field: id
- name: company_type
description: type of company
tests:
- not_null
- name: company_type_name
- not_null
- name: COMPANY_TYPE_NAME
description: name type of company
tests:
- accepted_values:
values:
- PERSONAL
- CORPORATE
- CORPORATE
- name: company_business_type
description: company business type
tests:
- not_null
- not_null
- name: company_business_type_name
description: company business type name
tests:
- accepted_values:
values:
- IMPORTER
- name: company_segment_id
description: company segment id
tests:
- not_null
- not_null
- name: company_segment_name
description: company segment name
tests:
@@ -92,120 +116,109 @@ models:
- PLATINIUM MEMBER
- GOLD MEMBER
- STANDARD SEGMENT
- name: cost_id
description: cost id
tests:
- relationships:
to: ref('base_exchange__transactions')
field: id
- name: base_value
description: test base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: quote_value
description: test quote value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_value_rm
description: transaction value rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_base_tax
description: test transaction base tax combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: base_service_charge
description: test base service charge combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_tax_rm
description: test transaction tax rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_value
description: test cost base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_to_quote_currency_exchange_rate
description: test cost base to quote currency exchange rate combine with dbt_utils
- name: COST_BASE_TO_QUOTE_CURRENCY_EXCHANGE_RATE
description: test COST BASE TO QUOTE CURRENCY EXCHANGE RATE combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_quote_value
description: test cost quote value combine with dbt_utils
- name: COST_QUOTE_VALUE
description: test COST QUOTE VALUE combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_value_rm
- name: COST_TRANSACTION_VALUE_RM
description: test COST TRANSACTION VALUE RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_tax_rm
- name: COST_TRANSACTION_TAX_RM
description: test COST TRANSACTION TAX RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_service_charge
- name: COST_BASE_SERVICE_CHARGE
description: test COST BASE SERVICE CHARGE combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_service_charge_rm
- name: COST_TRANSACTION_SERVICE_CHARGE_RM
description: test COST TRANSACTION SERVICE CHARGE_RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_service_charge_rm
description: test transaction service charge rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: order_completed_datetime
column_B: order_created_datetime
or_equal: True
tests:
- dbt_utils.expression_is_true:
expression: "cost_transaction_value_rm + cost_transaction_tax_rm + cost_transaction_service_charge_rm < 2*(transaction_value_rm + transaction_base_tax + transaction_tax_rm)"
severity: warn
# FRESHNESS SOURCE
models/exchange/marts/warehouse/fct_exchange__wallet_debit_time_duration.sql
+45
View File
@@ -0,0 +1,45 @@
with
wallet_credit_log as (
select * from {{ ref("int_exchange__wallet_log_credit_amount") }}
),
wallet_debit_time_log as (
select
dense_rank() over (
partition by wallet_id order by updated_at
) as operation_sequence,
lag(updated_at) over (
partition by wallet_id order by updated_at
) previous_update_time,
datediff(
second, previous_update_time, updated_at
) as time_between_operation_in_seconds,
*
from wallet_credit_log
),
updated_wallet_debit_time_log as (
select
*,
iff(
(credit_status = 'DEBIT'), time_between_operation_in_seconds, 0
) as seconds_required_to_debit
from wallet_debit_time_log
),
final_wallet_debit_time_log as (
select
* exclude(seconds_required_to_debit),
iff(
equal_null(seconds_required_to_debit, null),
0,
seconds_required_to_debit
) as seconds_required_to_debit
from updated_wallet_debit_time_log
),
final_wallet_debit_time_log_reformed as (
select * exclude(TIME_BETWEEN_OPERATION_IN_SECONDS)
from final_wallet_debit_time_log
where credit_status = 'DEBIT' and seconds_required_to_debit != 0
)
select *
from final_wallet_debit_time_log_reformed
models/exchange/marts/warehouse/fct_exchange__wallet_log_top_10_top_up.sql
+17
View File
@@ -0,0 +1,17 @@
-- IMPORTS
with
wallet_credit_log as (
select * from {{ ref("int_exchange__wallet_log_credit_amount") }}
),
wallet_credit_top_up_frequency_report as (
select credited_amount, count(credited_amount) repetation
from wallet_credit_log
where credit_status = 'CREDIT'
group by credited_amount
order by repetation desc
limit 10
)
select *
from wallet_credit_top_up_frequency_report
models/exchange/marts/warehouse/fct_exchange__wallet_log_wallet_amount_comparison.sql
+24
View File
@@ -0,0 +1,24 @@
with wallet_log as (
select * from {{ref('int_exchange_wallet_final_log_sequence')}}
),
wallet_details as (
select * from {{ref('stg_exchange__wallet')}}
),
wallet_comparison_log as (
select
w.id as wallet_id,
w.amount as wallet_amount,
s.current_wallet_balance as wallet_balance_from_log,
w.owner_type,
s.credit_status,
s.updated_at as wallet_log_update_time,
s._dbt_ran_at,
w.updated_at as wallet_update_time
from wallet_details w
left join
wallet_log s
on w.id = s.wallet_id
)
select * from wallet_comparison_log
models/exchange/marts/warehouse/test_exchange__bookings.yml
+156
View File
@@ -0,0 +1,156 @@
version: 2
models:
- name: fct_exchange__bookings
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: BOOKING_UPDATED_DATETIME
column_B: BOOKING_CREATED_DATETIME
or_equal: True
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: BOOKING_DELETED_DATETIME
column_B: BOOKING_CREATED_DATETIME
or_equal: True
columns:
- name: ID
description: boking id
tests:
- unique
- not_null
- name: COMPANY_ID
description: company id
tests:
- unique
- not_null
- name: USER_ID
description: user id
tests:
- unique
- not_null
- name: MARKING_ID
description: marking id
tests:
- unique
- not_null
- name: SERVICE_TYPE
description: type of service
tests:
- not_null
- accepted_values:
values:
- 1
- 3
- 4
- name: SERVICE_TYPE_NAME
description: name of type of service
tests:
- not_null
- accepted_values:
values:
- 3 DAYS TRANSFER
- 1 DAY TRANSFER
- 1688 PAYMENT
- name: BANK_ID
description: bank id
tests:
- unique
- not_null
- name: FIX_VALUE
description: test fix value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: FIX_CURRENCY_ID
description: fix currency id
tests:
- not_null
- accepted_values:
values:
- 1
- 3
- 2
- name: FIX_CURRENCY_NAME
description: fix currency name
tests:
- not_null
- accepted_values:
values:
- Malaysian Ringgit
- US Dollar
- Yuan Renminbi
- name: ESTIMATE_BASE_TO_QUOTE_CURRENCY_EXCHANGE_RATE
description: test estimate base to quote currency exchange rate value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: QUOTE_CURRENCY_ID
description: quote currency id
tests:
- not_null
- accepted_values:
values:
- 3
- 2
- name: QUOTE_CURRENCY_NAME
description: quote currency name
tests:
- not_null
- accepted_values:
values:
- US Dollar
- Yuan Renminbi
- name: ESTIMATE_QUOTE_VALUE
description: test estimate quote value value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: ESTIMATE_BASE_VALUE
description: test estimate quote value value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: ESTIMATE_VALUE_RM
description: test estimate quote value value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: STATUS
description: status
tests:
- accepted_values:
values:
- 2
- 3
- 5
- name: STATUS_NAME
description: status name
tests:
- accepted_values:
values:
- SUSPENDED
- APPROVED
- COMPLETED
models/exchange/marts/warehouse/test_exchange__companies.yml
+146
View File
@@ -0,0 +1,146 @@
version: 2
models:
- name: dim_exchange__companies
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: ADDRESS_DELETED_AT
column_B: ADDRESS_CREATED_AT
or_equal: True
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: ADDRESS_UPDATED_AT
column_B: ADDRESS_CREATED_AT
or_equal: True
columns:
#Gerson todo Why this company id need to ref with addresses?
- name: ID
description: company id
tests:
- unique
- not_null
- relationships:
to: ref('base_exchange__addresses')
field: company_id
- name: NAME
description: company name
tests:
- not_null
#Yam todo there is marking_id in supplier, need mask them
- name: MARKING_ID
description: marking id
tests:
- unique
- not_null
- name: COMPANY_TYPE
description: type of company
tests:
- accepted_values:
values:
- 0
- 1
- 2
config:
where: "COMPANY_TYPE is not null"
- name: COMPANY_TYPE_NAME
description: name of type company
tests:
- accepted_values:
values:
- CORPORATE
- PERSONAL
config:
where: "COMPANY_TYPE_NAME is not null"
- name: BUSINESS_TYPE
description: type of business
tests:
- accepted_values:
values:
- 1
- 2
- 3
config:
where: "BUSINESS_TYPE is not null"
- name: BUSINESS_TYPE_NAME
description: name of type business
tests:
- accepted_values:
values:
- IMPORTER
- CURRENCY_VENDOR
- FREIGHT_FORWARDER
config:
where: "BUSINESS_TYPE_NAME is not null"
- name: SEGMENT_ID
description: segment id
tests:
- accepted_values:
values:
- 1
- 2
- 3
config:
where: "SEGMENT_ID is not null"
- name: SEGMENT_NAME
description: segment name
tests:
- accepted_values:
values:
- STANDARD SEGMENT
- GOLD MEMBER
- PLATINIUM MEMBER
config:
where: "SEGMENT_NAME is not null"
- name: IS_MIGRATED_COMPANY
description: is migrate company
tests:
- accepted_values:
values:
- 1
- 0
- name: STATUS
description: status
tests:
- accepted_values:
values:
- 0
- 1
- 2
- 4
- 5
- name: STATUS_NAME
description: status name
tests:
- accepted_values:
values:
- SUSPENDED
- APPROVED
- PENDING_SUBMISSION
- REJECTED
- PENDING_VERIFICATION
- name: COMPANY_CONTACT_EMAIL
description: company contact email
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: "[a-zA-Z0-9_]+([.][a-zA-Z0-9_]+)*@[a-zA-Z0-9_]+([.][a-zA-Z0-9_]+)*[.][a-zA-Z]{2,5}"
severity: warn
- name: marking_id
description: marking id
tests:
- relationships:
to: ref('base_exchange_companies')
field: id
models/exchange/marts/warehouse/test_exchange__transaction_orders.yml
+163
View File
@@ -0,0 +1,163 @@
version: 2
models:
- name: fct_exchange__transaction_orders
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: UPDATED_AT
column_B: CREATED_AT
or_equal: True
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: DELETED_AT
column_B: CREATED_AT
or_equal: True
columns:
- name: ID
description: transaction id
tests:
- unique
- not_null
- relationships:
to: ref('int_exchange__transaction_orders_get_user_ids')
field: id
- name: BOOKING_ID
description: booking id
tests:
- unique
- not_null
- relationships:
to: ref('int_exchange__transaction_orders_get_user_ids')
field: BOOKING_ID
- name: SELLER_COMPANY_ID
description: seller company id
tests:
- unique
- not_null
- relationships:
to: ref('int_exchange__transaction_orders_get_user_ids')
field: SELLER_COMPANY_ID
- name: BANK_ID
description: bank id
tests:
- unique
- not_null
- relationships:
to: ref('int_exchange__transaction_orders_get_user_ids')
field: BANK_ID
- name: COMPANY_ID
description: company id
tests:
- unique
- not_null
- relationships:
to: ref('int_exchange__transaction_orders_get_user_ids')
field: COMPANY_ID
- name: STATUS
description: status
tests:
- accepted_values:
values:
- 0
- 1
- 2
- 3
- 4
- 5
- name: STATUS_NAME
description: status name
tests:
- accepted_values:
values:
- COMPLETED
- SUSPENDED
- REJECTED
- PENDING_SUBMISSION
- PENDING_VERIFICATION
- APPROVED
- name: TRANSACTION_TYPE
description: transaction type
tests:
- accepted_values:
values:
- 1
- name: TRANSACTION_TYPE_NAME
description: name transaction type
tests:
- accepted_values:
values:
- PAYMENT
- name: PAYMENT_METHOD
description: method payment
tests:
- accepted_values:
values:
- 1
- 2
- 5
- 4
- name: PAYMENT_METHOD_NAME
description: name method payment
tests:
- accepted_values:
values:
- CASH
- CHEQUE
- PAYMENT_GATEWAY
- WALLET
- name: base_value
description: test base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: quote_value
description: test quote value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_value_rm
description: transaction value rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_base_tax
description: test transaction base tax combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_tax_rm
description: test transaction tax rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: base_service_charge
description: test base service charge combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: TRANSACTION_SERVICE_CHARGE_RM
description: test transaction service change rm value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
models/exchange/marts/warehouse/test_exchange__users.yml
+76
View File
@@ -0,0 +1,76 @@
version: 2
models:
- name: dim_exchange__users
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: DELETED_AT
column_B: CREATED_AT
or_equal: True
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: EMAIL_VERIFICATION_LAST_EMAIL_CREATED_AT
column_B: EMAIL_VERIFICATION_FIRST_EMAIL_CREATED_AT
or_equal: True
columns:
- name: ID
description: user id
tests:
- unique
- not_null
- name: NAME
description: user name
tests:
- not_null
- name: USER_TYPE
description: type of user
tests:
- not_null
- accepted_values:
values:
- 1
- 2
- name: USER_TYPE_NAME
description: type of user
tests:
- not_null
- accepted_values:
values:
- USER
- ADMIN
- name: STATUS
description: user status
tests:
- not_null
- accepted_values:
values:
- 1
- 2
- name: STATUS_NAME
description: user status name
tests:
- not_null
- accepted_values:
values:
- APPROVED
- PENDING_VERIFICATION
- name: COMPANY_ID
description: company id
tests:
- unique
- not_null
- name: EMAIL
description: user email
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: "[a-zA-Z0-9_]+([.][a-zA-Z0-9_]+)*@[a-zA-Z0-9_]+([.][a-zA-Z0-9_]+)*[.][a-zA-Z]{2,5}"
severity: warn
tests/generic_test__zero_or_positive.sql
View File