DBT restructure

This commit is contained in:
Yam ZhengLim
2023-04-09 15:13:42 +08:00
parent 1d63223a5f
commit 3db90f2ac6
57 changed files with 949 additions and 115 deletions
@@ -1,46 +0,0 @@
-- IMPORTS
WITH activity_logs AS (
SELECT * FROM {{ ref('base_exchange__activity_logs') }}
),
-- LOGIC
activity_logs_listout AS (
SELECT
activity_logs.id,
activity_logs.log_name,
activity_logs.description,
activity_logs.subject_id,
activity_logs.subject_type,
activity_logs.causer_id,
activity_logs.causer_type,
activity_logs.properties,
activity_logs.created_at,
activity_logs.updated_at,
CURRENT_TIMESTAMP() AS _dbt_ran_at
FROM activity_logs
),
-- FINAL
final__stg_exchange__activity_logs AS (
SELECT
id,
log_name,
description,
subject_id,
subject_type,
causer_id,
causer_type,
properties,
created_at,
updated_at,
_dbt_ran_at
FROM activity_logs_listout
)
SELECT * FROM final__stg_exchange__activity_logs
@@ -0,0 +1,10 @@
-- IMPORTS
WITH wallets AS (
SELECT * FROM {{ ref('base_exchange__wallets') }}
)
-- FINAL
SELECT * FROM wallets
@@ -0,0 +1,7 @@
-- IMPORTS
with wallet_logs as (select * from {{ ref("base_exchange__wallet_logs") }})
-- FINAL
select *
from wallet_logs
order by wallet_id, updated_at asc
@@ -0,0 +1,46 @@
-- IMPORTS
WITH activity_logs AS (
SELECT * FROM {{ source('src_shipping_mysql', 'activity_log') }}
),
-- LOGIC
activity_logs_rename AS (
SELECT
activity_logs.id,
activity_logs.log_name,
activity_logs.description,
activity_logs.subject_id,
activity_logs.subject_type,
activity_logs.causer_id,
activity_logs.causer_type,
activity_logs.properties,
activity_logs.created_at AS created_datetime,
activity_logs.updated_at AS updated_datetime,
current_timestamp() AS _dbt_ran_at
FROM activity_logs
),
-- FINAL
final__base_exchange__activity_logs AS (
SELECT
id,
log_name,
description,
subject_id,
subject_type,
causer_id,
causer_type,
properties,
created_datetime,
updated_datetime,
_dbt_ran_at
FROM activity_logs_rename
)
SELECT * FROM final__base_exchange__activity_logs
@@ -0,0 +1,23 @@
{% docs src_exchange_mysql %}
Data is extract from CIEF Selfhost MYSQL `exchange_production`, using custom python code.
- `_source_loaded_at` will add into the raw data when extract
```YML
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 12, period: hour}
error_after: {count: 24, period: hour}
```
- The raw data will use copy into snowflake in 2AM everyday
- Depend on target.name to determin which snowflake raw schema
```JINJA
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
```
{% enddocs %}
@@ -0,0 +1,6 @@
version: 2
models:
- name: stg_exchange__activity_logs
- name: stg_exchange__announcments
@@ -0,0 +1,619 @@
version: 2
sources:
- name: src_exchange_mysql
description: '{{ doc("src_exchange_mysql") }}'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
loader: custom_python_code
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 12, period: hour}
error_after: {count: 24, period: hour}
meta:
owner: "@yam"
model_maturity: prod
tags:
- exchange
- daily
tables:
- name: activity_log
description: Contains most table's activity log that CUD (created, updated, deleted) in www.exchange.cief-malaysia.com
columns:
- name: id
description: Primary key for 'activity_log'
tests:
- unique
- not_null
- name: description
description: system CUD activity list down
tests:
- accepted_values:
values: ['created', 'updated', 'deleted']
- name: subject_id
description: The exchange CUD table's id, need to refer subject_type column to know what table is CUD in the row
tests:
- not_null
- name: subject_type
description: What table that CUD using in the row
tests:
- not_null
- name: causer_id
description: Need to refer causer_type column to know who is CUD in the row. it is not well maintain there is some NULL
- name: causer_type
description: Who is CUD the row
- name: properties
description: Json data format that contain attributes of the activity, it is not well maintain, subject (Country, ServiceType, CurrencyRate. UserEmailVerification, File) have properties only
- name: created_at
description: Datetime that CUD activity occur
tests:
- not_null
- name: updated_at
description: Exactly same with created_at, redundant column
tests:
- dbt_expectations.expect_column_pair_values_to_be_equal:
column_A: updated_at
column_B: created_at
- name: addresses
description: Addresses that input when user register
columns:
- name: id
description: Primary key for 'addresses'
tests:
- unique
- not_null
- name: company_id
description: Foreign keys for 'companies'
tests:
- not_null
- name: country_id
description: Foreign keys for 'countries'
tests:
- not_null
- name: state_id
description: Foreign keys for 'states'
tests:
- not_null
- name: district_id
description: Foreign keys for 'districts'
tests:
- not_null
- name: postcode
description: The number/string that user input
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: integer
- name: street_one
description: The string that user input
tests:
- not_null
- name: street_two
description: The string that user input
- name: billing
description: If mark 1 means it is billing address, 0 means is not billing address, every company should have 1 billing address
tests:
- dbt_expectations.expect_column_max_to_be_between: #Every company should have a billing address
min_value: 1
max_value: 1
group_by: [company_id]
- dbt_expectations.expect_column_sum_to_be_between: #1 company should have 1 billing address only
min_value: 1
max_value: 1
group_by: [company_id]
row_condition: "deleted_at is null"
- name: deleted_at
description: System generate datetime that address was deleted
- name: created_at
description: System generate datetime that address was created
tests:
- not_null
- name: updated_at
description: System generate datetime that address was updated
tests:
- not_null
- name: announcement_segment
description: Superadmin / Operator guys will segment the announcment, but normally they will leave it default. 2 FK combine to becoem a relation table.
columns:
- name: announcement_id
description: Primary key for 'announcement_segment' & Foreign key for 'announcements'
tests:
- not_null
- name: segment_id
description: Primary key for 'announcement_segment' & Foreign key for 'segments'
tests:
- not_null
- name: created_at
description: Abandon column
tests:
- dbt_expectations.expect_column_values_to_be_null
- name: updated_at
description: Abandon column
tests:
- dbt_expectations.expect_column_values_to_be_null
- name: announcements
description: Announcements that show in our website which create by superadmin
columns:
- name: id
description: Primary key for 'announcements'
tests:
- unique
- not_null
- name: title
description: title of announcement, using string
tests:
- not_null
- name: description
description: description of announcement, using longtext, have <br> html for break
tests:
- not_null
- name: starting_on
description: announcement starting datetime
tests:
- not_null
- name: ending_on
description: announcement ending datetime
tests:
- not_null
- name: is_all_day
description: column might not in use, system auto generate
tests:
- not_null
- accepted_values:
values: [0]
- name: duration
description: column might not in use, system auto generate
tests:
- not_null
- accepted_values:
values: [0]
- name: is_recurring
description: column might not in use, system auto generate
tests:
- not_null
- accepted_values:
values: [0]
- name: recurrence_pattern
description: column might not in use, system auto generate
tests:
- dbt_expectations.expect_column_values_to_be_null
- name: deleted_at
description: system generate datetime that announcment was deleted
tests:
- not_null
- name: created_at
description: system generate datetime that announcment was created
tests:
- not_null
- name: updated_at
description: system generate datetime that announcment was updated
tests:
- not_null
- name: bank_logs
description: Abandon table
config:
enabled: false
- name: banks
description: Table which store banks details.
columns:
- name: id
description: Primary key for 'banks'
tests:
- unique
- not_null
- name: company_id
description: Foreign keys for 'companies'
tests:
- not_null
- name: reference
description: A string which write by user/company for the bank transfer reference or other remind ussage
- name: holder_name
description: Bank holder name
tests:
- not_null
- name: account_no
description: Bank account number
tests:
- not_null
- name: booking_logs
description: Raw `bookings` table from Snowflake
columns:
- name: id
description: Primary key for 'booking_logs'
tests:
- unique
- not_null
- name: bookings
description: Raw `bookings` table from Snowflake
columns:
- name: id
description: Primary key for 'bookings'
tests:
- unique
- not_null
- name: companies
description: Raw `companies` table from Snowflake
columns:
- name: id
description: Primary key for 'companies'
tests:
- unique
- not_null
- name: contacts
description: Raw `contacts` table from Snowflake
columns:
- name: id
description: Primary key for 'contacts'
tests:
- unique
- not_null
- name: countries
description: Raw `countries` table from Snowflake
columns:
- name: id
description: Primary key for 'countries'
tests:
- unique
- not_null
- name: currencies
description: Raw `currencies` table from Snowflake
columns:
- name: id
description: Primary key for 'currencies'
tests:
- unique
- not_null
- name: currency_logs
description: Raw `currency_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'currency_logs'
tests:
- unique
- not_null
- name: currency_rate_logs
description: Raw `currency_rate_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'currency_rate_logs'
tests:
- unique
- not_null
- name: currency_rates
description: Raw `currency_rates` table from Snowflake
columns:
- name: id
description: Primary key for 'currency_rates'
tests:
- unique
- not_null
- name: districts
description: Raw `districts` table from Snowflake
columns:
- name: id
description: Primary key for 'districts'
tests:
- unique
- not_null
- name: documents
description: Raw `documents` table from Snowflake
columns:
- name: id
description: Primary key for 'documents'
tests:
- unique
- not_null
- name: employees
description: Raw `employees` table from Snowflake
columns:
- name: company_id
description: Primary key for 'employees'
- name: failed_jobs
description: Raw `failed_jobs` table from Snowflake
columns:
- name: id
description: Primary key for 'failed_jobs'
tests:
- unique
- not_null
- name: files
description: Raw `files` table from Snowflake
columns:
- name: id
description: Primary key for 'files'
tests:
- unique
- not_null
- name: group_transactions
description: Raw `group_transactions` table from Snowflake
columns:
- name: id
description: Primary key for 'group_transactions'
tests:
- unique
- not_null
- name: groups
description: Raw `groups` table from Snowflake
columns:
- name: id
description: Primary key for 'groups'
tests:
- unique
- not_null
- name: jobs
description: Raw `jobs` table from Snowflake
columns:
- name: id
description: Primary key for 'jobs'
tests:
- unique
- not_null
- name: migrations
description: Raw `migrations` table from Snowflake
columns:
- name: id
description: Primary key for 'migrations'
tests:
- unique
- not_null
- name: model_has_roles
description: Raw `model_has_roles` table from Snowflake
columns:
- name: role_id
description: Primary key for 'model_has_roles'
- name: notifications
description: Raw `notifications` table from Snowflake
columns:
- name: id
description: Primary key for 'notifications'
tests:
- unique
- not_null
- name: password_resets
description: Raw `password_resets` table from Snowflake
columns:
- name: id
description: Primary key for 'password_resets'
tests:
- unique
- not_null
- name: permissions
description: Raw `permissions` table from Snowflake
columns:
- name: id
description: Primary key for 'permissions'
tests:
- unique
- not_null
- name: role_has_permissions
description: Raw `role_has_permissions` table from Snowflake
columns:
- name: permission_id
description: Primary key for 'role_has_permissions'
- name: roles
description: Raw `roles` table from Snowflake
columns:
- name: id
description: Primary key for 'roles'
tests:
- unique
- not_null
- name: segment_companies
description: Raw `segment_companies` table from Snowflake
columns:
- name: segment_id
description: Primary key for 'segment_companies'
- name: segment_constants
description: Raw `segment_constants` table from Snowflake
columns:
- name: id
description: Primary key for 'segment_constants'
tests:
- unique
- not_null
- name: segments
description: Raw `segments` table from Snowflake
columns:
- name: id
description: Primary key for 'segments'
tests:
- unique
- not_null
- name: service_types
description: Raw `service_types` table from Snowflake
columns:
- name: id
description: Primary key for 'service_types'
tests:
- unique
- not_null
description: Raw `states` table from Snowflake
columns:
- name: id
description: Primary key for 'states'
tests:
- unique
- not_null
- name: transaction_detail
description: Raw `transaction_detail` table from Snowflake
columns:
- name: id
description: Primary key for 'transaction_detail'
tests:
- unique
- not_null
- name: transaction_logs
description: Raw `transaction_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'transaction_logs'
tests:
- unique
- not_null
- name: transactions
description: Raw `transactions` table from Snowflake
columns:
- name: id
description: Primary key for 'transactions'
tests:
- unique
- not_null
- name: user_email_verifications
description: Raw `user_email_verifications` table from Snowflake
columns:
- name: id
description: Primary key for 'user_email_verifications'
tests:
- unique
- not_null
- name: users
description: Raw `users` table from Snowflake
columns:
- name: id
description: Primary key for 'users'
tests:
- unique
- not_null
- name: wallet_logs
description: Raw `wallet_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'wallet_logs'
tests:
- unique
- not_null
- name: wallets
description: Raw `wallets` table from Snowflake
columns:
- name: id
description: Primary key for 'wallets'
tests:
- unique
- not_null
@@ -0,0 +1,94 @@
-- IMPORTS
WITH activity_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'activity_log') }}
),
-- LOGIC
activity_logs_rename AS (
SELECT
activity_logs.id,
activity_logs.log_name,
activity_logs.description,
activity_logs.subject_id,
activity_logs.subject_type,
activity_logs.causer_id,
activity_logs.causer_type,
activity_logs.properties,
activity_logs.created_at,
activity_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM activity_logs
),
-- FINAL
final__base_exchange__activity_logs AS (
SELECT
id,
log_name,
description,
subject_id,
subject_type,
causer_id,
causer_type,
properties,
created_at,
updated_at,
_dbt_ran_at
FROM activity_logs_rename
)
SELECT * FROM final__base_exchange__activity_logs
-- IMPORTS
WITH activity_logs AS (
SELECT * FROM {{ ref('base_exchange__activity_logs') }}
),
-- LOGIC
activity_logs_listout AS (
SELECT
activity_logs.id,
activity_logs.log_name,
activity_logs.description,
activity_logs.subject_id,
activity_logs.subject_type,
activity_logs.causer_id,
activity_logs.causer_type,
activity_logs.properties,
activity_logs.created_at,
activity_logs.updated_at,
CURRENT_TIMESTAMP() AS _dbt_ran_at
FROM activity_logs
),
-- FINAL
final__stg_exchange__activity_logs AS (
SELECT
id,
log_name,
description,
subject_id,
subject_type,
causer_id,
causer_type,
properties,
created_at,
updated_at,
_dbt_ran_at
FROM activity_logs_listout
)
SELECT * FROM final__stg_exchange__activity_logs