cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-19 04:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dbt_upload_correct_folder_structure' into 'main'

Browse Source 
DBT structure change

See merge request CIEFWorldwideSdnBhd/dbt_cloud!2
This commit is contained in:
Yam ZhengLim
2023-05-09 09:54:01 +00:00
parent b3373e5b58 2d2501cbfe
commit 291e5ead44
317 changed files with 7448 additions and 10782 deletions
Download Patch File Download Diff File Expand all files Collapse all files
analyses/exchange/blog_ger.sql
-3
View File
@@ -1,3 +0,0 @@
SELECT distinct QUOTE_CURRENCY_ID ,QUOTE_CURRENCY_NAME
FROM {{ ref('fct_exchange__bookings') }}
analyses/exchange/company_conversionrate.sql
-299
View File
@@ -1,299 +0,0 @@
-- HOW TO USE IT?
-- MUST: company_migration_segment filter need to be choose
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
company_identity_documents AS (
SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }}
),
email_verifications AS (
SELECT * FROM {{ ref('fct_exchange__email_verifications') }}
),
-- LOGIC
remove_supplier_company AS (
SELECT
*
FROM companies
WHERE business_type = 2
),
company_register_events AS (
SELECT
remove_supplier_company.id AS company_id,
'Register' AS register_event,
MIN(remove_supplier_company.created_at) AS event_datetime
FROM remove_supplier_company
GROUP BY
remove_supplier_company.id
),
company_email_verification_send_events AS (
SELECT
company_id,
'Email Verification Send' AS email_verification_send_event,
verification_email_created_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY verification_email_created_datetime) AS first_row_number
FROM email_verifications
QUALIFY
first_row_number = 1
),
company_email_verification_complete_events AS (
SELECT
company_id,
'Email Verification Complete' AS email_verification_complete_event,
verification_email_updated_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY verification_email_updated_datetime) AS first_row_number
FROM email_verifications
WHERE
is_completed = '1'
QUALIFY
first_row_number = 1
),
company_upload_identity_events AS (
SELECT
company_id,
'Upload Identity' AS upload_identity_event,
identity_created_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY identity_created_datetime) AS first_row_number
FROM company_identity_documents
QUALIFY
first_row_number = 1
),
company_approve_identity_events AS (
SELECT
company_id,
'Approve Identity' AS approve_identity_event,
identity_approved_datetine AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY identity_created_datetime) AS first_row_number
FROM company_identity_documents
WHERE
identity_status in ('2','3')
QUALIFY
first_row_number = 1
),
company_first_time_order_events AS (
SELECT
first_time_transaction_orders.company_id,
companies.id,
'First Time Order' AS first_time_order_event,
first_time_transaction_orders.created_at AS event_datetime
FROM
(SELECT
*,
row_number() OVER
(PARTITION BY transaction_orders.company_id
ORDER BY transaction_orders.created_at) AS row_rank_index
FROM transaction_orders
QUALIFY
row_rank_index = 1
) AS first_time_transaction_orders
INNER JOIN companies
ON(first_time_transaction_orders.company_id = companies.id)
),
company_funnel_events AS (
SELECT
company_id,
register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_register_events
WHERE register_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_email_verification_send_events
WHERE email_verification_send_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_email_verification_complete_events
WHERE email_verification_complete_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_upload_identity_events
WHERE upload_identity_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_approve_identity_events
WHERE approve_identity_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
first_time_order_event,
event_datetime
FROM company_first_time_order_events
WHERE first_time_order_event IS NOT NULL
),
--FINAL
sem_exchange__company_funnel_events AS (
SELECT
'all_company' AS company_migration_segment,
company_funnel_events.event_datetime,
company_funnel_events.company_id,
remove_supplier_company.is_migrated_company,
count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS register_company,
count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company,
count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company,
count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company,
count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event,
count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company
FROM company_funnel_events
INNER JOIN remove_supplier_company
ON (company_funnel_events.company_id = remove_supplier_company.id)
UNION
SELECT
'exclude_migrated_company' AS company_migration_segment,
company_funnel_events.event_datetime,
company_funnel_events.company_id,
remove_supplier_company.is_migrated_company,
count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS register_company,
count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company,
count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company,
count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company,
count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event,
count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company
FROM company_funnel_events
INNER JOIN remove_supplier_company
ON (company_funnel_events.company_id = remove_supplier_company.id)
WHERE remove_supplier_company.is_migrated_company = '0'
)
SELECT count(*) FROM sem_exchange__company_funnel_events
analyses/shipping/y2022_companies_ltv.sql
-84
View File
@@ -1,84 +0,0 @@
WITH container_packing_lists AS (
SELECT * FROM {{ ref('base_shipping__container_packing_lists') }}
),
containers AS (
SELECT * FROM {{ ref('base_shipping__containers') }}
),
packing_lists AS (
SELECT * FROM {{ ref('base_shipping__packing_lists') }}
),
packages AS (
SELECT * FROM {{ ref('base_shipping__packages') }}
),
company_modules AS (
SELECT * FROM {{ ref('base_shipping__company_modules') }}
),
company_connections AS (
SELECT * FROM {{ ref('base_shipping__company_connections') }}
),
orders AS (
SELECT * FROM {{ ref('base_shipping__orders') }}
)
-- LOGIC
company_module_join_companies AS (
SELECT
company_modules.id AS company_id,
company_modules.name AS company_name,
company_connections.invitee_reference AS marking
FROM
company_modules
LEFT JOIN company_connections
ON (company_modules.id = company_connections.invitee_id)
),
final AS (
SELECT
container_packing_lists.*,
containers.*,
packing_lists.*,
packages.*,
packages.created_at AS packages_created_at,
((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm
FROM container_packing_lists
JOIN containers
ON (container_packing_lists.container_id = containers.id)
JOIN packing_lists
ON (container_packing_lists.packing_list_id = packing_lists.id)
JOIN packages
ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null)
WHERE packing_lists.deleted_at is null AND packages_created_at >= '2022-01-01')
-- ),
-- final_group_by_company_module AS (
-- SELECT
-- final.owner_id,
-- SUM(final.cbm) AS life_time_cbm
-- FROM final
-- GROUP BY
-- owner_id
-- )
SELECT * FROM final
-- SELECT
-- containers.*, packing_lists.id as packing_list_id, SUM((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm
-- FROM
-- container_packing_lists as packed_containers
-- JOIN containers ON packed_containers.container_id = containers.id
-- JOIN packing_lists ON packed_containers.packing_list_id = packing_lists.id
-- JOIN packages ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null)
-- WHERE packing_lists.deleted_at is null GROUP BY packing_lists.id;
dbt_project.yml
+42 -76
View File
@@ -32,91 +32,56 @@ clean-targets: # directories to be removed by `dbt clean`
# using the `{{ config(...) }}` macro.
models:
cief_dbt_cloud:
# Applies to all files under models/example/
exchange:
marts:
reporting:
description: data modeling use in Preset (Superset)
staging:
+tags:
- staging
autocount:
+materialized: view
+schema: exchange__reporting
warehouse:
description: data modeling contain all fct & dim
+materialized: table
+schema: exchange__warehouse
intermediate:
description: data modeling use for pipeline transformation, contain all int
+schema: autocount__staging
crisp:
+materialized: view
+schema: exchange__intermediate
+schema: crisp__staging
staging:
description: light_transformation & will be occur in staging, contain all stg
+materialized: view
+schema: exchange__staging
base:
description: light_transformation such as rename and remove test or wrong input data
+materialized: view
+schema: exchange__base
shipping:
marts:
reporting:
description: data modeling use in Preset (Superset)
exchange:
+materialized: view
+schema: shipping__reporting
warehouse:
description: data modeling contain all fct & dim
+schema: exchange__staging
base:
+materialized: view
+schema: exchange__base
shipping:
staging:
+materialized: view
+schema: shipping__warehouse
+schema: shipping__staging
intermediate:
description: data modeling use for pipeline transformation, contain all int
+materialized: view
+schema: shipping__intermediate
base:
+materialized: view
+schema: shipping__base
intermediate:
+materialized: ephemeral
+schema: intermediate
+tags:
- intermediate
staging:
description: light_transformation & will be occur in staging, contain all stg
+materialized: view
+schema: shipping__staging
base:
description: light_transformation such as rename and remove test or wrong input data
+materialized: view
+schema: shipping__base
crisp:
marts:
reporting:
description: data modeling use in Preset (Superset)
+materialized: view
+schema: crisp__reporting
warehouse:
description: data modeling contain all fct & dim
+materialized: view
+schema: crisp__warehouse
intermediate:
description: data modeling use for pipeline transformation, contain all int
+materialized: view
+schema: crisp__intermediate
staging:
description: light_transformation & will be occur in staging, contain all stg
+materialized: view
+schema: crisp__staging
base:
description: light_transformation such as rename and remove test or wrong input data
+materialized: view
+schema: crisp__base
marts:
+tags:
- marts
warehouse:
+materialized: table
+schema: warehouse
+tags:
- warehouse
reporting:
+materialized: table
+schema: reporting
+tags:
- reporting
seeds:
cief_dbt_cloud:
exchange:
@@ -126,6 +91,7 @@ seeds:
crisp:
+schema: crisp__seeds
snapshots:
cief_dbt_cloud:
exchange:
models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql
-68
View File
@@ -1,68 +0,0 @@
--IMPORT
WITH peoples AS (
SELECT * FROM {{ ref('stg_crisp__peoples') }}
),
people_data AS (
SELECT * FROM {{ ref('stg_crisp__people_data') }}
),
--LOGIC
people_get_marking_id AS (
SELECT
peoples.id,
peoples.name,
peoples.phone,
peoples.email,
peoples.website,
peoples.score,
peoples.segment_list,
peoples.role,
peoples.seniority,
peoples.title,
peoples.gender,
peoples.timezone,
peoples.geo_country,
peoples.geo_city,
peoples.geo_region_code,
peoples.address,
peoples.locale_list,
peoples.geo_latitude,
peoples.geo_longitude,
peoples.last_activated_datetime,
peoples.people_created_datetime,
peoples.people_updated_datetime,
peoples.company_name,
peoples.company_legal_name,
peoples.company_description,
peoples.company_phone,
peoples.company_email_list,
peoples.company_website,
peoples.company_domain,
peoples.company_tag_list,
peoples.company_number_of_employees,
peoples.company_market_cap,
peoples.company_country,
peoples.company_city,
peoples.company_region,
peoples.company_latitude,
peoples.company_logitude,
peoples.company_timezone,
people_data.company_name AS marking_company_name,
people_data.whatsapp_number,
people_data.whatsapp_business_number,
people_data.facebook_page_name,
people_data.facebook_page_id,
people_data.exchange_marking_id,
people_data.izyim_marking_id,
people_data.lite_marking_id
FROM peoples
LEFT JOIN people_data
ON (peoples.id = people_data.people_id)
)
--FINAL
select * from people_get_marking_id
models/crisp/marts/warehouse/dim_crisp__companies.sql
-86
View File
@@ -1,86 +0,0 @@
--IMPORT
WITH users_and_companies AS (
SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }}
),
--LOGIC
split_company_data AS (
SELECT
exchange_marking_id,
izyim_marking_id,
lite_marking_id,
marking_company_name,
whatsapp_number,
whatsapp_business_number,
facebook_page_name,
facebook_page_id,
company_name,
company_legal_name,
company_description,
company_phone,
company_email_list,
company_website,
company_domain,
company_tag_list,
company_number_of_employees,
company_market_cap,
company_country,
company_city,
company_region,
company_latitude,
company_logitude,
company_timezone
FROM users_and_companies
),
companies_remove_duplicate AS (
SELECT
DISTINCT *
FROM split_company_data
),
--FINAL
final_dim_crisp__companies AS (
SELECT
exchange_marking_id,
izyim_marking_id,
lite_marking_id,
IFF(COUNT(DISTINCT(marking_company_name)) > 1, LISTAGG(marking_company_name, ','), MAX(marking_company_name) ) AS marking_company_name,
IFF(COUNT(DISTINCT(whatsapp_number)) > 1, LISTAGG(whatsapp_number, ','), MAX(whatsapp_number) ) AS whatsapp_number,
IFF(COUNT(DISTINCT(whatsapp_business_number)) > 1, LISTAGG(whatsapp_business_number, ','), MAX(whatsapp_business_number) ) AS whatsapp_business_number,
IFF(COUNT(DISTINCT(facebook_page_name)) > 1, LISTAGG(facebook_page_name, ','), MAX(facebook_page_name) ) AS facebook_page_name,
IFF(COUNT(DISTINCT(facebook_page_id)) > 1, LISTAGG(facebook_page_id, ','), MAX(facebook_page_id) ) AS facebook_page_id,
IFF(COUNT(DISTINCT(company_name)) > 1, LISTAGG(company_name, ','), MAX(company_name) ) AS company_name,
IFF(COUNT(DISTINCT(company_legal_name)) > 1, LISTAGG(company_legal_name, ','), MAX(company_legal_name) ) AS company_legal_name,
IFF(COUNT(DISTINCT(company_description)) > 1, LISTAGG(company_description, ','), MAX(company_description) ) AS company_description,
IFF(COUNT(DISTINCT(company_phone)) > 1, LISTAGG(company_phone, ','), MAX(company_phone) ) AS company_phone,
IFF(COUNT(DISTINCT(company_email_list)) > 1, LISTAGG(company_email_list, ','), MAX(company_email_list) ) AS company_email_list,
IFF(COUNT(DISTINCT(company_website)) > 1, LISTAGG(company_website, ','), MAX(company_website) ) AS company_website,
IFF(COUNT(DISTINCT(company_domain)) > 1, LISTAGG(company_domain, ','), MAX(company_domain) ) AS company_domain,
IFF(COUNT(DISTINCT(company_tag_list)) > 1, LISTAGG(company_tag_list, ','), MAX(company_tag_list) ) AS company_tag_list,
IFF(COUNT(DISTINCT(company_number_of_employees)) > 1, LISTAGG(company_number_of_employees, ','), MAX(company_number_of_employees) ) AS company_number_of_employees,
IFF(COUNT(DISTINCT(company_market_cap)) > 1, LISTAGG(company_market_cap, ','), MAX(company_market_cap) ) AS company_market_cap,
IFF(COUNT(DISTINCT(company_country)) > 1, LISTAGG(company_country, ','), MAX(company_country) ) AS company_country,
IFF(COUNT(DISTINCT(company_city)) > 1, LISTAGG(company_city, ','), MAX(company_city) ) AS company_city,
IFF(COUNT(DISTINCT(company_region)) > 1, LISTAGG(company_region, ','), MAX(company_region) ) AS company_region,
IFF(COUNT(DISTINCT(company_latitude)) > 1, LISTAGG(company_latitude, ','), MAX(company_latitude) ) AS company_latitude,
IFF(COUNT(DISTINCT(company_logitude)) > 1, LISTAGG(company_logitude, ','), MAX(company_logitude) ) AS company_logitude,
IFF(COUNT(DISTINCT(company_timezone)) > 1, LISTAGG(company_timezone, ','), MAX(company_timezone) ) AS company_timezone
FROM companies_remove_duplicate
WHERE
(IFF(exchange_marking_id IS NULL, 1, 0) +
IFF(izyim_marking_id IS NULL, 1, 0) +
IFF(lite_marking_id IS NULL, 1, 0)) < 3
GROUP BY
exchange_marking_id,
izyim_marking_id,
lite_marking_id
)
-- SELECT EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID, count(*) FROM final_dim_crisp__companies GROUP BY EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID ORDER BY count(*) desc
SELECT * FROM final_dim_crisp__companies
models/crisp/marts/warehouse/dim_crisp__users.sql
-48
View File
@@ -1,48 +0,0 @@
--IMPORT
WITH users_and_companies AS (
SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }}
),
--LOGIC
split_user_data AS (
SELECT
id,
name,
phone,
email,
website,
score,
segment_list,
role,
seniority,
title,
gender,
timezone,
geo_country,
geo_city,
geo_region_code,
address,
locale_list,
geo_latitude,
geo_longitude,
last_activated_datetime,
people_created_datetime,
people_updated_datetime,
exchange_marking_id AS company_exchange_marking_id,
izyim_marking_id AS company_izyim_marking_id,
lite_marking_id AS company_lite_marking_id
FROM users_and_companies
),
--FINAL
final_dim_crisp__users AS (
SELECT
*
FROM split_user_data
)
SELECT * FROM final_dim_crisp__users
-- SELECT company_lite_marking_id, count(distinct(company_exchange_marking_id)) AS t1, count(distinct(company_izyim_marking_id)) AS t2
-- FROM final_dim_crisp__users GROUP BY company_lite_marking_id HAVING t1 >1 or t2 > 1
-- SELECT company_IZYIM_MARKING_ID, company_exchange_marking_id FROM final_dim_crisp__users
-- WHERE company_IZYIM_MARKING_ID IN ('1324FEN','3791YBE', '1291NSC', '2698LXA', '2431HEB', '2634PLT', '8299LIE', '1882KAI', '1889LBH', '913DLT', '1650GTB', '5903CSB', '6631GEB', '1656HBP', '1905JYW', '8265LSB', '7260UUT', '5989HSY') ORDER BY company_IZYIM_MARKING_ID
models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql
-15
View File
@@ -1,15 +0,0 @@
--IMPORT
WITH snap_user_email_subscriptions AS (
SELECT * FROM {{ ref('stg_crisp__people_email_subscriptions') }}
),
--FINAL
final_fct_crisp__user_email_subscriptions AS (
SELECT
*
FROM snap_user_email_subscriptions
)
SELECT * FROM snap_user_email_subscriptions
models/crisp/source/src_crisp__get_message_in_conversations.yml
-26
View File
@@ -1,26 +0,0 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: get_message_in_conversations
description: Raw `get_people_subscription_status` table from Snowflake
columns:
- name: fingerprint
description: Primary key for 'get_message_in_conversations'
tests:
- unique
- not_null
- name: delivered
description: the platform which after the message is output from origin, and delivered to the user/operator. When an operator send a message but the user is not online anymore, we send it via email only after 2min during these 2min, delivered is empty
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
models/crisp/source/src_crisp__get_people_data.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: get_people_data
description: Raw `get_people_data` table from Snowflake
columns:
- name: people_id
description: Primary key for 'get_people_data'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
models/crisp/source/src_crisp__get_people_subscription_status.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: get_people_subscription_status
description: Raw `get_people_subscription_status` table from Snowflake
columns:
- name: people_id
description: Primary key for 'get_people_subscription_status'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
models/crisp/source/src_crisp__list_conversations.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: list_conversations
description: Raw `get_people_subscription_status` table from Snowflake
columns:
- name: SESSION_ID
description: Primary key for 'list_conversations'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
models/crisp/source/src_crisp__list_people_profiles.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: list_people_profiles
description: Raw `list_people_profiles` table from Snowflake
columns:
- name: people_id
description: Primary key for 'list_people_profiles'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
models/crisp/source/src_crisp__list_website_operator.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: list_website_operator
description: Raw `list_website_operator` table from Snowflake
columns:
- name: details__user_id
description: Primary key for 'list_website_operator'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
models/crisp/staging/stg_crisp__conversations.sql
-22
View File
@@ -1,22 +0,0 @@
-- IMPORT
WITH list_conversations AS (
SELECT * FROM {{ source('src_crisp_api', 'list_conversations') }}
),
--LOGIC
--FINAL
final_stg_crisp__conversations AS (
SELECT
session_id AS id,
people_id AS user_id,
state,
status,
IFF(is_blocked = TRUE, 1, 0) AS is_blocked,
assigned__user_id AS operator_id,
convert_timezone('Asia/Kuala_Lumpur', created_at) AS conversation_created_datetime,
convert_timezone('Asia/Kuala_Lumpur', updated_at) AS conversation_updated_datetime
FROM list_conversations
)
SELECT * FROM final_stg_crisp__conversations
models/crisp/staging/stg_crisp__messages.sql
-32
View File
@@ -1,32 +0,0 @@
-- IMPORT
WITH get_message_in_conversations AS (
SELECT * FROM {{ source('src_crisp_api', 'get_message_in_conversations') }}
),
-- LOGIC
-- FINAL
final_stg_crisp__messages AS (
SELECT
fingerprint AS id,
session_id AS conversation_id,
user__user_id AS user_id,
user__nickname AS user_name,
from_user AS message_from_side,
origin AS message_generate_platform,
delivered AS message_deliverd_platform,
read AS message_read_platform,
type AS message_format,
content__namespace AS event_type,
content__text AS event_message,
content AS message,
edited AS is_edited_message,
content__type AS attachment_type,
content__name AS attachment_name,
content__url AS attachment_url,
convert_timezone('Asia/Kuala_Lumpur', timestamp) AS message_created_datetime
FROM get_message_in_conversations ORDER BY session_id, message_created_datetime
)
SELECT * FROM final_stg_crisp__messages
models/crisp/staging/stg_crisp__operators.sql
-20
View File
@@ -1,20 +0,0 @@
-- IMPORT
WITH list_website_operator AS (
SELECT * FROM {{ source('src_crisp_api', 'list_website_operator') }}
),
--LOGIC
--FINAL
final_stg_crisp__operators AS (
SELECT
details__user_id AS id,
details__title AS title,
details__first_name AS first_name,
details__last_name AS last,
details__email AS email
FROM list_website_operator
)
SELECT * FROM final_stg_crisp__operators
models/crisp/staging/stg_crisp__people_data.sql
-23
View File
@@ -1,23 +0,0 @@
-- IMPORT
WITH get_people_data AS (
SELECT * FROM {{ source('src_crisp_api', 'get_people_data') }}
),
--LOGIC
--FINAL
final_stg_crisp__get_people_data AS (
SELECT
people_id,
data__company_name AS company_name,
data__wa_number AS whatsapp_number,
data__whatsapp_business_number AS whatsapp_business_number,
data__fb_page AS facebook_page_name,
data__fb_page_id AS facebook_page_id,
data__exchange_marking AS exchange_marking_id,
data__izyim_marking AS izyim_marking_id,
data__lite_marking AS lite_marking_id
FROM get_people_data
)
SELECT * FROM final_stg_crisp__get_people_data
models/crisp/staging/stg_crisp__people_email_subscriptions.sql
-20
View File
@@ -1,20 +0,0 @@
-- IMPORT
WITH get_people_subscription_status AS (
SELECT * FROM {{ ref('snap_crisp__get_people_subscription_status') }}
),
--LOGIC
--FINAL
final_stg_crisp__people_email_subscriptions AS (
SELECT
dbt_scd_id,
people_id,
IFF(email = TRUE, 1, 0) AS is_subscribed,
DBT_VALID_FROM AS dbt_valid_from_datetime,
DBT_VALID_TO AS dbt_valid_to_datetime
FROM get_people_subscription_status
)
SELECT * FROM final_stg_crisp__people_email_subscriptions
models/exchange/base/addresses/base_exchange__addresses.sql
-48
View File
@@ -1,48 +0,0 @@
-- IMPORTS
WITH addresses AS (
SELECT * FROM {{ source('src_exchange_mysql', 'addresses') }}
),
-- LOGIC
addresses_rename AS (
SELECT
addresses.id,
addresses.company_id,
addresses.country_id,
addresses.state_id,
addresses.district_id,
addresses.postcode,
addresses.street_one AS address_line_one,
addresses.street_two AS address_line_two,
addresses.billing AS is_billing_address,
addresses.deleted_at,
addresses.created_at,
addresses.updated_At,
current_timestamp() as _dbt_ran_at
FROM addresses
),
-- FINAL
final__base_exchange__addresses AS (
SELECT
id,
company_id,
country_id,
state_id,
district_id,
postcode,
address_line_one,
address_line_two,
is_billing_address,
deleted_at,
created_at,
updated_At,
_dbt_ran_at
FROM addresses_rename
)
SELECT * FROM final__base_exchange__addresses
models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql
-34
View File
@@ -1,34 +0,0 @@
-- IMPORTS
WITH announcement_segments AS (
SELECT * FROM {{ source('src_exchange_mysql', 'announcement_segment') }}
),
-- LOGIC
announcement_segments_rename AS (
SELECT
announcement_segments.announcement_id,
announcement_segments.segment_id,
announcement_segments.created_at,
announcement_segments.updated_at,
current_timestamp() AS _dbt_ran_at
FROM announcement_segments
),
-- FINAL
final__base_exchange__announcement_segments AS (
SELECT
announcement_id,
segment_id,
created_at,
updated_at,
_dbt_ran_at
FROM announcement_segments_rename
)
SELECT * FROM final__base_exchange__announcement_segments
models/exchange/base/announcements/base_exchange__announcements.sql
-50
View File
@@ -1,50 +0,0 @@
-- IMPORTS
WITH announcements AS (
SELECT * FROM {{ source('src_exchange_mysql', 'announcements') }}
),
-- LOGIC
announcements_rename AS (
SELECT
id,
title,
description,
starting_on AS started_at,
ending_on AS ended_at,
is_all_day,
duration,
is_recurring,
recurrence_pattern,
deleted_at,
created_at,
updated_at,
current_timestamp() as _dbt_ran_at
FROM announcements
),
-- FINAL
final__base_exchange__announcements AS (
SELECT
id,
title,
description,
started_at,
ended_at,
is_all_day,
duration,
is_recurring,
recurrence_pattern,
deleted_at,
created_at,
updated_at,
current_timestamp() as _dbt_ran_at
FROM announcements_rename
)
SELECT * FROM final__base_exchange__announcements
models/exchange/base/bank_logs/base_exchange__bank_logs.sql
-61
View File
@@ -1,61 +0,0 @@
-- IMPORTS
WITH bank_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'bank_logs') }}
),
-- LOGIC
bank_logs_rename AS (
SELECT
bank_logs.id,
bank_logs.bank_id,
bank_logs.company_id,
bank_logs.reference,
bank_logs.bank_name,
bank_logs.holder_name,
bank_logs.account_no,
bank_logs.bank_branch,
bank_logs.swift AS swift_code,
bank_logs.snap AS cnaps_code,
bank_logs.type AS bank_type,
bank_logs.default AS is_default,
bank_logs.status,
bank_logs.country_id,
bank_logs.deleted_at,
bank_logs.created_at,
bank_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM bank_logs
),
-- FINAL
final__base_exchange__bank_logs AS (
SELECT
id,
bank_id,
company_id,
reference,
bank_name,
holder_name,
account_no,
bank_branch,
swift_code,
cnaps_code,
bank_type,
is_default,
status,
country_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM bank_logs_rename
)
SELECT * FROM final__base_exchange__bank_logs
models/exchange/base/banks/base_exchange__banks.sql
-58
View File
@@ -1,58 +0,0 @@
-- IMPORTS
WITH banks AS (
SELECT * FROM {{ source('src_exchange_mysql', 'banks') }}
),
-- LOGIC
banks_rename AS (
SELECT
banks.id,
banks.company_id,
banks.reference,
banks.bank_name,
banks.holder_name,
banks.account_no,
banks.bank_branch,
banks.swift AS swift_code,
banks.snap AS cnaps_code,
banks.type AS bank_type,
banks.default AS is_default,
banks.status,
banks.country_id,
banks.deleted_at,
banks.created_at,
banks.updated_at,
current_timestamp() AS _dbt_ran_at
FROM banks
),
-- FINAL
final__base_exchange__banks AS (
SELECT
id,
company_id,
reference,
bank_name,
holder_name,
account_no,
bank_branch,
swift_code,
cnaps_code,
bank_type,
is_default,
status,
country_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM banks_rename
)
SELECT * FROM final__base_exchange__banks
models/exchange/base/booking_logs/base_exchange__booking_logs.sql
-55
View File
@@ -1,55 +0,0 @@
-- IMPORTS
WITH booking_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'booking_logs') }}
),
-- LOGIC
booking_logs_rename AS (
SELECT
booking_logs.id,
booking_logs.booking_id,
booking_logs.company_id,
booking_logs.marking AS marking_id,
booking_logs.service_id AS service_type,
booking_logs.bank_id,
booking_logs.fix_amount AS fix_value,
booking_logs.fix_currency_id,
booking_logs.convertible_currency_id AS quote_currency_id,
booking_logs.conversion_currency_id AS base_currency_id,
booking_logs.status,
booking_logs.deleted_at,
booking_logs.created_at,
booking_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM booking_logs
),
-- FINAL
final_base_exchange__booking_logs AS (
SELECT
id,
booking_id,
company_id,
marking_id,
service_type,
bank_id,
fix_value,
fix_currency_id,
quote_currency_id,
base_currency_id,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM booking_logs_rename
)
SELECT * FROM final_base_exchange__booking_logs
models/exchange/base/bookings/base_exchange__bookings.sql
-53
View File
@@ -1,53 +0,0 @@
-- IMPORTS
WITH bookings AS (
SELECT * FROM {{ source('src_exchange_mysql', 'bookings') }}
),
-- LOGIC
bookings_rename AS (
SELECT
bookings.id,
bookings.company_id,
bookings.marking AS marking_id,
bookings.service_id AS service_type,
bookings.bank_id,
bookings.fix_amount AS fix_value,
bookings.fix_currency_id,
bookings.convertible_currency_id AS quote_currency_id,
bookings.conversion_currency_id AS base_currency_id,
bookings.status,
bookings.deleted_at,
bookings.created_at,
bookings.updated_at,
current_timestamp() AS _dbt_ran_at
FROM bookings
),
-- FINAL
final_base_exchange__bookings AS (
SELECT
id,
company_id,
marking_id,
service_type,
bank_id,
fix_value,
fix_currency_id,
quote_currency_id,
base_currency_id,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM bookings_rename
)
SELECT * FROM final_base_exchange__bookings
models/exchange/base/companies/base_exchange__companies.sql
-47
View File
@@ -1,47 +0,0 @@
-- IMPORTS
WITH companies AS (
SELECT * FROM {{ source('src_exchange_mysql', 'companies') }}
),
-- LOGIC
companies_rename AS (
SELECT
companies.id,
companies.name,
companies.reference AS marking_id,
companies.debtor AS account_software_id,
companies.type AS company_type,
companies.business_type,
companies.status,
companies.deleted_at,
companies.created_at,
companies.updated_at,
current_timestamp() AS _dbt_ran_at
FROM companies
),
-- FINAL
final__base_exchange__companies AS (
SELECT
id,
name,
marking_id,
account_software_id,
company_type,
business_type,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM companies_rename
)
SELECT * FROM final__base_exchange__companies
models/exchange/base/contacts/base_exchange__contacts.sql
-48
View File
@@ -1,48 +0,0 @@
-- IMPORTS
WITH contacts AS (
SELECT * FROM {{ source('src_exchange_mysql', 'contacts') }}
),
-- LOGIC
companies_rename AS (
SELECT
contacts.id,
contacts.company_id,
contacts.country_id,
contacts.reference,
contacts.phone,
contacts.email,
contacts.wechat_id,
contacts.deleted_at,
contacts.created_at,
contacts.updated_at,
current_timestamp() AS _dbt_ran_at
FROM contacts
),
-- FINAL
final__base_exchange__contacts AS (
SELECT
id,
company_id,
country_id,
reference,
phone,
email,
wechat_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM companies_rename
)
SELECT * FROM final__base_exchange__contacts
models/exchange/base/countries/base_exchange__countries.sql
-41
View File
@@ -1,41 +0,0 @@
-- IMPORTS
WITH countries AS (
SELECT * FROM {{ source('src_exchange_mysql', 'countries') }}
),
-- LOGIC
countries_rename AS (
SELECT
countries.id,
countries.name,
countries.short_code,
countries.phone_code,
countries.deleted_at,
countries.created_at,
countries.updated_at,
current_timestamp() AS _dbt_ran_at
FROM countries
),
-- FINAL
final__base_exchange__countries AS (
SELECT
id,
name,
short_code,
phone_code,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM countries_rename
)
SELECT * FROM final__base_exchange__countries
models/exchange/base/currency_logs/base_exchange__currency_logs.sql
-39
View File
@@ -1,39 +0,0 @@
-- IMPORTS
WITH currency_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'currency_logs') }}
),
-- LOGIC
currency_logs_rename AS (
SELECT
currency_logs.id,
currency_logs.currency_id,
currency_logs.created_by,
currency_logs.deleted_at,
currency_logs.created_at,
currency_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM currency_logs
),
-- FINAL
final__base_exchange__currency_logs AS (
SELECT
id,
currency_id,
created_by,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM currency_logs_rename
)
SELECT * FROM final__base_exchange__currency_logs
models/exchange/base/currency_rates/base_exchange__currency_rates.sql
-43
View File
@@ -1,43 +0,0 @@
-- IMPORTS
WITH currency_rates AS (
SELECT * FROM {{ source('src_exchange_mysql', 'currency_rates') }}
),
-- LOGIC
currency_rates_rename AS (
SELECT
currency_rates.id,
currency_rates.currency_id AS quote_currency_id,
currency_rates.selling AS base_to_quote_currency_exchange_rate,
currency_rates.payment_method_type AS payment_method,
currency_rates.service_id AS service_type,
currency_rates.deleted_at,
currency_rates.created_at,
currency_rates.updated_at,
current_timestamp() AS _dbt_ran_at
FROM currency_rates
),
-- FINAL
final__base_exchange__currency_rates AS (
SELECT
id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
payment_method,
service_type,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM currency_rates_rename
)
SELECT * FROM final__base_exchange__currency_rates
models/exchange/base/districts/base_exchange__districts.sql
-44
View File
@@ -1,44 +0,0 @@
-- IMPORTS
WITH districts AS (
SELECT * FROM {{ source('src_exchange_mysql', 'districts') }}
),
-- LOGIC
districts_rename AS (
SELECT
districts.id,
districts.country_id,
districts.state_id,
districts.name,
districts.postcode,
districts.status,
districts.deleted_at,
districts.created_at,
districts.updated_at,
current_timestamp() AS _dbt_ran_at
FROM districts
),
-- FINAL
final__base_exchange__districts AS (
SELECT
id,
country_id,
state_id,
name,
postcode,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM districts_rename
)
SELECT * FROM final__base_exchange__districts
models/exchange/base/documents/base_exchange__documents.sql
-53
View File
@@ -1,53 +0,0 @@
-- IMPORTS
WITH documents AS (
SELECT * FROM {{ source('src_exchange_mysql', 'documents') }}
),
-- LOGIC
documents_rename AS (
SELECT
documents.id,
documents.owner_type,
documents.owner_id,
documents.document_type,
documents.reference,
documents.status,
documents.issued_date AS issued_at,
documents.expired_date AS expired_at,
documents.approver AS approved_by,
documents.approval_date AS approved_at,
documents.deleted_at,
documents.created_at,
documents.updated_at,
current_timestamp() AS _dbt_ran_at
FROM documents
),
-- FINAL
final__base_exchange__documents AS (
SELECT
id,
owner_type,
owner_id,
document_type,
reference,
status,
issued_at,
expired_at,
approved_by,
approved_at,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM documents_rename
)
SELECT * FROM final__base_exchange__documents
models/exchange/base/employees/base_exchange__employees.sql
-32
View File
@@ -1,32 +0,0 @@
-- IMPORTS
WITH employees AS (
SELECT * FROM {{ source('src_exchange_mysql', 'employees') }}
),
-- LOGIC
employees_rename AS (
SELECT
employees.company_id,
employees.user_id,
employees.status,
current_timestamp() AS _dbt_ran_at
FROM employees
),
-- FINAL
final__base_exchange__employees AS (
SELECT
company_id,
user_id,
status,
_dbt_ran_at
FROM employees_rename
)
SELECT * FROM final__base_exchange__employees
models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql
-40
View File
@@ -1,40 +0,0 @@
-- IMPORTS
WITH failed_jobs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'failed_jobs') }}
),
-- LOGIC
failed_jobs_rename AS (
SELECT
failed_jobs.id,
failed_jobs.connection,
failed_jobs.queue,
failed_jobs.payload,
failed_jobs.exception,
failed_jobs.failed_at,
current_timestamp() AS _dbt_ran_at
FROM failed_jobs
),
-- FINAL
final__stg_failed_jobs__exchange AS (
SELECT
id,
connection,
queue,
payload,
exception,
failed_at,
_dbt_ran_at
FROM failed_jobs_rename
)
SELECT * FROM final__stg_failed_jobs__exchange
models/exchange/base/files/base_exchange__files.sql
-42
View File
@@ -1,42 +0,0 @@
-- IMPORTS
WITH files AS (
SELECT * FROM {{ source('src_exchange_mysql', 'files') }}
),
-- LOGIC
files_rename AS (
SELECT
files.id,
files.document_id,
files.file,
files.file_type,
files.deleted_at,
files.created_at,
files.updated_at,
current_timestamp() AS _dbt_ran_at
FROM files
),
-- FINAL
final__base_exchange__files AS (
SELECT
id,
document_id,
file,
file_type,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM files_rename
)
SELECT * FROM final__base_exchange__files
models/exchange/base/group_transactions/base_exchange__group_transactions.sql
-34
View File
@@ -1,34 +0,0 @@
-- IMPORTS
WITH group_transactions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'group_transactions') }}
),
-- LOGIC
group_transactions_rename AS (
SELECT
group_transactions.id,
group_transactions.group_id,
group_transactions.transaction_id,
current_timestamp() AS _dbt_ran_at
FROM group_transactions
),
-- FINAL
final__base_exchange__group_transactions AS (
SELECT
id,
group_id,
transaction_id,
_dbt_ran_at
FROM group_transactions_rename
)
SELECT * FROM final__base_exchange__group_transactions
models/exchange/base/groups/base_exchange__groups.sql
-55
View File
@@ -1,55 +0,0 @@
-- IMPORTS
WITH groups AS (
SELECT * FROM {{ source('src_exchange_mysql', 'groups') }}
),
-- LOGIC
groups_rename AS (
SELECT
groups.id,
groups.reference AS marking_id,
groups.issuer AS issued_by,
groups.receiver AS received_by,
groups.amount AS base_amount,
groups.original_amount AS quote_amount,
groups.currency_id AS base_currency_id,
groups.original_currency_id AS quote_currency_id,
groups.currency_rate AS base_to_quote_currency_exchange_rate,
groups.tax AS base_tax,
groups.service_charge AS base_service_charge,
groups.status,
groups.created_at,
groups.updated_at,
current_timestamp() AS _dbt_ran_at
FROM groups
),
-- FINAL
final__base_exchange__groups AS (
SELECT
id,
marking_id,
issued_by,
received_by,
base_amount,
quote_amount,
base_currency_id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
status,
created_at,
updated_at,
_dbt_ran_at
FROM groups_rename
)
SELECT * FROM final__base_exchange__groups
models/exchange/base/jobs/base_exchange__jobs.sql
-41
View File
@@ -1,41 +0,0 @@
-- IMPORTS
WITH jobs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'jobs') }}
),
-- LOGIC
jobs_rename AS (
SELECT
jobs.id,
jobs.queue,
jobs.payload,
jobs.attempts,
TO_TIMESTAMP(jobs.reserved_at) AS reserved_at,
TO_TIMESTAMP(jobs.available_at) AS available_at,
TO_TIMESTAMP(jobs.created_at) AS created_at,
current_timestamp() AS _dbt_ran_at
FROM jobs
),
-- FINAL
final__base_exchange__jobs AS (
SELECT
id,
queue,
payload,
attempts,
reserved_at,
available_at,
created_at,
_dbt_ran_at
FROM jobs_rename
)
SELECT * FROM final__base_exchange__jobs
models/exchange/base/migrations/base_exchange__migrations.sql
-33
View File
@@ -1,33 +0,0 @@
-- IMPORTS
WITH migrations AS (
SELECT * FROM {{ source('src_exchange_mysql', 'migrations') }}
),
-- LOGIC
migrations_rename AS (
SELECT
migrations.id,
migrations.migration,
migrations.batch,
current_timestamp() AS _dbt_ran_at
FROM migrations
),
-- FINAL
final__base_exchange__migrations AS (
SELECT
id,
migration,
batch,
_dbt_ran_at
FROM migrations_rename
)
SELECT * FROM final__base_exchange__migrations
models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql
-34
View File
@@ -1,34 +0,0 @@
-- IMPORTS
WITH model_has_roles AS (
SELECT * FROM {{ source('src_exchange_mysql', 'model_has_roles') }}
),
-- LOGIC
migrations_rename AS (
SELECT
model_has_roles.role_id,
model_has_roles.model_type,
model_has_roles.model_id,
current_timestamp() AS _dbt_ran_at
FROM model_has_roles
),
-- FINAL
final__base_exchange__model_has_roles AS (
SELECT
role_id,
model_type,
model_id,
_dbt_ran_at
FROM migrations_rename
)
SELECT * FROM final__base_exchange__model_has_roles
models/exchange/base/notifications/base_exchange__notifications.sql
-53
View File
@@ -1,53 +0,0 @@
-- IMPORTS
WITH notifications AS (
SELECT * FROM {{ source('src_exchange_mysql', 'notifications') }}
),
-- LOGIC
notifications_rename AS (
SELECT
notifications.id,
notifications.title,
notifications.description,
notifications.subject_type,
notifications.subject_id,
notifications.target_type,
notifications.target_id,
notifications.causer_type,
notifications.causer_id,
notifications.status,
notifications.deleted_at,
notifications.created_at,
notifications.updated_at,
current_timestamp() AS _dbt_ran_at
FROM notifications
),
-- FINAL
final__base_exchange__notifications AS (
SELECT
id,
title,
description,
subject_type,
subject_id,
target_type,
target_id,
causer_type,
causer_id,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM notifications_rename
)
SELECT * FROM final__base_exchange__notifications
models/exchange/base/password_resets/base_exchange__password_resets.sql
-42
View File
@@ -1,42 +0,0 @@
-- IMPORTS
WITH password_resets AS (
SELECT * FROM {{ source('src_exchange_mysql', 'password_resets') }}
),
-- LOGIC
password_resets_rename AS (
SELECT
password_resets.id,
password_resets.user_id,
password_resets.token,
password_resets.is_expired,
password_resets.is_complete,
password_resets.created_at,
password_resets.updated_at,
current_timestamp() AS _dbt_ran_at
FROM password_resets
),
-- FINAL
final__base_exchange__password_resets AS (
SELECT
id,
user_id,
token,
is_expired,
is_complete,
created_at,
updated_at,
_dbt_ran_at
FROM password_resets_rename
)
SELECT * FROM final__base_exchange__password_resets
models/exchange/base/permissions/base_exchange__permissions.sql
-38
View File
@@ -1,38 +0,0 @@
-- IMPORTS
WITH permissions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'permissions') }}
),
-- LOGIC
permissions_rename AS (
SELECT
permissions.id,
permissions.name,
permissions.guard_name,
permissions.created_at,
permissions.updated_at,
current_timestamp() AS _dbt_ran_at
FROM permissions
),
-- FINAL
final__base_exchange__permissions AS (
SELECT
id,
name,
guard_name,
created_at,
updated_at,
_dbt_ran_at
FROM permissions_rename
)
SELECT * FROM final__base_exchange__permissions
models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql
-32
View File
@@ -1,32 +0,0 @@
-- IMPORTS
WITH role_has_permissions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'role_has_permissions') }}
),
-- LOGIC
role_has_permissions_rename AS (
SELECT
role_has_permissions.permission_id,
role_has_permissions.role_id,
current_timestamp() AS _dbt_ran_at
FROM role_has_permissions
),
-- FINAL
final__exchange_base__role_has_permissions AS (
SELECT
permission_id,
role_id,
_dbt_ran_at
FROM role_has_permissions_rename
)
SELECT * FROM final__exchange_base__role_has_permissions
models/exchange/base/roles/base_exchange__roles.sql
-40
View File
@@ -1,40 +0,0 @@
-- IMPORTS
WITH roles AS (
SELECT * FROM {{ source('src_exchange_mysql', 'roles') }}
),
-- LOGIC
roles_rename AS (
SELECT
roles.id,
roles.name,
roles.guard_name,
roles.type,
roles.created_at,
roles.updated_at,
current_timestamp() AS _dbt_ran_at
FROM roles
),
-- FINAL
final__base_exchange__roles AS (
SELECT
id,
name,
guard_name,
type,
created_at,
updated_at,
_dbt_ran_at
FROM roles_rename
)
SELECT * FROM final__base_exchange__roles
models/exchange/base/segment_companies/base_exchange__segment_companies.sql
-43
View File
@@ -1,43 +0,0 @@
-- IMPORTS
WITH segment_companies AS (
SELECT * FROM {{ ref('snap_exchange__segment_companies') }}
),
-- LOGIC
segment_companies_rename AS (
SELECT
segment_companies.concat_id,
segment_companies.segment_id,
segment_companies.company_id,
segment_companies.deleted_at,
segment_companies.created_at,
segment_companies.updated_at,
segment_companies.dbt_valid_from,
segment_companies.dbt_valid_to,
current_timestamp() AS _dbt_ran_at
FROM segment_companies
),
-- FINAL
final__base_exchange__segment_companies AS (
SELECT
concat_id,
segment_id,
company_id,
deleted_at,
created_at,
updated_at,
dbt_valid_from,
dbt_valid_to,
_dbt_ran_at
FROM segment_companies_rename
)
SELECT * FROM final__base_exchange__segment_companies
models/exchange/base/segment_constants/base_exchange__segment_constants.sql
-43
View File
@@ -1,43 +0,0 @@
-- IMPORTS
WITH segment_constants AS (
SELECT * FROM {{ source('src_exchange_mysql', 'segment_constants') }}
),
-- LOGIC
segment_constants_rename AS (
SELECT
segment_constants.id,
segment_constants.segment_id,
segment_constants.name,
segment_constants.reference AS marking_id,
segment_constants.detail,
segment_constants.deleted_at,
segment_constants.created_at,
segment_constants.updated_at,
current_timestamp() AS _dbt_ran_at
FROM segment_constants
),
-- FINAL
final_base_exchange__segment_constants AS (
SELECT
id,
segment_id,
name,
marking_id,
detail,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM segment_constants_rename
)
SELECT * FROM final_base_exchange__segment_constants
models/exchange/base/segments/base_exchange__segments.sql
-40
View File
@@ -1,40 +0,0 @@
-- IMPORTS
WITH segments AS (
SELECT * FROM {{ source('src_exchange_mysql', 'segments') }}
),
-- LOGIC
segments_rename AS (
SELECT
segments.id,
segments.name,
segments.type,
segments.status,
segments.deleted_at,
segments.created_at,
segments.updated_at,
current_timestamp() AS _dbt_ran_at
FROM segments
),
-- FINAL
final__base_exchange__segments AS (
SELECT
id,
name,
type,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM segments_rename
)
SELECT * FROM final__base_exchange__segments
models/exchange/base/service_types/base_exchange__service_types.sql
-39
View File
@@ -1,39 +0,0 @@
-- IMPORTS
WITH service_types AS (
SELECT * FROM {{ source('src_exchange_mysql', 'service_types') }}
),
-- LOGIC
service_types_rename AS (
SELECT
service_types.id,
service_types.name,
service_types.status,
service_types.deleted_at,
service_types.created_at,
service_types.updated_at,
CURRENT_TIMESTAMP() AS _dbt_ran_at
FROM service_types
),
-- FINAL
final__base_exchange__service_types AS (
SELECT
id,
name,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM service_types_rename
)
SELECT * FROM final__base_exchange__service_types
models/exchange/base/states/base_exchange__states.sql
-41
View File
@@ -1,41 +0,0 @@
-- IMPORTS
WITH states AS (
SELECT * FROM {{ source('src_exchange_mysql', 'states') }}
),
-- LOGIC
states_rename AS (
SELECT
states.id,
states.country_id,
states.name,
states.status,
states.deleted_at,
states.created_at,
states.updated_at,
current_timestamp() AS _dbt_ran_at
FROM states
),
-- FINAL
final__base_exchange__states AS (
SELECT
id,
country_id,
name,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM states_rename
)
SELECT * FROM final__base_exchange__states
models/exchange/base/transaction_details/base_exchange__transaction_details.sql
-48
View File
@@ -1,48 +0,0 @@
-- IMPORTS
WITH transaction_details AS (
SELECT * FROM {{ source('src_exchange_mysql', 'transaction_detail') }}
),
-- LOGIC
transaction_details_rename AS (
SELECT
transaction_details.id,
transaction_details.transaction_id,
transaction_details.product_code,
transaction_details.product_name,
transaction_details.quantity,
transaction_details.price,
transaction_details.amount,
transaction_details.deleted_at,
transaction_details.created_at,
transaction_details.updated_at,
current_timestamp() AS _dbt_ran_at
FROM transaction_details
),
-- FINAL
final__base_exchange__transaction_details AS (
SELECT
id,
transaction_id,
product_code,
product_name,
quantity,
price,
amount,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM transaction_details_rename
)
SELECT * FROM final__base_exchange__transaction_details
models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql
-73
View File
@@ -1,73 +0,0 @@
-- IMPORTS
WITH transaction_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'transaction_logs') }}
),
-- LOGIC
transaction_logs_rename AS (
SELECT
transaction_logs.id,
transaction_logs.transaction_id,
transaction_logs.owner_type,
transaction_logs.owner_id,
transaction_logs.type AS transaction_type,
transaction_logs.issuer AS issued_by,
transaction_logs.receiver AS received_by,
transaction_logs.recipient_bank_account_id,
transaction_logs.payment_method,
transaction_logs.payment_reference,
transaction_logs.bill_no,
transaction_logs.amount AS base_value,
transaction_logs.original_amount AS quote_value,
transaction_logs.currency_id AS base_currency_id,
transaction_logs.original_currency_id AS quote_currency_id,
transaction_logs.currency_rate AS base_to_quote_currency_exchange_rate,
transaction_logs.tax AS base_tax,
transaction_logs.service_charge AS base_service_charge,
transaction_logs.expires_on AS expired_at,
transaction_logs.status,
transaction_logs.deleted_at,
transaction_logs.created_at,
transaction_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM transaction_logs
),
-- FINAL
final__base_exchange__transaction_logs AS (
SELECT
id,
transaction_id,
owner_type,
owner_id,
transaction_type,
issued_by,
received_by,
recipient_bank_account_id,
payment_method,
payment_reference,
bill_no,
base_value,
quote_value,
base_currency_id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
expired_at,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM transaction_logs_rename
)
SELECT * FROM final__base_exchange__transaction_logs
models/exchange/base/transactions/base_exchange__transactions.sql
-71
View File
@@ -1,71 +0,0 @@
-- IMPORTS
WITH transactions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'transactions') }}
),
-- LOGIC
transactions_rename AS (
SELECT
transactions.id,
transactions.owner_type,
transactions.owner_id,
transactions.type AS transaction_type,
transactions.issuer AS issued_by,
transactions.receiver AS received_by,
transactions.recipient_bank_account_id,
transactions.payment_method,
transactions.payment_reference,
transactions.bill_no,
transactions.amount AS base_value,
transactions.original_amount AS quote_value,
transactions.currency_id AS base_currency_id,
transactions.original_currency_id AS quote_currency_id,
transactions.currency_rate AS base_to_quote_currency_exchange_rate,
transactions.tax AS base_tax,
transactions.service_charge AS base_service_charge,
transactions.expires_on AS expired_at,
transactions.status,
transactions.deleted_at,
transactions.created_at,
transactions.updated_at,
current_timestamp() AS _dbt_ran_at
FROM transactions
),
-- FINAL
final__base_exchange__transactions AS (
SELECT
id,
owner_type,
owner_id,
transaction_type,
issued_by,
received_by,
recipient_bank_account_id,
payment_method,
payment_reference,
bill_no,
base_value,
quote_value,
base_currency_id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
expired_at,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM transactions_rename
)
SELECT * FROM final__base_exchange__transactions
models/exchange/base/users/base_exchange__users.sql
-47
View File
@@ -1,47 +0,0 @@
-- IMPORTS
WITH users AS (
SELECT * FROM {{ source('src_exchange_mysql', 'users') }}
),
-- LOGIC
users_rename AS (
SELECT
users.id,
users.name,
users.email,
users.type AS user_type,
users.status,
users.remember_token,
users.active_at,
users.deleted_at,
users.created_at,
users.updated_at,
current_timestamp() AS _dbt_ran_at
FROM users
),
-- FINAL
final__base_exchange__users AS (
SELECT
id,
name,
email,
user_type,
status,
remember_token,
active_at,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM users_rename
)
SELECT * FROM final__base_exchange__users
models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql
-47
View File
@@ -1,47 +0,0 @@
-- IMPORTS
WITH wallet_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'wallet_logs') }}
),
-- LOGIC
wallet_logs_rename AS (
SELECT
wallet_logs.id,
wallet_logs.wallet_id,
wallet_logs.owner_type,
wallet_logs.owner_id,
wallet_logs.code,
wallet_logs.currency_id,
wallet_logs.amount,
wallet_logs.deleted_at,
wallet_logs.created_at,
wallet_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM wallet_logs
),
-- FINAL
final__base_exchange__wallet_logs AS (
SELECT
id,
wallet_id,
owner_type,
owner_id,
code,
currency_id,
amount,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM wallet_logs_rename
)
SELECT * FROM final__base_exchange__wallet_logs
models/exchange/base/wallets/base_exchange__wallets.sql
-45
View File
@@ -1,45 +0,0 @@
-- IMPORTS
WITH wallets AS (
SELECT * FROM {{ source('src_exchange_mysql', 'wallets') }}
),
-- LOGIC
wallets_rename AS (
SELECT
wallets.id,
wallets.owner_type,
wallets.owner_id,
wallets.code,
wallets.currency_id,
wallets.amount,
wallets.deleted_at,
wallets.created_at,
wallets.updated_at,
current_timestamp() AS _dbt_ran_at
FROM wallets
),
-- FINAL
final__base_exchange__wallets AS (
SELECT
id,
owner_type,
owner_id,
code,
currency_id,
amount,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM wallets_rename
)
SELECT * FROM final__base_exchange__wallets
models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql
-112
View File
@@ -1,112 +0,0 @@
-- IMPORTS
WITH booking_logs AS (
SELECT * FROM {{ ref('stg_exchange__booking_logs') }}
),
currency_rate_logs AS (
SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }}
),
-- LOGIC
booking_logs_calculate_value_join_currency_rate_logs AS (
SELECT
booking_logs.id,
booking_logs.booking_id,
booking_logs.company_id,
booking_logs.user_id,
booking_logs.marking_id,
booking_logs.service_type,
booking_logs.service_type_name,
booking_logs.bank_id,
booking_logs.fix_value,
booking_logs.fix_currency_id,
booking_logs.fix_currency_name,
booking_logs.fix_currency_short_code,
currency_rate_logs.payment_method AS estimate_payment_method,
currency_rate_logs.payment_method_name AS estimate_payment_method_name,
currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate,
booking_logs.quote_currency_id,
booking_logs.quote_currency_name,
booking_logs.quote_currency_short_code,
IFF(booking_logs.quote_currency_id = booking_logs.fix_currency_id,
fix_value,
ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value,
booking_logs.base_currency_id,
booking_logs.base_currency_name,
booking_logs.base_currency_short_code,
IFF(booking_logs.base_currency_id = booking_logs.fix_currency_id,
fix_value,
ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value,
-- If we have quote & base value, both of them are not contain 1 (MYR), then need to redo this function
CASE
WHEN (booking_logs.base_currency_id = '1')
THEN ROUND(estimate_base_value,2)
WHEN (booking_logs.quote_currency_id = '1')
THEN ROUND(estimate_quote_value,2)
ELSE
NULL
END AS estimate_value_rm,
booking_logs.status,
booking_logs.status_name,
booking_logs.deleted_at,
booking_logs.created_at,
booking_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM booking_logs
LEFT JOIN currency_rate_logs
ON (booking_logs.created_at >= currency_rate_logs.created_date_from)
AND (booking_logs.created_at < currency_rate_logs.created_date_to)
AND (booking_logs.service_type = currency_rate_logs.service_type)
AND (booking_logs.quote_currency_id = currency_rate_logs.quote_currency_id)
AND (currency_rate_logs.payment_method = '1')
),
-- FINAL
final__int_booking_logs_get_estimate_rates__exchange AS (
SELECT
id,
booking_id,
company_id,
user_id,
marking_id,
service_type,
service_type_name,
bank_id,
fix_value,
fix_currency_id,
fix_currency_name,
fix_currency_short_code,
estimate_payment_method,
estimate_payment_method_name,
estimate_base_to_quote_currency_exchange_rate,
base_currency_id,
base_currency_name,
base_currency_short_code,
estimate_base_value,
estimate_value_rm,
quote_currency_id,
quote_currency_name,
quote_currency_short_code,
estimate_quote_value,
status,
status_name,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM booking_logs_calculate_value_join_currency_rate_logs
)
SELECT * FROM final__int_booking_logs_get_estimate_rates__exchange
models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql
-81
View File
@@ -1,81 +0,0 @@
-- Import
WITH companies AS (
SELECT * FROM {{ ref('stg_exchange__companies') }}
),
segments AS (
SELECT * FROM {{ ref('stg_exchange__segments') }}
),
companies_and_segments_relations AS (
SELECT * FROM {{ ref('stg_exchange__companies_and_segments_relations') }}
),
-- LOGIC
get_is_migrated_companies AS (
SELECT
segment_id,
company_id
FROM companies_and_segments_relations
WHERE
segment_id = 5
),
-- Because we do not have timestamp to get maximum segment_id
get_segment_row_created_sequences AS (
SELECT
*,
ROW_NUMBER() OVER (ORDER BY NULL) AS row_created_sequence
FROM companies_and_segments_relations
WHERE
deleted_at IS NULL
AND
segment_id IN (1,2,3,4)
AND
dbt_valid_to IS NULL
),
get_company_latest_segments AS (
SELECT
*,
row_number() OVER
(PARTITION BY company_id
ORDER BY row_created_sequence DESC) AS is_latest_segment
FROM get_segment_row_created_sequences
QUALIFY
is_latest_segment = 1
),
-- FINAL
final__int_exchange__companies_level_segment AS (
SELECT
companies.*,
get_company_latest_segments.segment_id,
segments.name AS segment_name,
IFF(get_is_migrated_companies.company_id IS NOT NULL, 1, 0) AS is_migrated_company
FROM companies
LEFT JOIN get_company_latest_segments
ON (companies.id = get_company_latest_segments.company_id)
LEFT JOIN segments
ON (get_company_latest_segments.segment_id = segments.id)
LEFT JOIN get_is_migrated_companies
ON (companies.id = get_is_migrated_companies.company_id)
)
SELECT * FROM final__int_exchange__companies_level_segment
-- select * from get_is_migrated_company
models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql
-45
View File
@@ -1,45 +0,0 @@
-- IMPORTS
WITH user_email_verifications AS (
SELECT * FROM {{ ref('stg_exchange__user_email_verifications')}}
),
-- LOGIC
dedupe_user_email_verifications AS (
SELECT
user_id,
-- Function below search for '1' in is_complete groupconcat string, if found then true, if null than null, else false
IFF(CONTAINS( LISTAGG(is_completed, ',') WITHIN GROUP(ORDER BY is_completed), '1'),'1','0') AS is_completed,
IFF(CONTAINS( LISTAGG(is_activated, ',') WITHIN GROUP(ORDER BY is_activated), '1'),'1','0') AS is_activated,
IFF(CONTAINS( LISTAGG(is_sent, ',') WITHIN GROUP(ORDER BY is_sent), '1'), '1', '0') AS is_sent,
COUNT(*) AS email_sent,
MIN(created_at) AS first_email_created_at,
MAX(created_at) AS last_email_created_at,
current_timestamp() AS _dbt_ran_at
FROM user_email_verifications
GROUP BY
user_id
),
-- FINAL
final__int_exchange__dedupe_user_email_verifications AS (
SELECT
user_id,
is_completed,
is_activated,
is_sent,
email_sent,
first_email_created_at,
last_email_created_at,
_dbt_ran_at
FROM
dedupe_user_email_verifications
)
SELECT * FROM final__int_exchange__dedupe_user_email_verifications
models/exchange/intermediate/int_exchange__latest_company_contacts.sql
-38
View File
@@ -1,38 +0,0 @@
--IMPORT
WITH contacts AS (
SELECT * FROM {{ ref('stg_exchange__contacts') }}
),
--LOGIC
get_latest_company_contacts AS (
SELECT
*,
row_number() OVER
(PARTITION BY company_id
ORDER BY contact_created_datetime DESC) AS last_row_number
FROM contacts
QUALIFY
last_row_number = 1
),
--FINAL
final__int_exchange__latest_company_contacts AS (
SELECT
id,
company_id,
country_id,
country_name,
reference,
phone,
email,
wechat_id,
contact_deleted_datetime,
contact_created_datetime,
contact_updated_datetime
FROM get_latest_company_contacts
)
SELECT * FROM final__int_exchange__latest_company_contacts
models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql
-72
View File
@@ -1,72 +0,0 @@
-- IMPORT
WITH transaction_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_costs') }}
),
-- LOGIC
duplicate_id_need_remove AS (
SELECT
ID,
TRANSACTION_ORDER_ID,
BASE_VALUE,
QUOTE_VALUE,
BASE_TO_QUOTE_CURRENCY_EXCHANGE_RATE,
STATUS_NAME,
CREATED_AT,
UPDATED_AT,
row_number() OVER
(PARTITION BY transaction_costs.transaction_order_id
ORDER BY transaction_costs.updated_at DESC) AS row_index
FROM
transaction_costs
WHERE
deleted_at IS NULL
QUALIFY
row_index = 2
),
-- duplicate_id_need_remove AS (
-- SELECT
-- id
-- FROM mark_duplicate_is_not_deleted_id
-- WHERE
-- row_index = 2
-- ),
removed_duplicate_id_table AS (
SELECT
transaction_costs.*
FROM transaction_costs
LEFT JOIN duplicate_id_need_remove
ON (transaction_costs.id = duplicate_id_need_remove.id)
WHERE
duplicate_id_need_remove.id IS NULL
),
--FINAL
final__int_exchange__transaction_costs_remove_duplicate_row AS (
SELECT * FROM removed_duplicate_id_table
)
SELECT * FROM final__int_exchange__transaction_costs_remove_duplicate_row
-- SELECT * FROM duplicate_transaction_order_id_which_not_deleted
-- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at
-- SELECT * FROM duplicate_id_need_remove
-- SELECT * FROM removed_duplicate_id_table WHERE deleted_at is not null
-- SELECT * FROM mark_duplicate_is_not_deleted_id WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at
-- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at
models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql
-28
View File
@@ -1,28 +0,0 @@
-- IMPORTS
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_log_orders_remove_undo_status') }}
),
bookings AS (
SELECT * FROM {{ ref('stg_exchange__bookings') }}
),
-- LOGICS
transaction_log_orders_join_bookings AS (
SELECT
transaction_log_orders.*,
bookings.user_id
FROM transaction_log_orders
LEFT JOIN bookings
ON (transaction_log_orders.booking_id = bookings.id)
),
-- FINAL
final__int_exchange__transaction_log_orders_get_user_company_ids AS (
SELECT * FROM transaction_log_orders_join_bookings
)
SELECT * FROM final__int_exchange__transaction_log_orders_get_user_company_ids
models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql
-26
View File
@@ -1,26 +0,0 @@
-- IMPORT
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('stg_exchange__transaction_log_orders') }}
),
-- LOGIC
transaction_log_orders_mark_undo_status AS (
SELECT
*
FROM transaction_log_orders
QUALIFY
row_number() OVER
(PARTITION BY transaction_log_orders.status, transaction_log_orders.transaction_order_id
ORDER BY transaction_log_orders.updated_at DESC) = 1
),
-- FINAL
final__int_exchange__transaction_log_orders_remove_undo_status AS (
SELECT * FROM transaction_log_orders_mark_undo_status
)
SELECT * FROM final__int_exchange__transaction_log_orders_remove_undo_status
models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql
-30
View File
@@ -1,30 +0,0 @@
-- IMPORTS
WITH transaction_orders AS (
SELECT * FROM {{ ref('stg_exchange__transaction_orders') }}
),
bookings AS (
SELECT * FROM {{ ref('stg_exchange__bookings') }}
),
-- LOGICS
transaction_orders_join_bookings AS (
SELECT
transaction_orders.*,
bookings.user_id
FROM transaction_orders
LEFT JOIN bookings
ON (transaction_orders.booking_id = bookings.id)
),
-- FINAL
final__int_exchange__transaction_orders_get_user_company_ids AS (
SELECT * FROM transaction_orders_join_bookings
)
SELECT * FROM final__int_exchange__transaction_orders_get_user_company_ids
models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql
-234
View File
@@ -1,234 +0,0 @@
-- IMPORT
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_log_orders') }}
),
latest_transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
booking_logs AS (
SELECT * FROM {{ ref('fct_exchange__booking_logs') }}
),
latest_bookings AS (
SELECT * FROM {{ ref('fct_exchange__bookings') }}
),
latest_transaction_costs AS (
SELECT * FROM {{ ref('fct_exchange__transaction_costs') }}
),
transaction_log_costs AS (
SELECT * FROM {{ ref('fct_exchange__transaction_log_costs') }}
),
companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
dates AS (
SELECT * FROM {{ ref('dim_exchange__dates') }}
),
--LOGIC
transaction_order_logs_status_datetime AS (
SELECT
transaction_order_id,
MIN(created_at) AS created_datetime,
MAX(CASE WHEN status = '0' THEN updated_at END) AS pending_submission_datetime,
MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime,
MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime,
MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime,
MAX(CASE WHEN status = '4' THEN updated_at END) AS rejected_datetime,
MAX(CASE WHEN status = '5' THEN updated_at END) AS suspended_datetime
FROM transaction_log_orders
GROUP BY
transaction_order_id
),
cost_logs_status_datetime AS (
SELECT
transaction_order_id,
MIN(created_at) AS created_datetime,
MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime,
MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime,
MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime
FROM transaction_log_costs
GROUP BY
transaction_order_id
),
latest_transaction_costs_remove_deleted AS (
SELECT
*
FROM
latest_transaction_costs
WHERE
deleted_at IS NULL
),
previous_order_date_row_rank AS (
SELECT
*,
row_number() OVER
(PARTITION BY latest_transaction_orders.company_id
ORDER BY latest_transaction_orders.created_at) AS row_rank_index
FROM
latest_transaction_orders
),
get_previous_order_date AS (
SELECT
previous_order_date_row_rank.id,
previous_order_date_row_rank_clone.created_at AS previous_order_created_datetime
FROM previous_order_date_row_rank
LEFT JOIN previous_order_date_row_rank AS previous_order_date_row_rank_clone
ON previous_order_date_row_rank.row_rank_index = previous_order_date_row_rank_clone.row_rank_index + 1
AND previous_order_date_row_rank.company_id = previous_order_date_row_rank_clone.company_id
),
--FINAL
final__rep_exchange__daily_transactions AS (
SELECT
latest_transaction_orders.id,
latest_transaction_orders.booking_id,
latest_bookings.marking_id AS booking_marking_id,
latest_bookings.service_type,
latest_bookings.service_type_name,
latest_bookings.booking_created_datetime,
latest_transaction_orders.seller_company_id,
latest_transaction_orders.bank_id,
latest_transaction_orders.user_id,
latest_transaction_orders.company_id,
companies.marking_id AS company_marking_id,
companies.company_type,
companies.company_type_name,
companies.business_type AS company_business_type,
companies.business_type_name AS company_business_type_name,
companies.segment_id AS company_segment_id,
companies.segment_name AS company_segment_name,
companies.is_migrated_company AS company_is_migrated_company,
companies.country_id AS company_country_id,
companies.country_name AS company_country_name,
companies.state_id AS company_state_id,
companies.state_name AS company_state_name,
companies.district_id AS company_district_id,
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.wallet_id AS company_wallet_id,
companies.have_wallet AS company_have_wallet,
-- i stay here
latest_bookings.is_first_time_user AS is_first_time_booking_user,
latest_bookings.is_first_time_company AS is_first_time_booking_company,
latest_transaction_orders.is_first_time_user,
latest_transaction_orders.is_first_time_company,
latest_transaction_orders.status,
latest_transaction_orders.status_name,
latest_transaction_orders.transaction_type,
latest_transaction_orders.transaction_type_name,
latest_transaction_orders.payment_method,
latest_transaction_orders.payment_method_name,
latest_transaction_orders.base_currency_id,
latest_transaction_orders.base_currency_name,
latest_transaction_orders.quote_currency_id,
latest_transaction_orders.quote_currency_name,
latest_transaction_orders.base_to_quote_currency_exchange_rate,
latest_transaction_orders.base_value,
latest_transaction_orders.quote_value,
latest_transaction_orders.transaction_value_rm,
latest_transaction_orders.transaction_base_tax,
latest_transaction_orders.transaction_tax_rm,
latest_transaction_orders.base_service_charge,
latest_transaction_orders.transaction_service_charge_rm,
get_previous_order_date.previous_order_created_datetime AS previous_order_created_datetime,
transaction_order_logs_status_datetime.created_datetime AS order_created_datetime,
order_created_dates.date_actual AS order_created_date,
order_created_dates.first_day_of_week AS order_created_week,
transaction_order_logs_status_datetime.pending_submission_datetime AS order_pending_submission_datetime,
transaction_order_logs_status_datetime.pending_verification_datetime AS order_pending_verification_datetime,
transaction_order_logs_status_datetime.approved_datetime AS order_approved_datetime,
transaction_order_logs_status_datetime.completed_datetime AS order_completed_datetime,
transaction_order_logs_status_datetime.rejected_datetime AS order_rejected_datetime,
transaction_order_logs_status_datetime.suspended_datetime AS order_suspended_datetime,
latest_transaction_costs_remove_deleted.id AS cost_id,
latest_transaction_costs_remove_deleted.supplier_company_id AS cost_supplier_company_id,
latest_transaction_costs_remove_deleted.status AS cost_status,
latest_transaction_costs_remove_deleted.status_name AS cost_status_name,
latest_transaction_costs_remove_deleted.base_currency_id AS cost_base_currency_id,
latest_transaction_costs_remove_deleted.base_currency_name AS cost_base_currency_name,
latest_transaction_costs_remove_deleted.quote_currency_id AS cost_quote_currency_id,
latest_transaction_costs_remove_deleted.quote_currency_name AS cost_quote_currency_name,
latest_transaction_costs_remove_deleted.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate,
latest_transaction_costs_remove_deleted.base_value AS cost_base_value,
latest_transaction_costs_remove_deleted.quote_value AS cost_quote_value,
latest_transaction_costs_remove_deleted.transaction_value_rm AS cost_transaction_value_rm,
latest_transaction_costs_remove_deleted.transaction_base_tax AS cost_transaction_base_tax,
latest_transaction_costs_remove_deleted.transaction_tax_rm AS cost_transaction_tax_rm,
latest_transaction_costs_remove_deleted.base_service_charge AS cost_base_service_charge,
latest_transaction_costs_remove_deleted.transaction_service_charge_rm AS cost_transaction_service_charge_rm,
cost_logs_status_datetime.created_datetime AS cost_created_datetime,
cost_logs_status_datetime.pending_verification_datetime AS cost_pending_verification_datetime,
cost_logs_status_datetime.approved_datetime AS cost_approved_datetime,
cost_logs_status_datetime.completed_datetime AS cost_completed_datetime
FROM latest_transaction_orders
LEFT JOIN latest_bookings
ON (latest_transaction_orders.booking_id = latest_bookings.id)
LEFT JOIN transaction_order_logs_status_datetime
ON (latest_transaction_orders.id = transaction_order_logs_status_datetime.transaction_order_id)
LEFT JOIN latest_transaction_costs_remove_deleted
ON (latest_transaction_orders.id = latest_transaction_costs_remove_deleted.transaction_order_id)
LEFT JOIN cost_logs_status_datetime
ON (latest_transaction_orders.id = cost_logs_status_datetime.transaction_order_id)
LEFT JOIN companies
ON (latest_transaction_orders.company_id = companies.id)
LEFT JOIN get_previous_order_date
ON(latest_transaction_orders.id = get_previous_order_date.id)
LEFT JOIN dates AS order_created_dates
ON (date(transaction_order_logs_status_datetime.created_datetime) = order_created_dates.date_day )
ORDER BY latest_transaction_orders.updated_at ASC
)
SELECT * FROM final__rep_exchange__daily_transactions
-- @YAM: not unique order id
--(2007, 40808, 2013, 40920, 41101, 125169, 18604, 18552, 18550, 35795, 117444, 72332, 40798, 51174, 18553, 2010, 66195, 89413, 1678, 71516, 5673, 78366, 66208, 18546, 1672, 1684, 18579, 66202, 132074, 18554, 43249, 40804, 86259, 4881, 2311, 103084, 18549, 41097, 106321, 17634, 66207, 18582, 1661, 63970) and so on....
-- (OMAIR)
-- SELECT booking_id, count(*) FROM booking_logs WHERE status= 3 GROUP BY booking_id ORDER BY count(*) desc
-- SELECT booking_id, company_id, user_id, fix_value, status_name, updated_at FROM booking_logs WHERE booking_id IN (18901,18618,22960) ORDER BY booking_id, updated_at
-- SELECT * FROM booking_logs WHERE booking_id = 22173 ORDER BY updated_at
models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml
-211
View File
@@ -1,211 +0,0 @@
version: 2
models:
- name: rep_exchange__daily_orders
columns:
- name: id
description: transaction orders id
tests:
- unique
- not_null
- name: booking_id
description: transaction orders id
tests:
- not_null
- relationships:
to: ref('base_exchange__bookings')
field: id
- name: service_type
description: type of service
tests:
- not_null
- name: service_type_name
description: name of type of service
tests:
- not_null
- name: booking_created_datetime
description: date of boking created
tests:
- not_null
- name: seller_company_id
description: seller company id
tests:
- not_null
- name: bank_id
description: bank id
tests:
- not_null
- name: company_id
description: user id
tests:
- not_null
- name: company_marking_id
description: company marking id
tests:
- relationships:
to: ref('base_exchange__companies')
field: marking_id
- name: company_type
description: type of company
tests:
- not_null
- name: company_type_name
description: name type of company
tests:
- accepted_values:
values:
- PERSONAL
- CORPORATE
- name: company_business_type
description: company business type
tests:
- not_null
- name: company_business_type_name
description: company business type name
tests:
- accepted_values:
values:
- IMPORTER
- name: company_segment_id
description: company segment id
tests:
- not_null
- name: company_segment_name
description: company segment name
tests:
- accepted_values:
values:
- PLATINIUM MEMBER
- GOLD MEMBER
- STANDARD SEGMENT
- name: cost_id
description: cost id
tests:
- relationships:
to: ref('base_exchange__transactions')
field: id
- name: base_value
description: test base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: quote_value
description: test quote value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_value_rm
description: transaction value rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_base_tax
description: test transaction base tax combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: base_service_charge
description: test base service charge combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_tax_rm
description: test transaction tax rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_value
description: test cost base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_to_quote_currency_exchange_rate
description: test cost base to quote currency exchange rate combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_quote_value
description: test cost quote value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_value_rm
description: test COST TRANSACTION VALUE RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_tax_rm
description: test COST TRANSACTION TAX RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_service_charge
description: test COST BASE SERVICE CHARGE combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_service_charge_rm
description: test COST TRANSACTION SERVICE CHARGE_RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_service_charge_rm
description: test transaction service charge rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: order_completed_datetime
column_B: order_created_datetime
or_equal: True
tests:
- dbt_utils.expression_is_true:
expression: "cost_transaction_value_rm + cost_transaction_tax_rm + cost_transaction_service_charge_rm < 2*(transaction_value_rm + transaction_base_tax + transaction_tax_rm)"
severity: warn
# FRESHNESS SOURCE
models/exchange/marts/warehouse/dim_exchange__companies.sql
-178
View File
@@ -1,178 +0,0 @@
-- IMPORTS
WITH companies AS (
SELECT * FROM {{ ref('int_exchange__companies_level_segment') }}
),
company_addresses AS (
SELECT * FROM {{ ref('stg_exchange__company_addresses') }}
),
current_wallets AS (
SELECT * FROM {{ ref('stg_exchange__current_wallets') }}
),
companies_and_users_relations AS (
SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }}
),
company_contacts AS (
SELECT * FROM {{ ref('int_exchange__latest_company_contacts') }}
),
documents AS (
SELECT * FROM {{ ref('stg_exchange__documents') }}
),
-- LOGICS
latest_billing_company_addresses AS (
SELECT
*,
-- 1 will be latest updated address, rank by updated_at date
row_number() OVER
(PARTITION BY company_addresses.company_id
ORDER BY company_addresses.updated_at DESC) AS lastest_address_rank
FROM company_addresses
WHERE
is_billing_address = 1
QUALIFY
lastest_address_rank = 1
),
company_current_wallets AS (
SELECT
id,
owner_id AS company_id,
amount,
currency_id,
currency_name,
created_at,
updated_at
FROM current_wallets
WHERE
owner_type = 'App\\Models\\Company'
),
count_company_user AS (
SELECT
company_id,
COUNT(user_id) AS count_user
FROM companies_and_users_relations
GROUP BY company_id
),
latest_company_upload_identity_documents AS (
SELECT
id,
owner_type,
owner_id AS company_id,
document_type AS identity_document_type_name,
reference AS identity_number,
status,
status_name,
approved_at,
created_at,
row_number() OVER
(PARTITION BY company_id
ORDER BY created_at DESC) AS latest_row_number
FROM documents
WHERE
owner_type = 'App\\Models\\Company'
AND
document_type IN ('IDENTITY_CARD','SSM_REGISTRATION')
AND
status = '2'
QUALIFY
latest_row_number = '1'
),
-- FINAL
final__dim_exchange__companies AS (
SELECT
companies.id,
companies.name,
companies.marking_id,
companies.account_software_id,
companies.company_type,
companies.company_type_name,
companies.business_type,
companies.business_type_name,
companies.segment_id,
companies.segment_name,
companies.is_migrated_company,
companies.status,
companies.status_name,
companies.created_at,
company_contacts.reference AS company_contact_reference,
company_contacts.phone AS company_contact_phone,
company_contacts.email AS company_contact_email,
company_contacts.wechat_id AS company_contact_wechat_id,
latest_billing_company_addresses.id AS address_id,
latest_billing_company_addresses.country_id,
latest_billing_company_addresses.country_name,
latest_billing_company_addresses.country_short_code,
latest_billing_company_addresses.state_id,
latest_billing_company_addresses.state_name,
latest_billing_company_addresses.district_id,
latest_billing_company_addresses.district_name,
latest_billing_company_addresses.postcode,
latest_billing_company_addresses.address_line_one,
latest_billing_company_addresses.address_line_two,
latest_billing_company_addresses.is_billing_address,
latest_billing_company_addresses.deleted_at AS address_deleted_at,
latest_billing_company_addresses.created_at AS address_created_at,
latest_billing_company_addresses.updated_at AS address_updated_at,
company_current_wallets.id AS wallet_id,
IFF(company_current_wallets.id IS NOT NULL, 1, 0) AS have_wallet,
company_current_wallets.amount AS wallet_balance,
company_current_wallets.currency_id AS wallet_currency_id,
company_current_wallets.currency_name AS wallet_currency_name,
company_current_wallets.created_at AS wallet_created_at,
company_current_wallets.updated_at AS wallet_updated_at,
latest_company_upload_identity_documents.identity_document_type_name,
latest_company_upload_identity_documents.identity_number,
latest_company_upload_identity_documents.status AS identity_status,
latest_company_upload_identity_documents.status_name AS identity_status_name,
latest_company_upload_identity_documents.approved_at AS identity_approved_datetime,
latest_company_upload_identity_documents.created_at AS identity_created_datetime,
count_company_user.count_user AS number_of_user
FROM companies
LEFT JOIN latest_billing_company_addresses
ON (companies.id = latest_billing_company_addresses.company_id)
LEFT JOIN company_current_wallets
ON (companies.id = company_current_wallets.company_id)
LEFT JOIN count_company_user
ON (companies.id = count_company_user.company_id)
LEFT JOIN company_contacts
ON (companies.id = company_contacts.company_id)
LEFT JOIN latest_company_upload_identity_documents
ON (companies.id = latest_company_upload_identity_documents.company_id)
)
SELECT * FROM final__dim_exchange__companies
models/exchange/marts/warehouse/dim_exchange__dates.sql
-202
View File
@@ -1,202 +0,0 @@
-- Code and forum link refer: https://gitlab.com/gitlab-data/analytics/-/blob/master/transform/snowflake-dbt/models/sources/date/date_details_source.sql https://stackoverflow.com/questions/71016585/dbt-use-dbt-modeling-to-insert-rows-in-a-table-like-date-dimension-table-in-az
-- Import
-- @NOTE TO GETSON, join this into calculated
WITH holiday_and_non_working_day_lists AS (
SELECT * FROM {{ ref('seed_exchange__holiday_and_non_working_day_lists') }}
),
-- LOGIC
date_spine AS (
{{ dbt_utils.date_spine(
start_date="to_date('01/01/2015', 'mm/dd/yyyy')",
datepart="day",
end_date="dateadd(year, 40, current_date)"
)
}}
),
calculated as (
SELECT
date_day,
date_day AS date_actual,
DAYNAME(date_day) AS day_name,
DATE_PART('month', date_day) AS month_actual,
DATE_PART('year', date_day) AS year_actual,
DATE_PART(quarter, date_day) AS quarter_actual,
CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7
ELSE DATE_PART(dayofweek, date_day) END AS day_of_week,
CASE WHEN day_name = 'Mon' THEN date_day
ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week,
CASE WHEN day_name = 'Mon' THEN WEEK(date_day)
ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column
CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1'
THEN '1'
ELSE week_of_year_temp END AS week_of_year,
DATE_PART('day', date_day) AS day_of_month,
ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter,
ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year,
-- @TODO @GERSON ,done
-- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023)
CASE WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual
when DATE_PART('month', DATEADD( month,-1, date_day) ) =12 THEN year_actual-1
ELSE (year_actual+1) END AS fiscal_year,
-- @TODO @GERSON , done
-- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4)
CASE WHEN month_actual in (2,3,4) THEN '1'
WHEN month_actual in (5,6,7) THEN '2'
WHEN month_actual in (8,9,10) THEN '3'
WHEN month_actual in (11,12) THEN '4'
WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4'
-- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4'
ELSE '4' END AS fiscal_quarter,
-- @TODO @GERSON ,done
ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter,
ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year,
TO_CHAR(date_day, 'MMMM') AS month_name,
TRUNC(date_day, 'Month') AS first_day_of_month,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month,
FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year,
FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter,
-- @TODO @GERSON ,done
-- Change this 2 too
FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter,
LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter,
-- @TODO @GERSON,done
FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year,
LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year,
-- @TODO @GERSON ,done
DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year,
-- @TODO @GERSON ,done
CASE WHEN EXTRACT('month', date_day) = 1 THEN 12
ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year,
LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week,
(year_actual || '-Q' || fiscal_quarter) AS quarter_name,
-- @TODO @GERSON,done
(fiscal_year || '-' || DECODE(fiscal_quarter,
1, 'Q1',
2, 'Q2',
3, 'Q3',
4, 'Q4')) AS fiscal_quarter_name,
('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy,
DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute,
--here check
fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name,
('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy,
-- @TODO @GERSON
-- Use the Seed, we will not doing the hard code in SQL, we use CSV to refer the holiday and desc
h.holiday_desc,
h.is_holiday,
h.holiday_race,
h.holiday_country,
h.holiday_state,
h.is_non_working_day,
-- @TODO @GERSON
DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter,
IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter,
DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year,
IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year,
DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa,
DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings
FROM date_spine s
LEFT JOIN holiday_and_non_working_day_lists h
ON s.date_day = to_date(h.date, 'dd/mm/yyyy')
QUALIFY date_actual >= '2016-01-01'
ORDER BY date_day
),
final AS (
SELECT
date_day,
date_actual,
day_name,
month_actual,
year_actual,
quarter_actual,
day_of_week,
first_day_of_week,
week_of_year,
day_of_month,
day_of_quarter,
day_of_year,
fiscal_year,
fiscal_quarter,
day_of_fiscal_quarter,
day_of_fiscal_year,
month_name,
first_day_of_month,
last_day_of_month,
first_day_of_year,
last_day_of_year,
first_day_of_quarter,
last_day_of_quarter,
first_day_of_fiscal_quarter,
last_day_of_fiscal_quarter,
first_day_of_fiscal_year,
last_day_of_fiscal_year,
week_of_fiscal_year,
month_of_fiscal_year,
last_day_of_week,
quarter_name,
fiscal_quarter_name,
fiscal_quarter_name_fy,
fiscal_quarter_number_absolute,
fiscal_month_name,
fiscal_month_name_fy,
holiday_desc,
is_holiday,
holiday_race,
holiday_country,
holiday_state,
is_non_working_day,
last_month_of_fiscal_quarter,
is_first_day_of_last_month_of_fiscal_quarter,
last_month_of_fiscal_year,
is_first_day_of_last_month_of_fiscal_year,
snapshot_date_fpa,
snapshot_date_billings,
current_timestamp() as _dbt_run_at
FROM calculated
)
SELECT * FROM final
models/exchange/marts/warehouse/dim_exchange__users.sql
-45
View File
@@ -1,45 +0,0 @@
-- IMPORTS
WITH users AS (
SELECT * FROM {{ ref('stg_exchange__users') }}
),
dedupe_user_email_verifications AS (
SELECT * FROM {{ ref('int_exchange__dedupe_user_email_verifications') }}
),
-- LOGICS
users_join_dedupe_user_email_verifications AS (
SELECT
users.id,
users.name,
users.email,
users.user_type,
users.user_type_name,
users.status,
users.status_name,
users.company_id,
users.created_at,
users.deleted_at,
users.updated_at,
dedupe_user_email_verifications.is_sent AS email_verification_is_sent,
dedupe_user_email_verifications.is_completed AS email_verification_is_completed,
dedupe_user_email_verifications.is_activated AS email_verification_is_activated,
dedupe_user_email_verifications.email_sent AS email_verification_email_sent,
first_email_created_at AS email_verification_first_email_created_at,
last_email_created_at AS email_verification_last_email_created_at
FROM users
JOIN dedupe_user_email_verifications
ON (users.id = dedupe_user_email_verifications.user_id)
),
-- FINAL
final__dim_exchange__users AS (
SELECT * FROM users_join_dedupe_user_email_verifications
)
SELECT *
FROM final__dim_exchange__users
models/exchange/marts/warehouse/fct_exchange__booking_logs.sql
-81
View File
@@ -1,81 +0,0 @@
-- IMPORTS
WITH booking_logs AS (
SELECT * FROM {{ ref('int_exchange__booking_logs_get_estimate_rates') }}
),
-- LOGICS
first_time_user_booking_id AS (
SELECT
booking_id,
user_id,
created_at,
row_number() OVER
(PARTITION BY booking_logs.user_id
ORDER BY booking_logs.created_at) AS cummulative_count_booking
FROM
booking_logs
GROUP BY
booking_id,
user_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
first_time_company_booking_id AS (
SELECT
booking_id,
company_id,
created_at,
row_number() OVER
(PARTITION BY booking_logs.company_id
ORDER BY booking_logs.created_at) AS cummulative_count_booking
FROM
booking_logs
GROUP BY
booking_id,
company_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
booking_logs_lists AS (
SELECT
booking_logs.*,
IFF(first_time_user_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_user,
IFF(first_time_company_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_company
FROM
booking_logs
LEFT JOIN first_time_user_booking_id
ON (booking_logs.booking_id = first_time_user_booking_id.booking_id)
LEFT JOIN first_time_company_booking_id
ON (booking_logs.booking_id = first_time_company_booking_id.booking_id)
ORDER BY
created_at
),
-- FINAL
final_fct_exchange__booking_logs AS (
SELECT
*
FROM booking_logs_lists
)
SELECT * FROM final_fct_exchange__booking_logs
models/exchange/marts/warehouse/fct_exchange__bookings.sql
-60
View File
@@ -1,60 +0,0 @@
-- IMPORTS
WITH bookings AS (
SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_rates') }}
),
-- LOGICS
bookings_lists AS (
SELECT
bookings.*,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY bookings.company_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1
ELSE 0
END AS is_first_time_company,
CASE
WHEN bookings.status = '3'
AND
MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL))
OVER (PARTITION BY bookings.company_id)
= bookings.booking_created_datetime
THEN 1
ELSE 0
END AS is_first_time_company_completed,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY bookings.user_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1
ELSE 0
END AS is_first_time_user,
CASE
WHEN bookings.status = '3'
AND
MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL))
OVER (PARTITION BY bookings.user_id)
= bookings.booking_created_datetime
THEN 1
ELSE 0
END AS is_first_time_user_completed
FROM
bookings
ORDER BY
booking_created_datetime
),
-- FINAL
final_fct_exchange__bookings AS (
SELECT
*
FROM bookings_lists
)
SELECT * FROM final_fct_exchange__bookings
models/exchange/marts/warehouse/fct_exchange__email_verifications.sql
-48
View File
@@ -1,48 +0,0 @@
--IMPORT
WITH user_email_verifications AS (
SELECT * FROM {{ ref('stg_exchange__user_email_verifications') }}
),
companies_and_users_relations AS (
SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }}
),
--LOGIC
user_email_verifications_get_company_id AS (
SELECT
user_email_verifications.id,
user_email_verifications.user_id,
companies_and_users_relations.company_id,
user_email_verifications.email,
user_email_verifications.is_sent,
user_email_verifications.is_completed,
user_email_verifications.is_activated,
user_email_verifications.created_at,
user_email_verifications.updated_at
FROM user_email_verifications
LEFT JOIN companies_and_users_relations
ON (user_email_verifications.user_id = companies_and_users_relations.user_id)
),
--FINAL
final__fct_exchange__company_email_verifications AS (
SELECT
id,
user_id,
company_id,
email,
is_sent,
is_completed,
is_activated,
created_at AS verification_email_created_datetime,
updated_at AS verification_email_updated_datetime
FROM user_email_verifications_get_company_id
)
SELECT * FROM final__fct_exchange__company_email_verifications
models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql
-60
View File
@@ -1,60 +0,0 @@
-- IMPORTS
WITH transaction_costs AS (
SELECT * FROM {{ ref('int_exchange__transaction_costs_remove_duplicate_row') }}
),
-- LOGICS
transaction_costs_lists AS (
SELECT
id,
transaction_order_id,
supplier_company_id,
bank_id,
buyer_company_id,
status,
status_name,
transaction_type,
transaction_type_name,
payment_method,
payment_method_name,
base_currency_id,
base_currency_name,
quote_currency_id,
quote_currency_name,
base_to_quote_currency_exchange_rate,
base_value,
quote_value,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_value,
99999999) AS transaction_value_rm,
base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_tax,
99999999) AS transaction_tax_rm,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_service_charge,
99999999) AS transaction_service_charge_rm,
deleted_at,
created_at,
updated_at
FROM transaction_costs
),
-- FINAL
final__fct_exchange__transaction_costs AS (
SELECT * FROM transaction_costs_lists
)
SELECT * FROM final__fct_exchange__transaction_costs
models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql
-61
View File
@@ -1,61 +0,0 @@
-- IMPORTS
WITH transaction_log_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_log_costs') }}
),
-- LOGICS
transaction_log_costs_lists AS (
SELECT
id,
transaction_cost_id,
transaction_order_id,
supplier_company_id,
bank_id,
buyer_company_id,
status,
status_name,
transaction_type,
transaction_type_name,
payment_method,
payment_method_name,
base_currency_id,
base_currency_name,
quote_currency_id,
quote_currency_name,
base_to_quote_currency_exchange_rate,
base_value,
quote_value,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_value,
99999999) AS transaction_value_rm,
base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_tax,
99999999) AS transaction_tax_rm,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_service_charge,
99999999) AS transaction_service_charge_rm,
deleted_at,
created_at,
updated_at
FROM transaction_log_costs
),
-- FINAL
final__fct_exchange__transaction_log_costs AS (
SELECT * FROM transaction_log_costs_lists
)
SELECT * FROM final__fct_exchange__transaction_log_costs
models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql
-116
View File
@@ -1,116 +0,0 @@
-- IMPORTS
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_log_orders_get_user_ids') }}
),
-- LOGICS
first_time_user_transaction_order_id AS (
SELECT
transaction_order_id,
user_id,
created_at,
row_number() OVER
(PARTITION BY transaction_log_orders.user_id
ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking
FROM
transaction_log_orders
GROUP BY
transaction_order_id,
user_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
first_time_company_transaction_order_id AS (
SELECT
transaction_order_id,
company_id,
created_at,
row_number() OVER
(PARTITION BY transaction_log_orders.company_id
ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking
FROM
transaction_log_orders
GROUP BY
transaction_order_id,
company_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
transaction_log_orders_lists AS (
SELECT
transaction_log_orders.id,
transaction_log_orders.transaction_order_id,
transaction_log_orders.booking_id,
transaction_log_orders.seller_company_id,
transaction_log_orders.bank_id,
transaction_log_orders.company_id,
transaction_log_orders.user_id,
transaction_log_orders.status,
transaction_log_orders.status_name,
transaction_log_orders.transaction_type,
transaction_log_orders.transaction_type_name,
transaction_log_orders.payment_method,
transaction_log_orders.payment_method_name,
transaction_log_orders.base_currency_id,
transaction_log_orders.base_currency_name,
transaction_log_orders.quote_currency_id,
transaction_log_orders.quote_currency_name,
transaction_log_orders.base_to_quote_currency_exchange_rate,
transaction_log_orders.base_value,
transaction_log_orders.quote_value,
-- use for future if base currency id other from '1'
IFF(transaction_log_orders.base_currency_id = '1',
transaction_log_orders.base_value,
99999999) AS transaction_value_rm,
transaction_log_orders.base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(transaction_log_orders.base_currency_id = '1',
transaction_log_orders.base_tax,
99999999) AS transaction_tax_rm,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(transaction_log_orders.base_currency_id = '1',
transaction_log_orders.base_service_charge,
99999999) AS transaction_service_charge_rm,
transaction_log_orders.deleted_at,
transaction_log_orders.created_at,
transaction_log_orders.updated_at,
IFF(first_time_user_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_user,
IFF(first_time_company_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_company
FROM transaction_log_orders
LEFT JOIN first_time_user_transaction_order_id
ON (transaction_log_orders.transaction_order_id = first_time_user_transaction_order_id.transaction_order_id)
LEFT JOIN first_time_company_transaction_order_id
ON (transaction_log_orders.transaction_order_id = first_time_company_transaction_order_id.transaction_order_id)
),
-- FINAL
final__fct_exchange__transaction_log_orders AS (
SELECT * FROM transaction_log_orders_lists
)
SELECT * FROM final__fct_exchange__transaction_log_orders
models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql
-92
View File
@@ -1,92 +0,0 @@
-- IMPORTS
WITH transaction_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }}
),
-- LOGICS
transaction_orders_lists AS (
SELECT
transaction_orders.id,
transaction_orders.booking_id,
transaction_orders.seller_company_id,
transaction_orders.bank_id,
transaction_orders.user_id,
transaction_orders.company_id,
transaction_orders.status,
transaction_orders.status_name,
transaction_orders.transaction_type,
transaction_orders.transaction_type_name,
transaction_orders.payment_method,
transaction_orders.payment_method_name,
transaction_orders.base_currency_id,
transaction_orders.base_currency_name,
transaction_orders.quote_currency_id,
transaction_orders.quote_currency_name,
transaction_orders.base_to_quote_currency_exchange_rate,
transaction_orders.base_value,
transaction_orders.quote_value,
-- use for future if base currency id other from '1'
IFF(transaction_orders.base_currency_id = '1',
transaction_orders.base_value,
99999999) AS transaction_value_rm,
transaction_orders.base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(transaction_orders.base_currency_id = '1',
transaction_orders.base_tax,
99999999) AS transaction_tax_rm,
transaction_orders.base_service_charge,
-- use for future if base currency id other from '1'
IFF(transaction_orders.base_currency_id = '1',
transaction_orders.base_service_charge,
99999999) AS transaction_service_charge_rm,
transaction_orders.deleted_at,
transaction_orders.created_at,
transaction_orders.updated_at,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.company_id ORDER BY transaction_orders.created_at) = 1 THEN 1
ELSE 0
END AS is_first_time_company,
CASE
WHEN transaction_orders.status = '3'
AND
MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL))
OVER (PARTITION BY transaction_orders.company_id)
= transaction_orders.created_at
THEN 1
ELSE 0
END AS is_first_time_company_completed,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.user_id ORDER BY transaction_orders.created_at) = 1 THEN 1
ELSE 0
END AS is_first_time_user,
CASE
WHEN transaction_orders.status = '3'
AND
MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL))
OVER (PARTITION BY transaction_orders.user_id)
= transaction_orders.created_at
THEN 1
ELSE 0
END AS is_first_time_user_completed
FROM transaction_orders
),
-- FINAL
final__fct_exchange__transaction_orders AS (
SELECT * FROM transaction_orders_lists
)
SELECT * FROM final__fct_exchange__transaction_orders
models/exchange/marts/warehouse/test_exchange__transaction_orders.yml
View File
models/exchange/marts/warehouse/test_exchange__users.yml
View File
models/exchange/source/src_activity_log__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: activity_log
description: Raw `activity_log` table from Snowflake
columns:
- name: id
description: Primary key for 'activity_log'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_addresses__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: addresses
description: Raw `addresses` table from Snowflake
columns:
- name: id
description: Primary key for 'addresses'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_announcement_segment__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: announcement_segment
description: Raw `announcement_segment` table from Snowflake
columns:
- name: announcement_id
description: Primary key for 'announcement_segment'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_announcements__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: announcements
description: Raw `announcements` table from Snowflake
columns:
- name: id
description: Primary key for 'announcements'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_bank_logs__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: bank_logs
description: Raw `bank_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'bank_logs'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_banks__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: banks
description: Raw `banks` table from Snowflake
columns:
- name: id
description: Primary key for 'banks'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_booking_logs__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: booking_logs
description: Raw `bookings` table from Snowflake
columns:
- name: id
description: Primary key for 'booking_logs'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_bookings__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: bookings
description: Raw `bookings` table from Snowflake
columns:
- name: id
description: Primary key for 'bookings'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_companies__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: companies
description: Raw `companies` table from Snowflake
columns:
- name: id
description: Primary key for 'companies'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_contacts__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: contacts
description: Raw `contacts` table from Snowflake
columns:
- name: id
description: Primary key for 'contacts'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_countries__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: countries
description: Raw `countries` table from Snowflake
columns:
- name: id
description: Primary key for 'countries'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_currencies__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: currencies
description: Raw `currencies` table from Snowflake
columns:
- name: id
description: Primary key for 'currencies'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_currency_logs__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: currency_logs
description: Raw `currency_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'currency_logs'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_currency_rate_logs.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: currency_rate_logs
description: Raw `currency_rate_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'currency_rate_logs'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_currency_rates__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: currency_rates
description: Raw `currency_rates` table from Snowflake
columns:
- name: id
description: Primary key for 'currency_rates'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_districts__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: districts
description: Raw `districts` table from Snowflake
columns:
- name: id
description: Primary key for 'districts'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_documents__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: documents
description: Raw `documents` table from Snowflake
columns:
- name: id
description: Primary key for 'documents'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_employees__exchange.yml
-22
View File
@@ -1,22 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: employees
description: Raw `employees` table from Snowflake
columns:
- name: company_id
description: Primary key for 'employees'
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_failed_jobs__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: failed_jobs
description: Raw `failed_jobs` table from Snowflake
columns:
- name: id
description: Primary key for 'failed_jobs'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_files__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: files
description: Raw `files` table from Snowflake
columns:
- name: id
description: Primary key for 'files'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
models/exchange/source/src_group_transactions__exchange.yml
-24
View File
@@ -1,24 +0,0 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: group_transactions
description: Raw `group_transactions` table from Snowflake
columns:
- name: id
description: Primary key for 'group_transactions'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}

Some files were not shown because too many files have changed in this diff Show More