init from dbt cloud hosting repo

This commit is contained in:
Yam ZhengLim
2023-03-28 23:57:11 +08:00
parent 06a10a7ac7
commit b8d9903e7f
253 changed files with 12008 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
target/
dbt_packages/
logs/
+15
View File
@@ -0,0 +1,15 @@
Welcome to your new dbt project!
### Using the starter project
Try running the following commands:
- dbt run
- dbt test
### Resources:
- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
- Join the [dbt community](http://community.getbdt.com/) to learn from other analytics engineers
- Find [dbt events](https://events.getdbt.com) near you
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
View File
+3
View File
@@ -0,0 +1,3 @@
SELECT distinct QUOTE_CURRENCY_ID ,QUOTE_CURRENCY_NAME
FROM {{ ref('fct_exchange__bookings') }}
+341
View File
@@ -0,0 +1,341 @@
-- MARCRO for count the cummulative_activity_number
{% set funnel_activity = ['register','email_verified','identity_documents_uploaded','identity_documents_verified','first_time_booking','first_time_order','first_time_order_completed','repeat_completed_order_after_30days','repeat_completed_order_after_60days']%}
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
company_identity_documents AS (
SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }}
),
email_verifications AS (
SELECT * FROM {{ ref('fct_exchange__email_verifications') }}
),
bookings AS (
SELECT * FROM {{ ref('fct_exchange__bookings') }}
),
-- LOGIC
remove_supplier_company AS (
SELECT
*
FROM companies
WHERE business_type NOT IN ('1','3')
),
clean_company_transaction_orders AS (
SELECT
transaction_orders.*
FROM transaction_orders
-- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders
INNER JOIN remove_supplier_company
ON (transaction_orders.company_id = remove_supplier_company.id)
),
clean_company_bookings AS (
SELECT
bookings.*
FROM bookings
-- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders
INNER JOIN remove_supplier_company
ON (bookings.company_id = remove_supplier_company.id)
),
company_first_time_order_date AS (
SELECT
id,
company_id,
created_at
FROM clean_company_transaction_orders
WHERE
is_first_time_company_completed = '1'
),
--start defind funnel
repeat_completed_order_after_60days_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'repeat_completed_order_after_60days' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
LEFT JOIN company_first_time_order_date
ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id)
WHERE
DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 60
GROUP BY
clean_company_transaction_orders.company_id
),
repeat_completed_order_after_30days_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'repeat_completed_order_after_30days' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
LEFT JOIN company_first_time_order_date
ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id)
WHERE
DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 30
GROUP BY
clean_company_transaction_orders.company_id
),
first_time_order_completed_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'first_time_order_completed' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
WHERE
clean_company_transaction_orders.is_first_time_company_completed = 1
GROUP BY
clean_company_transaction_orders.company_id
),
first_time_order_activities AS (
SELECT
clean_company_transaction_orders.company_id,
0 AS is_data_cleansing_generate_row,
'first_time_order' AS activity,
MIN(clean_company_transaction_orders.created_at) AS activity_datetime
FROM clean_company_transaction_orders
WHERE
clean_company_transaction_orders.is_first_time_company = 1
GROUP BY
clean_company_transaction_orders.company_id
),
first_time_booking_activities AS (
SELECT
IFNULL( unclean_first_time_booking_activities.company_id , first_time_order_activities.company_id ) AS company_id,
IFF( unclean_first_time_booking_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'first_time_booking' AS activity,
IFNULL( unclean_first_time_booking_activities.activity_datetime , first_time_order_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
clean_company_bookings.company_id,
'first_time_booking' AS activity,
MIN(clean_company_bookings.booking_created_datetime) AS activity_datetime
FROM clean_company_bookings
WHERE
clean_company_bookings.is_first_time_company = 1
GROUP BY
clean_company_bookings.company_id
) AS unclean_first_time_booking_activities
FULL OUTER JOIN first_time_order_activities
ON (unclean_first_time_booking_activities.company_id = first_time_order_activities.company_id)
),
identity_documents_verified_activities AS (
SELECT
IFNULL( unclean_identity_documents_verified_activities.company_id , first_time_booking_activities.company_id ) AS company_id,
IFF( unclean_identity_documents_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'identity_documents_verified' AS activity,
IFNULL( unclean_identity_documents_verified_activities.activity_datetime , first_time_booking_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
company_id,
'identity_documents_verified' AS activity,
MIN(identity_created_datetime) AS activity_datetime
FROM company_identity_documents
WHERE
identity_status = '2'
GROUP BY
company_id
) AS unclean_identity_documents_verified_activities
FULL OUTER JOIN first_time_booking_activities
ON (unclean_identity_documents_verified_activities.company_id = first_time_booking_activities.company_id)
),
identity_documents_uploaded_activities AS (
SELECT
IFNULL( unclean_identity_documents_uploaded_activities.company_id , identity_documents_verified_activities.company_id ) AS company_id,
IFF( unclean_identity_documents_uploaded_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'identity_documents_uploaded' AS activity,
IFNULL( unclean_identity_documents_uploaded_activities.activity_datetime , identity_documents_verified_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
company_id,
'identity_documents_uploaded' AS activity,
MIN(identity_created_datetime) AS activity_datetime
FROM company_identity_documents
GROUP BY
company_id
) AS unclean_identity_documents_uploaded_activities
FULL OUTER JOIN identity_documents_verified_activities
ON (unclean_identity_documents_uploaded_activities.company_id = identity_documents_verified_activities.company_id)
),
email_verified_activities AS (
SELECT
IFNULL( unclean_email_verified_activities.company_id , identity_documents_uploaded_activities.company_id ) AS company_id,
IFF( unclean_email_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'email_verified' AS activity,
IFNULL( unclean_email_verified_activities.activity_datetime , identity_documents_uploaded_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
company_id,
'email_verified' AS activity,
MIN(verification_email_created_datetime) AS activity_datetime
FROM email_verifications
WHERE
is_completed = '1'
GROUP BY
company_id
) AS unclean_email_verified_activities
FULL OUTER JOIN identity_documents_uploaded_activities
ON (unclean_email_verified_activities.company_id = identity_documents_uploaded_activities.company_id)
),
register_activities AS (
SELECT
IFNULL( unclean_register_activities.company_id , email_verified_activities.company_id ) AS company_id,
IFF( unclean_register_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row,
'register' AS activity,
IFNULL( unclean_register_activities.activity_datetime , email_verified_activities.activity_datetime ) AS activity_datetime
FROM
(
SELECT
id AS company_id,
'register' AS activity,
MIN(created_at) AS activity_datetime
FROM remove_supplier_company
GROUP BY
id
) AS unclean_register_activities
FULL OUTER JOIN email_verified_activities
ON (unclean_register_activities.company_id = email_verified_activities.company_id)
),
--all activites Union into a table
union_all_activities AS (
SELECT *
FROM register_activities
{% for activity in funnel_activity[1:] %}
UNION
SELECT *
FROM {{activity}}_activities
{% endfor %}
),
union_all_activities_flag_migrated_company AS (
SELECT
union_all_activities.*,
remove_supplier_company.is_migrated_company
FROM union_all_activities
LEFT JOIN remove_supplier_company
ON (union_all_activities.company_id = remove_supplier_company.id)
),
--FINAL
sem_exchange__company_funnels AS (
SELECT
company_id,
is_data_cleansing_generate_row,
is_migrated_company,
activity_datetime,
{% for activity in funnel_activity %}
count(iff(activity = '{{activity}}', company_id, NULL))
OVER (ORDER BY activity_datetime) AS cumulative_{{activity}}_activity,
{% endfor %}
{% for activity in funnel_activity %}
count(iff(activity = '{{activity}}' AND is_migrated_company = 0, company_id, NULL))
OVER (ORDER BY activity_datetime) AS exclude_migrated_cumulative_{{activity}}_activity,
{% endfor %}
'__macro_end' AS __macro_end
FROM union_all_activities_flag_migrated_company
ORDER BY activity_datetime
)
SELECT * FROM sem_exchange__company_funnels
@@ -0,0 +1,299 @@
-- HOW TO USE IT?
-- MUST: company_migration_segment filter need to be choose
-- IMPORT
WITH companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
company_identity_documents AS (
SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }}
),
email_verifications AS (
SELECT * FROM {{ ref('fct_exchange__email_verifications') }}
),
-- LOGIC
remove_supplier_company AS (
SELECT
*
FROM companies
WHERE business_type = 2
),
company_register_events AS (
SELECT
remove_supplier_company.id AS company_id,
'Register' AS register_event,
MIN(remove_supplier_company.created_at) AS event_datetime
FROM remove_supplier_company
GROUP BY
remove_supplier_company.id
),
company_email_verification_send_events AS (
SELECT
company_id,
'Email Verification Send' AS email_verification_send_event,
verification_email_created_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY verification_email_created_datetime) AS first_row_number
FROM email_verifications
QUALIFY
first_row_number = 1
),
company_email_verification_complete_events AS (
SELECT
company_id,
'Email Verification Complete' AS email_verification_complete_event,
verification_email_updated_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY verification_email_updated_datetime) AS first_row_number
FROM email_verifications
WHERE
is_completed = '1'
QUALIFY
first_row_number = 1
),
company_upload_identity_events AS (
SELECT
company_id,
'Upload Identity' AS upload_identity_event,
identity_created_datetime AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY identity_created_datetime) AS first_row_number
FROM company_identity_documents
QUALIFY
first_row_number = 1
),
company_approve_identity_events AS (
SELECT
company_id,
'Approve Identity' AS approve_identity_event,
identity_approved_datetine AS event_datetime,
row_number() OVER
(PARTITION BY company_id
ORDER BY identity_created_datetime) AS first_row_number
FROM company_identity_documents
WHERE
identity_status in ('2','3')
QUALIFY
first_row_number = 1
),
company_first_time_order_events AS (
SELECT
first_time_transaction_orders.company_id,
companies.id,
'First Time Order' AS first_time_order_event,
first_time_transaction_orders.created_at AS event_datetime
FROM
(SELECT
*,
row_number() OVER
(PARTITION BY transaction_orders.company_id
ORDER BY transaction_orders.created_at) AS row_rank_index
FROM transaction_orders
QUALIFY
row_rank_index = 1
) AS first_time_transaction_orders
INNER JOIN companies
ON(first_time_transaction_orders.company_id = companies.id)
),
company_funnel_events AS (
SELECT
company_id,
register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_register_events
WHERE register_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_email_verification_send_events
WHERE email_verification_send_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_email_verification_complete_events
WHERE email_verification_complete_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
upload_identity_event,
NULL AS approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_upload_identity_events
WHERE upload_identity_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
approve_identity_event,
NULL AS first_time_order_event,
event_datetime
FROM company_approve_identity_events
WHERE approve_identity_event IS NOT NULL
UNION
SELECT
company_id,
NULL AS register_event,
NULL AS email_verification_send_event,
NULL AS email_verification_complete_event,
NULL AS upload_identity_event,
NULL AS approve_identity_event,
first_time_order_event,
event_datetime
FROM company_first_time_order_events
WHERE first_time_order_event IS NOT NULL
),
--FINAL
sem_exchange__company_funnel_events AS (
SELECT
'all_company' AS company_migration_segment,
company_funnel_events.event_datetime,
company_funnel_events.company_id,
remove_supplier_company.is_migrated_company,
count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS register_company,
count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company,
count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company,
count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company,
count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event,
count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company
FROM company_funnel_events
INNER JOIN remove_supplier_company
ON (company_funnel_events.company_id = remove_supplier_company.id)
UNION
SELECT
'exclude_migrated_company' AS company_migration_segment,
company_funnel_events.event_datetime,
company_funnel_events.company_id,
remove_supplier_company.is_migrated_company,
count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS register_company,
count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company,
count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company,
count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company,
count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event,
count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL))
OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company
FROM company_funnel_events
INNER JOIN remove_supplier_company
ON (company_funnel_events.company_id = remove_supplier_company.id)
WHERE remove_supplier_company.is_migrated_company = '0'
)
SELECT count(*) FROM sem_exchange__company_funnel_events
+84
View File
@@ -0,0 +1,84 @@
WITH container_packing_lists AS (
SELECT * FROM {{ ref('base_shipping__container_packing_lists') }}
),
containers AS (
SELECT * FROM {{ ref('base_shipping__containers') }}
),
packing_lists AS (
SELECT * FROM {{ ref('base_shipping__packing_lists') }}
),
packages AS (
SELECT * FROM {{ ref('base_shipping__packages') }}
),
company_modules AS (
SELECT * FROM {{ ref('base_shipping__company_modules') }}
),
company_connections AS (
SELECT * FROM {{ ref('base_shipping__company_connections') }}
),
orders AS (
SELECT * FROM {{ ref('base_shipping__orders') }}
)
-- LOGIC
company_module_join_companies AS (
SELECT
company_modules.id AS company_id,
company_modules.name AS company_name,
company_connections.invitee_reference AS marking
FROM
company_modules
LEFT JOIN company_connections
ON (company_modules.id = company_connections.invitee_id)
),
final AS (
SELECT
container_packing_lists.*,
containers.*,
packing_lists.*,
packages.*,
packages.created_at AS packages_created_at,
((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm
FROM container_packing_lists
JOIN containers
ON (container_packing_lists.container_id = containers.id)
JOIN packing_lists
ON (container_packing_lists.packing_list_id = packing_lists.id)
JOIN packages
ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null)
WHERE packing_lists.deleted_at is null AND packages_created_at >= '2022-01-01')
-- ),
-- final_group_by_company_module AS (
-- SELECT
-- final.owner_id,
-- SUM(final.cbm) AS life_time_cbm
-- FROM final
-- GROUP BY
-- owner_id
-- )
SELECT * FROM final
-- SELECT
-- containers.*, packing_lists.id as packing_list_id, SUM((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm
-- FROM
-- container_packing_lists as packed_containers
-- JOIN containers ON packed_containers.container_id = containers.id
-- JOIN packing_lists ON packed_containers.packing_list_id = packing_lists.id
-- JOIN packages ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null)
-- WHERE packing_lists.deleted_at is null GROUP BY packing_lists.id;
+142
View File
@@ -0,0 +1,142 @@
# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: 'cief_dbt_cloud'
version: '1.0.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project.
profile: 'default'
# These configurations specify where dbt should look for different types of files.
# The `source-paths` config, for example, states that models in this project can be
# found in the "models/" directory. You probably won't need to change these!
model-paths: ["models"]
analysis-paths: ["analyses"]
test-paths: ["tests"]
seed-paths: ["seeds"]
macro-paths: ["macros"]
snapshot-paths: ["snapshots"]
target-path: "target" # directory which will store compiled SQL files
clean-targets: # directories to be removed by `dbt clean`
- "target"
- "dbt_packages"
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
# In this example config, we tell dbt to build all models in the example/ directory
# as tables. These settings can be overridden in the individual model files
# using the `{{ config(...) }}` macro.
models:
cief_dbt_cloud:
# Applies to all files under models/example/
exchange:
marts:
reporting:
description: data modeling use in Preset (Superset)
+materialized: view
+schema: exchange__reporting
warehouse:
description: data modeling contain all fct & dim
+materialized: table
+schema: exchange__warehouse
intermediate:
description: data modeling use for pipeline transformation, contain all int
+materialized: view
+schema: exchange__intermediate
staging:
description: light_transformation & will be occur in staging, contain all stg
+materialized: view
+schema: exchange__staging
base:
description: light_transformation such as rename and remove test or wrong input data
+materialized: view
+schema: exchange__base
shipping:
marts:
reporting:
description: data modeling use in Preset (Superset)
+materialized: view
+schema: shipping__reporting
warehouse:
description: data modeling contain all fct & dim
+materialized: view
+schema: shipping__warehouse
intermediate:
description: data modeling use for pipeline transformation, contain all int
+materialized: view
+schema: shipping__intermediate
staging:
description: light_transformation & will be occur in staging, contain all stg
+materialized: view
+schema: shipping__staging
base:
description: light_transformation such as rename and remove test or wrong input data
+materialized: view
+schema: shipping__base
crisp:
marts:
reporting:
description: data modeling use in Preset (Superset)
+materialized: view
+schema: crisp__reporting
warehouse:
description: data modeling contain all fct & dim
+materialized: view
+schema: crisp__warehouse
intermediate:
description: data modeling use for pipeline transformation, contain all int
+materialized: view
+schema: crisp__intermediate
staging:
description: light_transformation & will be occur in staging, contain all stg
+materialized: view
+schema: crisp__staging
base:
description: light_transformation such as rename and remove test or wrong input data
+materialized: view
+schema: crisp__base
seeds:
cief_dbt_cloud:
exchange:
+schema: exchange__seeds
shipping:
+schema: shipping__seeds
crisp:
+schema: crisp__seeds
snapshots:
cief_dbt_cloud:
exchange:
+target_schema: "{% if target.name == 'prod' %}PROD_EXCHANGE__SNAPSHOTS{% else %}{{ target.schema }}_EXCHANGE__SNAPSHOTS{% endif %}"
shipping:
+target_schema: "{% if target.name == 'prod' %}PROD_SHIPPING__SNAPSHOTS{% else %}{{ target.schema }}_SHIPPING__SNAPSHOTS{% endif %}"
crisp:
+target_schema: "{% if target.name == 'prod' %}PROD_CRISP__SNAPSHOTS{% else %}{{ target.schema }}_CRISP__SNAPSHOTS{% endif %}"
tests:
+severity: warn
View File
@@ -0,0 +1,77 @@
--IMPORT
WITH messages AS (
SELECT * FROM {{ ref('stg_crisp__messages') }}
ORDER BY conversation_id, message_created_datetime, id
),
conversations AS (
SELECT * FROM {{ ref('stg_crisp__conversations') }}
),
--LOGIC
mark_first_dialog_message AS (
SELECT
CASE
WHEN LAG(event_type) OVER (ORDER BY conversation_id, message_created_datetime, id) = 'state:resolved'
THEN 1
ELSE 0
END AS first_dialog_message_flag,
*
FROM messages
),
message_with_dialog_id AS (
SELECT
(SUM(first_dialog_message_flag) OVER (ORDER BY conversation_id, message_created_datetime, id ROWS UNBOUNDED PRECEDING))+1 AS dialog_id,
*
FROM mark_first_dialog_message
ORDER BY conversation_id, message_created_datetime
),
--FINAL
final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_type AS (
SELECT
message_with_dialog_id.dialog_id,
message_with_dialog_id.id,
message_with_dialog_id.conversation_id,
CASE
WHEN message_with_dialog_id.message_from_side = 'operator' THEN message_with_dialog_id.user_id
WHEN message_with_dialog_id.message_from_side = 'user' and conversations.user_id IS NOT NULL THEN conversations.user_id
WHEN message_with_dialog_id.message_from_side = 'user' and conversations.user_id IS NULL THEN null
ELSE 'ERROR please contact DATA team'
END AS user_id,
message_with_dialog_id.user_name,
message_with_dialog_id.message_from_side,
CASE
WHEN message_with_dialog_id.message_from_side = 'user' AND message_with_dialog_id.user_id IS NOT NULL THEN 'user'
WHEN message_with_dialog_id.message_from_side = 'operator' AND message_with_dialog_id.user_id IS NOT NULL THEN 'operator'
WHEN message_with_dialog_id.message_from_side = 'operator' AND message_with_dialog_id.user_id IS NULL THEN 'bot'
ELSE 'ERROR please contact DATA team'
END AS message_generate_by,
message_with_dialog_id.message_generate_platform,
message_with_dialog_id.message_deliverd_platform,
message_with_dialog_id.message_read_platform,
message_with_dialog_id.message_format,
message_with_dialog_id.event_type,
message_with_dialog_id.event_message,
message_with_dialog_id.message,
message_with_dialog_id.is_edited_message,
message_with_dialog_id.attachment_type,
message_with_dialog_id.attachment_name,
message_with_dialog_id.attachment_url,
message_with_dialog_id.message_created_datetime
FROM message_with_dialog_id
LEFT JOIN conversations
ON (message_with_dialog_id.user_id = conversations.id) --The user_id will become conversation_id if message_from_side is user
)
SELECT * FROM final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_type
@@ -0,0 +1,68 @@
--IMPORT
WITH peoples AS (
SELECT * FROM {{ ref('stg_crisp__peoples') }}
),
people_data AS (
SELECT * FROM {{ ref('stg_crisp__people_data') }}
),
--LOGIC
people_get_marking_id AS (
SELECT
peoples.id,
peoples.name,
peoples.phone,
peoples.email,
peoples.website,
peoples.score,
peoples.segment_list,
peoples.role,
peoples.seniority,
peoples.title,
peoples.gender,
peoples.timezone,
peoples.geo_country,
peoples.geo_city,
peoples.geo_region_code,
peoples.address,
peoples.locale_list,
peoples.geo_latitude,
peoples.geo_longitude,
peoples.last_activated_datetime,
peoples.people_created_datetime,
peoples.people_updated_datetime,
peoples.company_name,
peoples.company_legal_name,
peoples.company_description,
peoples.company_phone,
peoples.company_email_list,
peoples.company_website,
peoples.company_domain,
peoples.company_tag_list,
peoples.company_number_of_employees,
peoples.company_market_cap,
peoples.company_country,
peoples.company_city,
peoples.company_region,
peoples.company_latitude,
peoples.company_logitude,
peoples.company_timezone,
people_data.company_name AS marking_company_name,
people_data.whatsapp_number,
people_data.whatsapp_business_number,
people_data.facebook_page_name,
people_data.facebook_page_id,
people_data.exchange_marking_id,
people_data.izyim_marking_id,
people_data.lite_marking_id
FROM peoples
LEFT JOIN people_data
ON (peoples.id = people_data.people_id)
)
--FINAL
select * from people_get_marking_id
@@ -0,0 +1,86 @@
--IMPORT
WITH users_and_companies AS (
SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }}
),
--LOGIC
split_company_data AS (
SELECT
exchange_marking_id,
izyim_marking_id,
lite_marking_id,
marking_company_name,
whatsapp_number,
whatsapp_business_number,
facebook_page_name,
facebook_page_id,
company_name,
company_legal_name,
company_description,
company_phone,
company_email_list,
company_website,
company_domain,
company_tag_list,
company_number_of_employees,
company_market_cap,
company_country,
company_city,
company_region,
company_latitude,
company_logitude,
company_timezone
FROM users_and_companies
),
companies_remove_duplicate AS (
SELECT
DISTINCT *
FROM split_company_data
),
--FINAL
final_dim_crisp__companies AS (
SELECT
exchange_marking_id,
izyim_marking_id,
lite_marking_id,
IFF(COUNT(DISTINCT(marking_company_name)) > 1, LISTAGG(marking_company_name, ','), MAX(marking_company_name) ) AS marking_company_name,
IFF(COUNT(DISTINCT(whatsapp_number)) > 1, LISTAGG(whatsapp_number, ','), MAX(whatsapp_number) ) AS whatsapp_number,
IFF(COUNT(DISTINCT(whatsapp_business_number)) > 1, LISTAGG(whatsapp_business_number, ','), MAX(whatsapp_business_number) ) AS whatsapp_business_number,
IFF(COUNT(DISTINCT(facebook_page_name)) > 1, LISTAGG(facebook_page_name, ','), MAX(facebook_page_name) ) AS facebook_page_name,
IFF(COUNT(DISTINCT(facebook_page_id)) > 1, LISTAGG(facebook_page_id, ','), MAX(facebook_page_id) ) AS facebook_page_id,
IFF(COUNT(DISTINCT(company_name)) > 1, LISTAGG(company_name, ','), MAX(company_name) ) AS company_name,
IFF(COUNT(DISTINCT(company_legal_name)) > 1, LISTAGG(company_legal_name, ','), MAX(company_legal_name) ) AS company_legal_name,
IFF(COUNT(DISTINCT(company_description)) > 1, LISTAGG(company_description, ','), MAX(company_description) ) AS company_description,
IFF(COUNT(DISTINCT(company_phone)) > 1, LISTAGG(company_phone, ','), MAX(company_phone) ) AS company_phone,
IFF(COUNT(DISTINCT(company_email_list)) > 1, LISTAGG(company_email_list, ','), MAX(company_email_list) ) AS company_email_list,
IFF(COUNT(DISTINCT(company_website)) > 1, LISTAGG(company_website, ','), MAX(company_website) ) AS company_website,
IFF(COUNT(DISTINCT(company_domain)) > 1, LISTAGG(company_domain, ','), MAX(company_domain) ) AS company_domain,
IFF(COUNT(DISTINCT(company_tag_list)) > 1, LISTAGG(company_tag_list, ','), MAX(company_tag_list) ) AS company_tag_list,
IFF(COUNT(DISTINCT(company_number_of_employees)) > 1, LISTAGG(company_number_of_employees, ','), MAX(company_number_of_employees) ) AS company_number_of_employees,
IFF(COUNT(DISTINCT(company_market_cap)) > 1, LISTAGG(company_market_cap, ','), MAX(company_market_cap) ) AS company_market_cap,
IFF(COUNT(DISTINCT(company_country)) > 1, LISTAGG(company_country, ','), MAX(company_country) ) AS company_country,
IFF(COUNT(DISTINCT(company_city)) > 1, LISTAGG(company_city, ','), MAX(company_city) ) AS company_city,
IFF(COUNT(DISTINCT(company_region)) > 1, LISTAGG(company_region, ','), MAX(company_region) ) AS company_region,
IFF(COUNT(DISTINCT(company_latitude)) > 1, LISTAGG(company_latitude, ','), MAX(company_latitude) ) AS company_latitude,
IFF(COUNT(DISTINCT(company_logitude)) > 1, LISTAGG(company_logitude, ','), MAX(company_logitude) ) AS company_logitude,
IFF(COUNT(DISTINCT(company_timezone)) > 1, LISTAGG(company_timezone, ','), MAX(company_timezone) ) AS company_timezone
FROM companies_remove_duplicate
WHERE
(IFF(exchange_marking_id IS NULL, 1, 0) +
IFF(izyim_marking_id IS NULL, 1, 0) +
IFF(lite_marking_id IS NULL, 1, 0)) < 3
GROUP BY
exchange_marking_id,
izyim_marking_id,
lite_marking_id
)
-- SELECT EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID, count(*) FROM final_dim_crisp__companies GROUP BY EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID ORDER BY count(*) desc
SELECT * FROM final_dim_crisp__companies
@@ -0,0 +1,48 @@
--IMPORT
WITH users_and_companies AS (
SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }}
),
--LOGIC
split_user_data AS (
SELECT
id,
name,
phone,
email,
website,
score,
segment_list,
role,
seniority,
title,
gender,
timezone,
geo_country,
geo_city,
geo_region_code,
address,
locale_list,
geo_latitude,
geo_longitude,
last_activated_datetime,
people_created_datetime,
people_updated_datetime,
exchange_marking_id AS company_exchange_marking_id,
izyim_marking_id AS company_izyim_marking_id,
lite_marking_id AS company_lite_marking_id
FROM users_and_companies
),
--FINAL
final_dim_crisp__users AS (
SELECT
*
FROM split_user_data
)
SELECT * FROM final_dim_crisp__users
-- SELECT company_lite_marking_id, count(distinct(company_exchange_marking_id)) AS t1, count(distinct(company_izyim_marking_id)) AS t2
-- FROM final_dim_crisp__users GROUP BY company_lite_marking_id HAVING t1 >1 or t2 > 1
-- SELECT company_IZYIM_MARKING_ID, company_exchange_marking_id FROM final_dim_crisp__users
-- WHERE company_IZYIM_MARKING_ID IN ('1324FEN','3791YBE', '1291NSC', '2698LXA', '2431HEB', '2634PLT', '8299LIE', '1882KAI', '1889LBH', '913DLT', '1650GTB', '5903CSB', '6631GEB', '1656HBP', '1905JYW', '8265LSB', '7260UUT', '5989HSY') ORDER BY company_IZYIM_MARKING_ID
@@ -0,0 +1,137 @@
--IMPORT
WITH messages AS (
SELECT * FROM {{ ref('int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by') }}
ORDER BY conversation_id, message_created_datetime, id
),
--LOGIC
messages_with_ftu_id AS (
SELECT
*,
FIRST_VALUE(CASE WHEN messages.message_generate_by = 'user' THEN messages.user_id END)
IGNORE NULLS
OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS first_user_id,
LAST_VALUE(CASE WHEN messages.message_generate_by = 'user' THEN messages.user_id END)
IGNORE NULLS
OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS last_user_id,
FIRST_VALUE(CASE WHEN messages.message_generate_by = 'operator' THEN messages.user_id END)
IGNORE NULLS
OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS first_operator_id,
LAST_VALUE(CASE WHEN messages.message_generate_by = 'operator' THEN messages.user_id END)
IGNORE NULLS
OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS last_operator_id
from messages
),
dialog_message_last_row AS (
SELECT
RANK() OVER (PARTITION BY dialog_id ORDER BY message_created_datetime DESC) AS last_row_desc_rank,
*
FROM messages
QUALIFY last_row_desc_rank = 1
),
--FINAL
final_fct_crisp__dialogs AS (
SELECT
messages_with_ftu_id.dialog_id AS dialog_id,
max(messages_with_ftu_id.conversation_id) AS conversation_id,
array_agg(messages_with_ftu_id.id) AS message_ids,
CASE
WHEN max(dialog_message_last_row.event_type) = 'state:resolved' THEN 'close'
WHEN max(dialog_message_last_row.event_type) IS NULL THEN 'open'
ELSE 'ERROR, Please contact DATA team'
END AS dialog_status,
min(messages_with_ftu_id.message_created_datetime) AS dialog_created_datetime,
min(IFF(messages_with_ftu_id.message_generate_by = 'user',
messages_with_ftu_id.message_created_datetime,
null)) AS first_user_reply_datetime,
min(IFF(messages_with_ftu_id.message_generate_by = 'operator',
messages_with_ftu_id.message_created_datetime,
null)) AS first_operator_reply_datetime,
min(IFF(messages_with_ftu_id.message_generate_by = 'bot',
messages_with_ftu_id.message_created_datetime,
null)) AS first_bot_reply_datetime,
max(IFF(messages_with_ftu_id.message_generate_by = 'user',
messages_with_ftu_id.message_created_datetime,
null)) AS last_user_reply_datetime,
max(IFF(messages_with_ftu_id.message_generate_by = 'operator',
messages_with_ftu_id.message_created_datetime,
null)) AS last_operator_reply_datetime,
max(IFF(messages_with_ftu_id.message_generate_by = 'bot',
messages_with_ftu_id.message_created_datetime,
null)) AS last_bot_reply_datetime,
max(IFF((messages_with_ftu_id.event_type = 'state:resolved'),
messages_with_ftu_id.message_created_datetime,
null)) AS dialog_solved_datetime,
array_agg(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'user' ,
messages_with_ftu_id.user_id,
NULL))) AS user_ids,
count(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'user',
messages_with_ftu_id.user_id,
null))) AS number_of_user_involved,
max(messages_with_ftu_id.first_user_id) AS first_user_id,
max(messages_with_ftu_id.last_user_id) AS last_user_id,
array_agg(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'operator' ,
messages_with_ftu_id.user_id,
NULL))) AS operator_ids,
count(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'operator',
messages_with_ftu_id.user_id,
null))) AS number_of_operator_involved,
max(messages_with_ftu_id.first_operator_id) AS first_operator_id,
max(messages_with_ftu_id.last_operator_id) AS last_operator_id,
count(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'user',
messages_with_ftu_id.id,
null))) AS number_of_user_chat,
count(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'operator',
messages_with_ftu_id.id,
null))) AS number_of_operator_chat,
count(distinct(
IFF(messages_with_ftu_id.message_generate_by = 'bot',
messages_with_ftu_id.id,
null))) AS number_of_bot_chat
FROM messages_with_ftu_id
LEFT JOIN dialog_message_last_row
ON (dialog_message_last_row.dialog_id = messages_with_ftu_id.dialog_id)
GROUP BY
messages_with_ftu_id.dialog_id
)
SELECT * FROM final_fct_crisp__dialogs
--Check 'K Huan Huan' why result number_of_user_id = 2 session_id = 'session_75a3a65d-390b-4ed9-90a7-f98a8a39b768';
@@ -0,0 +1,15 @@
--IMPORT
WITH snap_user_email_subscriptions AS (
SELECT * FROM {{ ref('stg_crisp__people_email_subscriptions') }}
),
--FINAL
final_fct_crisp__user_email_subscriptions AS (
SELECT
*
FROM snap_user_email_subscriptions
)
SELECT * FROM snap_user_email_subscriptions
@@ -0,0 +1,26 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: get_message_in_conversations
description: Raw `get_people_subscription_status` table from Snowflake
columns:
- name: fingerprint
description: Primary key for 'get_message_in_conversations'
tests:
- unique
- not_null
- name: delivered
description: the platform which after the message is output from origin, and delivered to the user/operator. When an operator send a message but the user is not online anymore, we send it via email only after 2min during these 2min, delivered is empty
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: get_people_data
description: Raw `get_people_data` table from Snowflake
columns:
- name: people_id
description: Primary key for 'get_people_data'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: get_people_subscription_status
description: Raw `get_people_subscription_status` table from Snowflake
columns:
- name: people_id
description: Primary key for 'get_people_subscription_status'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: list_conversations
description: Raw `get_people_subscription_status` table from Snowflake
columns:
- name: SESSION_ID
description: Primary key for 'list_conversations'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: list_people_profiles
description: Raw `list_people_profiles` table from Snowflake
columns:
- name: people_id
description: Primary key for 'list_people_profiles'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_crisp_api
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: CRISP_API
tables:
- name: list_website_operator
description: Raw `list_website_operator` table from Snowflake
columns:
- name: details__user_id
description: Primary key for 'list_website_operator'
tests:
- unique
- not_null
loaded_at_field: api_called_at
freshness:
warn_after: {count: 1, period: day}
error_after: {count: 2, period: hour}
@@ -0,0 +1,22 @@
-- IMPORT
WITH list_conversations AS (
SELECT * FROM {{ source('src_crisp_api', 'list_conversations') }}
),
--LOGIC
--FINAL
final_stg_crisp__conversations AS (
SELECT
session_id AS id,
people_id AS user_id,
state,
status,
IFF(is_blocked = TRUE, 1, 0) AS is_blocked,
assigned__user_id AS operator_id,
convert_timezone('Asia/Kuala_Lumpur', created_at) AS conversation_created_datetime,
convert_timezone('Asia/Kuala_Lumpur', updated_at) AS conversation_updated_datetime
FROM list_conversations
)
SELECT * FROM final_stg_crisp__conversations
@@ -0,0 +1,32 @@
-- IMPORT
WITH get_message_in_conversations AS (
SELECT * FROM {{ source('src_crisp_api', 'get_message_in_conversations') }}
),
-- LOGIC
-- FINAL
final_stg_crisp__messages AS (
SELECT
fingerprint AS id,
session_id AS conversation_id,
user__user_id AS user_id,
user__nickname AS user_name,
from_user AS message_from_side,
origin AS message_generate_platform,
delivered AS message_deliverd_platform,
read AS message_read_platform,
type AS message_format,
content__namespace AS event_type,
content__text AS event_message,
content AS message,
edited AS is_edited_message,
content__type AS attachment_type,
content__name AS attachment_name,
content__url AS attachment_url,
convert_timezone('Asia/Kuala_Lumpur', timestamp) AS message_created_datetime
FROM get_message_in_conversations ORDER BY session_id, message_created_datetime
)
SELECT * FROM final_stg_crisp__messages
@@ -0,0 +1,20 @@
-- IMPORT
WITH list_website_operator AS (
SELECT * FROM {{ source('src_crisp_api', 'list_website_operator') }}
),
--LOGIC
--FINAL
final_stg_crisp__operators AS (
SELECT
details__user_id AS id,
details__title AS title,
details__first_name AS first_name,
details__last_name AS last,
details__email AS email
FROM list_website_operator
)
SELECT * FROM final_stg_crisp__operators
@@ -0,0 +1,23 @@
-- IMPORT
WITH get_people_data AS (
SELECT * FROM {{ source('src_crisp_api', 'get_people_data') }}
),
--LOGIC
--FINAL
final_stg_crisp__get_people_data AS (
SELECT
people_id,
data__company_name AS company_name,
data__wa_number AS whatsapp_number,
data__whatsapp_business_number AS whatsapp_business_number,
data__fb_page AS facebook_page_name,
data__fb_page_id AS facebook_page_id,
data__exchange_marking AS exchange_marking_id,
data__izyim_marking AS izyim_marking_id,
data__lite_marking AS lite_marking_id
FROM get_people_data
)
SELECT * FROM final_stg_crisp__get_people_data
@@ -0,0 +1,20 @@
-- IMPORT
WITH get_people_subscription_status AS (
SELECT * FROM {{ ref('snap_crisp__get_people_subscription_status') }}
),
--LOGIC
--FINAL
final_stg_crisp__people_email_subscriptions AS (
SELECT
dbt_scd_id,
people_id,
IFF(email = TRUE, 1, 0) AS is_subscribed,
DBT_VALID_FROM AS dbt_valid_from_datetime,
DBT_VALID_TO AS dbt_valid_to_datetime
FROM get_people_subscription_status
)
SELECT * FROM final_stg_crisp__people_email_subscriptions
@@ -0,0 +1,54 @@
-- IMPORT
WITH list_people_profiles AS (
SELECT * FROM {{ source('src_crisp_api', 'list_people_profiles') }}
),
--LOGIC
--FINAL
final_stg_crisp__peoples AS (
SELECT
people_id AS id,
person__nickname AS name,
person__phone AS phone,
email AS email,
person__website AS website,
score AS score,
segments AS segment_list,
person__employment__role AS role,
person__employment__seniority AS seniority,
person__employment__title AS title,
person__gender AS gender,
person__timezone AS timezone,
person__geolocation__country AS geo_country,
person__geolocation__city AS geo_city,
person__geolocation__region AS geo_region_code,
person__address AS address,
person__locales AS locale_list,
person__geolocation__coordinates__latitude AS geo_latitude,
person__geolocation__coordinates__longitude AS geo_longitude,
convert_timezone('Asia/Kuala_Lumpur', active__last) AS last_activated_datetime,
convert_timezone('Asia/Kuala_Lumpur', created_at) AS people_created_datetime,
convert_timezone('Asia/Kuala_Lumpur', updated_at) AS people_updated_datetime,
company__name AS company_name,
company__legal_name AS company_legal_name,
company__description AS company_description,
company__phones AS company_phone,
company__emails AS company_email_list,
company__url AS company_website,
company__domain AS company_domain,
company__tags AS company_tag_list,
company__metrics__employees AS company_number_of_employees,
company__metrics__market_cap AS company_market_cap,
company__geolocation__country AS company_country,
company__geolocation__city AS company_city,
company__geolocation__region AS company_region,
company__geolocation__coordinates__latitude AS company_latitude,
company__geolocation__coordinates__longitude AS company_logitude,
company__timezone AS company_timezone
FROM list_people_profiles
)
SELECT * FROM final_stg_crisp__peoples
@@ -0,0 +1,46 @@
-- IMPORTS
WITH activity_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'activity_log') }}
),
-- LOGIC
activity_logs_rename AS (
SELECT
activity_logs.id,
activity_logs.log_name,
activity_logs.description,
activity_logs.subject_id,
activity_logs.subject_type,
activity_logs.causer_id,
activity_logs.causer_type,
activity_logs.properties,
activity_logs.created_at,
activity_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM activity_logs
),
-- FINAL
final__base_exchange__activity_logs AS (
SELECT
id,
log_name,
description,
subject_id,
subject_type,
causer_id,
causer_type,
properties,
created_at,
updated_at,
_dbt_ran_at
FROM activity_logs_rename
)
SELECT * FROM final__base_exchange__activity_logs
@@ -0,0 +1,48 @@
-- IMPORTS
WITH addresses AS (
SELECT * FROM {{ source('src_exchange_mysql', 'addresses') }}
),
-- LOGIC
addresses_rename AS (
SELECT
addresses.id,
addresses.company_id,
addresses.country_id,
addresses.state_id,
addresses.district_id,
addresses.postcode,
addresses.street_one AS address_line_one,
addresses.street_two AS address_line_two,
addresses.billing AS is_billing_address,
addresses.deleted_at,
addresses.created_at,
addresses.updated_At,
current_timestamp() as _dbt_ran_at
FROM addresses
),
-- FINAL
final__base_exchange__addresses AS (
SELECT
id,
company_id,
country_id,
state_id,
district_id,
postcode,
address_line_one,
address_line_two,
is_billing_address,
deleted_at,
created_at,
updated_At,
_dbt_ran_at
FROM addresses_rename
)
SELECT * FROM final__base_exchange__addresses
@@ -0,0 +1,34 @@
-- IMPORTS
WITH announcement_segments AS (
SELECT * FROM {{ source('src_exchange_mysql', 'announcement_segment') }}
),
-- LOGIC
announcement_segments_rename AS (
SELECT
announcement_segments.announcement_id,
announcement_segments.segment_id,
announcement_segments.created_at,
announcement_segments.updated_at,
current_timestamp() AS _dbt_ran_at
FROM announcement_segments
),
-- FINAL
final__base_exchange__announcement_segments AS (
SELECT
announcement_id,
segment_id,
created_at,
updated_at,
_dbt_ran_at
FROM announcement_segments_rename
)
SELECT * FROM final__base_exchange__announcement_segments
@@ -0,0 +1,50 @@
-- IMPORTS
WITH announcements AS (
SELECT * FROM {{ source('src_exchange_mysql', 'announcements') }}
),
-- LOGIC
announcements_rename AS (
SELECT
id,
title,
description,
starting_on AS started_at,
ending_on AS ended_at,
is_all_day,
duration,
is_recurring,
recurrence_pattern,
deleted_at,
created_at,
updated_at,
current_timestamp() as _dbt_ran_at
FROM announcements
),
-- FINAL
final__base_exchange__announcements AS (
SELECT
id,
title,
description,
started_at,
ended_at,
is_all_day,
duration,
is_recurring,
recurrence_pattern,
deleted_at,
created_at,
updated_at,
current_timestamp() as _dbt_ran_at
FROM announcements_rename
)
SELECT * FROM final__base_exchange__announcements
@@ -0,0 +1,61 @@
-- IMPORTS
WITH bank_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'bank_logs') }}
),
-- LOGIC
bank_logs_rename AS (
SELECT
bank_logs.id,
bank_logs.bank_id,
bank_logs.company_id,
bank_logs.reference,
bank_logs.bank_name,
bank_logs.holder_name,
bank_logs.account_no,
bank_logs.bank_branch,
bank_logs.swift AS swift_code,
bank_logs.snap AS cnaps_code,
bank_logs.type AS bank_type,
bank_logs.default AS is_default,
bank_logs.status,
bank_logs.country_id,
bank_logs.deleted_at,
bank_logs.created_at,
bank_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM bank_logs
),
-- FINAL
final__base_exchange__bank_logs AS (
SELECT
id,
bank_id,
company_id,
reference,
bank_name,
holder_name,
account_no,
bank_branch,
swift_code,
cnaps_code,
bank_type,
is_default,
status,
country_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM bank_logs_rename
)
SELECT * FROM final__base_exchange__bank_logs
@@ -0,0 +1,58 @@
-- IMPORTS
WITH banks AS (
SELECT * FROM {{ source('src_exchange_mysql', 'banks') }}
),
-- LOGIC
banks_rename AS (
SELECT
banks.id,
banks.company_id,
banks.reference,
banks.bank_name,
banks.holder_name,
banks.account_no,
banks.bank_branch,
banks.swift AS swift_code,
banks.snap AS cnaps_code,
banks.type AS bank_type,
banks.default AS is_default,
banks.status,
banks.country_id,
banks.deleted_at,
banks.created_at,
banks.updated_at,
current_timestamp() AS _dbt_ran_at
FROM banks
),
-- FINAL
final__base_exchange__banks AS (
SELECT
id,
company_id,
reference,
bank_name,
holder_name,
account_no,
bank_branch,
swift_code,
cnaps_code,
bank_type,
is_default,
status,
country_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM banks_rename
)
SELECT * FROM final__base_exchange__banks
@@ -0,0 +1,55 @@
-- IMPORTS
WITH booking_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'booking_logs') }}
),
-- LOGIC
booking_logs_rename AS (
SELECT
booking_logs.id,
booking_logs.booking_id,
booking_logs.company_id,
booking_logs.marking AS marking_id,
booking_logs.service_id AS service_type,
booking_logs.bank_id,
booking_logs.fix_amount AS fix_value,
booking_logs.fix_currency_id,
booking_logs.convertible_currency_id AS quote_currency_id,
booking_logs.conversion_currency_id AS base_currency_id,
booking_logs.status,
booking_logs.deleted_at,
booking_logs.created_at,
booking_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM booking_logs
),
-- FINAL
final_base_exchange__booking_logs AS (
SELECT
id,
booking_id,
company_id,
marking_id,
service_type,
bank_id,
fix_value,
fix_currency_id,
quote_currency_id,
base_currency_id,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM booking_logs_rename
)
SELECT * FROM final_base_exchange__booking_logs
@@ -0,0 +1,53 @@
-- IMPORTS
WITH bookings AS (
SELECT * FROM {{ source('src_exchange_mysql', 'bookings') }}
),
-- LOGIC
bookings_rename AS (
SELECT
bookings.id,
bookings.company_id,
bookings.marking AS marking_id,
bookings.service_id AS service_type,
bookings.bank_id,
bookings.fix_amount AS fix_value,
bookings.fix_currency_id,
bookings.convertible_currency_id AS quote_currency_id,
bookings.conversion_currency_id AS base_currency_id,
bookings.status,
bookings.deleted_at,
bookings.created_at,
bookings.updated_at,
current_timestamp() AS _dbt_ran_at
FROM bookings
),
-- FINAL
final_base_exchange__bookings AS (
SELECT
id,
company_id,
marking_id,
service_type,
bank_id,
fix_value,
fix_currency_id,
quote_currency_id,
base_currency_id,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM bookings_rename
)
SELECT * FROM final_base_exchange__bookings
@@ -0,0 +1,47 @@
-- IMPORTS
WITH companies AS (
SELECT * FROM {{ source('src_exchange_mysql', 'companies') }}
),
-- LOGIC
companies_rename AS (
SELECT
companies.id,
companies.name,
companies.reference AS marking_id,
companies.debtor AS account_software_id,
companies.type AS company_type,
companies.business_type,
companies.status,
companies.deleted_at,
companies.created_at,
companies.updated_at,
current_timestamp() AS _dbt_ran_at
FROM companies
),
-- FINAL
final__base_exchange__companies AS (
SELECT
id,
name,
marking_id,
account_software_id,
company_type,
business_type,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM companies_rename
)
SELECT * FROM final__base_exchange__companies
@@ -0,0 +1,48 @@
-- IMPORTS
WITH contacts AS (
SELECT * FROM {{ source('src_exchange_mysql', 'contacts') }}
),
-- LOGIC
companies_rename AS (
SELECT
contacts.id,
contacts.company_id,
contacts.country_id,
contacts.reference,
contacts.phone,
contacts.email,
contacts.wechat_id,
contacts.deleted_at,
contacts.created_at,
contacts.updated_at,
current_timestamp() AS _dbt_ran_at
FROM contacts
),
-- FINAL
final__base_exchange__contacts AS (
SELECT
id,
company_id,
country_id,
reference,
phone,
email,
wechat_id,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM companies_rename
)
SELECT * FROM final__base_exchange__contacts
@@ -0,0 +1,41 @@
-- IMPORTS
WITH countries AS (
SELECT * FROM {{ source('src_exchange_mysql', 'countries') }}
),
-- LOGIC
countries_rename AS (
SELECT
countries.id,
countries.name,
countries.short_code,
countries.phone_code,
countries.deleted_at,
countries.created_at,
countries.updated_at,
current_timestamp() AS _dbt_ran_at
FROM countries
),
-- FINAL
final__base_exchange__countries AS (
SELECT
id,
name,
short_code,
phone_code,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM countries_rename
)
SELECT * FROM final__base_exchange__countries
@@ -0,0 +1,43 @@
-- IMPORTS
WITH currencies AS (
SELECT * FROM {{ source('src_exchange_mysql', 'currencies') }}
),
-- LOGIC
currencies_rename AS (
SELECT
currencies.id,
currencies.country_id,
currencies.name,
currencies.short_code,
currencies.symbol,
currencies.deleted_at,
currencies.created_at,
currencies.updated_at,
current_timestamp() AS _dbt_ran_at
FROM currencies
),
-- FINAL
final__base_exchange__currencies AS (
SELECT
id,
country_id,
name,
short_code,
symbol,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM currencies_rename
)
SELECT * FROM final__base_exchange__currencies
@@ -0,0 +1,39 @@
-- IMPORTS
WITH currency_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'currency_logs') }}
),
-- LOGIC
currency_logs_rename AS (
SELECT
currency_logs.id,
currency_logs.currency_id,
currency_logs.created_by,
currency_logs.deleted_at,
currency_logs.created_at,
currency_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM currency_logs
),
-- FINAL
final__base_exchange__currency_logs AS (
SELECT
id,
currency_id,
created_by,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM currency_logs_rename
)
SELECT * FROM final__base_exchange__currency_logs
@@ -0,0 +1,50 @@
-- IMPORTS
WITH currency_rate_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'currency_rate_logs') }}
),
-- LOGIC
currency_rate_logs_clean_list AS (
SELECT
id,
round(1/selling, 5) AS selling
FROM
currency_rate_logs
WHERE
id IN ('579','580','581','582')
),
currency_rate_logs_rename_clean_wrong_rate AS (
SELECT
currency_rate_logs.id,
currency_rate_logs.currency_rate_id,
COALESCE(currency_rate_logs_clean_list.selling, currency_rate_logs.selling) AS base_to_quote_currency_exchange_rate,
currency_rate_logs.created_by,
currency_rate_logs.created_at,
currency_rate_logs.updated_at,
CURRENT_TIMESTAMP() AS _dbt_ran_at
FROM currency_rate_logs
LEFT JOIN currency_rate_logs_clean_list
ON (currency_rate_logs.id = currency_rate_logs_clean_list.id)
),
-- FINAL
final__base_exchange__currency_rate_logs AS (
SELECT
id,
currency_rate_id,
base_to_quote_currency_exchange_rate,
created_by,
created_at,
updated_at,
_dbt_ran_at
FROM currency_rate_logs_rename_clean_wrong_rate
)
SELECT * FROM final__base_exchange__currency_rate_logs
@@ -0,0 +1,43 @@
-- IMPORTS
WITH currency_rates AS (
SELECT * FROM {{ source('src_exchange_mysql', 'currency_rates') }}
),
-- LOGIC
currency_rates_rename AS (
SELECT
currency_rates.id,
currency_rates.currency_id AS quote_currency_id,
currency_rates.selling AS base_to_quote_currency_exchange_rate,
currency_rates.payment_method_type AS payment_method,
currency_rates.service_id AS service_type,
currency_rates.deleted_at,
currency_rates.created_at,
currency_rates.updated_at,
current_timestamp() AS _dbt_ran_at
FROM currency_rates
),
-- FINAL
final__base_exchange__currency_rates AS (
SELECT
id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
payment_method,
service_type,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM currency_rates_rename
)
SELECT * FROM final__base_exchange__currency_rates
@@ -0,0 +1,44 @@
-- IMPORTS
WITH districts AS (
SELECT * FROM {{ source('src_exchange_mysql', 'districts') }}
),
-- LOGIC
districts_rename AS (
SELECT
districts.id,
districts.country_id,
districts.state_id,
districts.name,
districts.postcode,
districts.status,
districts.deleted_at,
districts.created_at,
districts.updated_at,
current_timestamp() AS _dbt_ran_at
FROM districts
),
-- FINAL
final__base_exchange__districts AS (
SELECT
id,
country_id,
state_id,
name,
postcode,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM districts_rename
)
SELECT * FROM final__base_exchange__districts
@@ -0,0 +1,53 @@
-- IMPORTS
WITH documents AS (
SELECT * FROM {{ source('src_exchange_mysql', 'documents') }}
),
-- LOGIC
documents_rename AS (
SELECT
documents.id,
documents.owner_type,
documents.owner_id,
documents.document_type,
documents.reference,
documents.status,
documents.issued_date AS issued_at,
documents.expired_date AS expired_at,
documents.approver AS approved_by,
documents.approval_date AS approved_at,
documents.deleted_at,
documents.created_at,
documents.updated_at,
current_timestamp() AS _dbt_ran_at
FROM documents
),
-- FINAL
final__base_exchange__documents AS (
SELECT
id,
owner_type,
owner_id,
document_type,
reference,
status,
issued_at,
expired_at,
approved_by,
approved_at,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM documents_rename
)
SELECT * FROM final__base_exchange__documents
@@ -0,0 +1,32 @@
-- IMPORTS
WITH employees AS (
SELECT * FROM {{ source('src_exchange_mysql', 'employees') }}
),
-- LOGIC
employees_rename AS (
SELECT
employees.company_id,
employees.user_id,
employees.status,
current_timestamp() AS _dbt_ran_at
FROM employees
),
-- FINAL
final__base_exchange__employees AS (
SELECT
company_id,
user_id,
status,
_dbt_ran_at
FROM employees_rename
)
SELECT * FROM final__base_exchange__employees
@@ -0,0 +1,40 @@
-- IMPORTS
WITH failed_jobs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'failed_jobs') }}
),
-- LOGIC
failed_jobs_rename AS (
SELECT
failed_jobs.id,
failed_jobs.connection,
failed_jobs.queue,
failed_jobs.payload,
failed_jobs.exception,
failed_jobs.failed_at,
current_timestamp() AS _dbt_ran_at
FROM failed_jobs
),
-- FINAL
final__stg_failed_jobs__exchange AS (
SELECT
id,
connection,
queue,
payload,
exception,
failed_at,
_dbt_ran_at
FROM failed_jobs_rename
)
SELECT * FROM final__stg_failed_jobs__exchange
@@ -0,0 +1,42 @@
-- IMPORTS
WITH files AS (
SELECT * FROM {{ source('src_exchange_mysql', 'files') }}
),
-- LOGIC
files_rename AS (
SELECT
files.id,
files.document_id,
files.file,
files.file_type,
files.deleted_at,
files.created_at,
files.updated_at,
current_timestamp() AS _dbt_ran_at
FROM files
),
-- FINAL
final__base_exchange__files AS (
SELECT
id,
document_id,
file,
file_type,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM files_rename
)
SELECT * FROM final__base_exchange__files
@@ -0,0 +1,34 @@
-- IMPORTS
WITH group_transactions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'group_transactions') }}
),
-- LOGIC
group_transactions_rename AS (
SELECT
group_transactions.id,
group_transactions.group_id,
group_transactions.transaction_id,
current_timestamp() AS _dbt_ran_at
FROM group_transactions
),
-- FINAL
final__base_exchange__group_transactions AS (
SELECT
id,
group_id,
transaction_id,
_dbt_ran_at
FROM group_transactions_rename
)
SELECT * FROM final__base_exchange__group_transactions
@@ -0,0 +1,55 @@
-- IMPORTS
WITH groups AS (
SELECT * FROM {{ source('src_exchange_mysql', 'groups') }}
),
-- LOGIC
groups_rename AS (
SELECT
groups.id,
groups.reference AS marking_id,
groups.issuer AS issued_by,
groups.receiver AS received_by,
groups.amount AS base_amount,
groups.original_amount AS quote_amount,
groups.currency_id AS base_currency_id,
groups.original_currency_id AS quote_currency_id,
groups.currency_rate AS base_to_quote_currency_exchange_rate,
groups.tax AS base_tax,
groups.service_charge AS base_service_charge,
groups.status,
groups.created_at,
groups.updated_at,
current_timestamp() AS _dbt_ran_at
FROM groups
),
-- FINAL
final__base_exchange__groups AS (
SELECT
id,
marking_id,
issued_by,
received_by,
base_amount,
quote_amount,
base_currency_id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
status,
created_at,
updated_at,
_dbt_ran_at
FROM groups_rename
)
SELECT * FROM final__base_exchange__groups
@@ -0,0 +1,41 @@
-- IMPORTS
WITH jobs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'jobs') }}
),
-- LOGIC
jobs_rename AS (
SELECT
jobs.id,
jobs.queue,
jobs.payload,
jobs.attempts,
TO_TIMESTAMP(jobs.reserved_at) AS reserved_at,
TO_TIMESTAMP(jobs.available_at) AS available_at,
TO_TIMESTAMP(jobs.created_at) AS created_at,
current_timestamp() AS _dbt_ran_at
FROM jobs
),
-- FINAL
final__base_exchange__jobs AS (
SELECT
id,
queue,
payload,
attempts,
reserved_at,
available_at,
created_at,
_dbt_ran_at
FROM jobs_rename
)
SELECT * FROM final__base_exchange__jobs
@@ -0,0 +1,33 @@
-- IMPORTS
WITH migrations AS (
SELECT * FROM {{ source('src_exchange_mysql', 'migrations') }}
),
-- LOGIC
migrations_rename AS (
SELECT
migrations.id,
migrations.migration,
migrations.batch,
current_timestamp() AS _dbt_ran_at
FROM migrations
),
-- FINAL
final__base_exchange__migrations AS (
SELECT
id,
migration,
batch,
_dbt_ran_at
FROM migrations_rename
)
SELECT * FROM final__base_exchange__migrations
@@ -0,0 +1,34 @@
-- IMPORTS
WITH model_has_roles AS (
SELECT * FROM {{ source('src_exchange_mysql', 'model_has_roles') }}
),
-- LOGIC
migrations_rename AS (
SELECT
model_has_roles.role_id,
model_has_roles.model_type,
model_has_roles.model_id,
current_timestamp() AS _dbt_ran_at
FROM model_has_roles
),
-- FINAL
final__base_exchange__model_has_roles AS (
SELECT
role_id,
model_type,
model_id,
_dbt_ran_at
FROM migrations_rename
)
SELECT * FROM final__base_exchange__model_has_roles
@@ -0,0 +1,53 @@
-- IMPORTS
WITH notifications AS (
SELECT * FROM {{ source('src_exchange_mysql', 'notifications') }}
),
-- LOGIC
notifications_rename AS (
SELECT
notifications.id,
notifications.title,
notifications.description,
notifications.subject_type,
notifications.subject_id,
notifications.target_type,
notifications.target_id,
notifications.causer_type,
notifications.causer_id,
notifications.status,
notifications.deleted_at,
notifications.created_at,
notifications.updated_at,
current_timestamp() AS _dbt_ran_at
FROM notifications
),
-- FINAL
final__base_exchange__notifications AS (
SELECT
id,
title,
description,
subject_type,
subject_id,
target_type,
target_id,
causer_type,
causer_id,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM notifications_rename
)
SELECT * FROM final__base_exchange__notifications
@@ -0,0 +1,42 @@
-- IMPORTS
WITH password_resets AS (
SELECT * FROM {{ source('src_exchange_mysql', 'password_resets') }}
),
-- LOGIC
password_resets_rename AS (
SELECT
password_resets.id,
password_resets.user_id,
password_resets.token,
password_resets.is_expired,
password_resets.is_complete,
password_resets.created_at,
password_resets.updated_at,
current_timestamp() AS _dbt_ran_at
FROM password_resets
),
-- FINAL
final__base_exchange__password_resets AS (
SELECT
id,
user_id,
token,
is_expired,
is_complete,
created_at,
updated_at,
_dbt_ran_at
FROM password_resets_rename
)
SELECT * FROM final__base_exchange__password_resets
@@ -0,0 +1,38 @@
-- IMPORTS
WITH permissions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'permissions') }}
),
-- LOGIC
permissions_rename AS (
SELECT
permissions.id,
permissions.name,
permissions.guard_name,
permissions.created_at,
permissions.updated_at,
current_timestamp() AS _dbt_ran_at
FROM permissions
),
-- FINAL
final__base_exchange__permissions AS (
SELECT
id,
name,
guard_name,
created_at,
updated_at,
_dbt_ran_at
FROM permissions_rename
)
SELECT * FROM final__base_exchange__permissions
@@ -0,0 +1,32 @@
-- IMPORTS
WITH role_has_permissions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'role_has_permissions') }}
),
-- LOGIC
role_has_permissions_rename AS (
SELECT
role_has_permissions.permission_id,
role_has_permissions.role_id,
current_timestamp() AS _dbt_ran_at
FROM role_has_permissions
),
-- FINAL
final__exchange_base__role_has_permissions AS (
SELECT
permission_id,
role_id,
_dbt_ran_at
FROM role_has_permissions_rename
)
SELECT * FROM final__exchange_base__role_has_permissions
@@ -0,0 +1,40 @@
-- IMPORTS
WITH roles AS (
SELECT * FROM {{ source('src_exchange_mysql', 'roles') }}
),
-- LOGIC
roles_rename AS (
SELECT
roles.id,
roles.name,
roles.guard_name,
roles.type,
roles.created_at,
roles.updated_at,
current_timestamp() AS _dbt_ran_at
FROM roles
),
-- FINAL
final__base_exchange__roles AS (
SELECT
id,
name,
guard_name,
type,
created_at,
updated_at,
_dbt_ran_at
FROM roles_rename
)
SELECT * FROM final__base_exchange__roles
@@ -0,0 +1,43 @@
-- IMPORTS
WITH segment_companies AS (
SELECT * FROM {{ ref('snap_exchange__segment_companies') }}
),
-- LOGIC
segment_companies_rename AS (
SELECT
segment_companies.concat_id,
segment_companies.segment_id,
segment_companies.company_id,
segment_companies.deleted_at,
segment_companies.created_at,
segment_companies.updated_at,
segment_companies.dbt_valid_from,
segment_companies.dbt_valid_to,
current_timestamp() AS _dbt_ran_at
FROM segment_companies
),
-- FINAL
final__base_exchange__segment_companies AS (
SELECT
concat_id,
segment_id,
company_id,
deleted_at,
created_at,
updated_at,
dbt_valid_from,
dbt_valid_to,
_dbt_ran_at
FROM segment_companies_rename
)
SELECT * FROM final__base_exchange__segment_companies
@@ -0,0 +1,43 @@
-- IMPORTS
WITH segment_constants AS (
SELECT * FROM {{ source('src_exchange_mysql', 'segment_constants') }}
),
-- LOGIC
segment_constants_rename AS (
SELECT
segment_constants.id,
segment_constants.segment_id,
segment_constants.name,
segment_constants.reference AS marking_id,
segment_constants.detail,
segment_constants.deleted_at,
segment_constants.created_at,
segment_constants.updated_at,
current_timestamp() AS _dbt_ran_at
FROM segment_constants
),
-- FINAL
final_base_exchange__segment_constants AS (
SELECT
id,
segment_id,
name,
marking_id,
detail,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM segment_constants_rename
)
SELECT * FROM final_base_exchange__segment_constants
@@ -0,0 +1,40 @@
-- IMPORTS
WITH segments AS (
SELECT * FROM {{ source('src_exchange_mysql', 'segments') }}
),
-- LOGIC
segments_rename AS (
SELECT
segments.id,
segments.name,
segments.type,
segments.status,
segments.deleted_at,
segments.created_at,
segments.updated_at,
current_timestamp() AS _dbt_ran_at
FROM segments
),
-- FINAL
final__base_exchange__segments AS (
SELECT
id,
name,
type,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM segments_rename
)
SELECT * FROM final__base_exchange__segments
@@ -0,0 +1,39 @@
-- IMPORTS
WITH service_types AS (
SELECT * FROM {{ source('src_exchange_mysql', 'service_types') }}
),
-- LOGIC
service_types_rename AS (
SELECT
service_types.id,
service_types.name,
service_types.status,
service_types.deleted_at,
service_types.created_at,
service_types.updated_at,
CURRENT_TIMESTAMP() AS _dbt_ran_at
FROM service_types
),
-- FINAL
final__base_exchange__service_types AS (
SELECT
id,
name,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM service_types_rename
)
SELECT * FROM final__base_exchange__service_types
@@ -0,0 +1,41 @@
-- IMPORTS
WITH states AS (
SELECT * FROM {{ source('src_exchange_mysql', 'states') }}
),
-- LOGIC
states_rename AS (
SELECT
states.id,
states.country_id,
states.name,
states.status,
states.deleted_at,
states.created_at,
states.updated_at,
current_timestamp() AS _dbt_ran_at
FROM states
),
-- FINAL
final__base_exchange__states AS (
SELECT
id,
country_id,
name,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM states_rename
)
SELECT * FROM final__base_exchange__states
@@ -0,0 +1,48 @@
-- IMPORTS
WITH transaction_details AS (
SELECT * FROM {{ source('src_exchange_mysql', 'transaction_detail') }}
),
-- LOGIC
transaction_details_rename AS (
SELECT
transaction_details.id,
transaction_details.transaction_id,
transaction_details.product_code,
transaction_details.product_name,
transaction_details.quantity,
transaction_details.price,
transaction_details.amount,
transaction_details.deleted_at,
transaction_details.created_at,
transaction_details.updated_at,
current_timestamp() AS _dbt_ran_at
FROM transaction_details
),
-- FINAL
final__base_exchange__transaction_details AS (
SELECT
id,
transaction_id,
product_code,
product_name,
quantity,
price,
amount,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM transaction_details_rename
)
SELECT * FROM final__base_exchange__transaction_details
@@ -0,0 +1,73 @@
-- IMPORTS
WITH transaction_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'transaction_logs') }}
),
-- LOGIC
transaction_logs_rename AS (
SELECT
transaction_logs.id,
transaction_logs.transaction_id,
transaction_logs.owner_type,
transaction_logs.owner_id,
transaction_logs.type AS transaction_type,
transaction_logs.issuer AS issued_by,
transaction_logs.receiver AS received_by,
transaction_logs.recipient_bank_account_id,
transaction_logs.payment_method,
transaction_logs.payment_reference,
transaction_logs.bill_no,
transaction_logs.amount AS base_value,
transaction_logs.original_amount AS quote_value,
transaction_logs.currency_id AS base_currency_id,
transaction_logs.original_currency_id AS quote_currency_id,
transaction_logs.currency_rate AS base_to_quote_currency_exchange_rate,
transaction_logs.tax AS base_tax,
transaction_logs.service_charge AS base_service_charge,
transaction_logs.expires_on AS expired_at,
transaction_logs.status,
transaction_logs.deleted_at,
transaction_logs.created_at,
transaction_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM transaction_logs
),
-- FINAL
final__base_exchange__transaction_logs AS (
SELECT
id,
transaction_id,
owner_type,
owner_id,
transaction_type,
issued_by,
received_by,
recipient_bank_account_id,
payment_method,
payment_reference,
bill_no,
base_value,
quote_value,
base_currency_id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
expired_at,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM transaction_logs_rename
)
SELECT * FROM final__base_exchange__transaction_logs
@@ -0,0 +1,71 @@
-- IMPORTS
WITH transactions AS (
SELECT * FROM {{ source('src_exchange_mysql', 'transactions') }}
),
-- LOGIC
transactions_rename AS (
SELECT
transactions.id,
transactions.owner_type,
transactions.owner_id,
transactions.type AS transaction_type,
transactions.issuer AS issued_by,
transactions.receiver AS received_by,
transactions.recipient_bank_account_id,
transactions.payment_method,
transactions.payment_reference,
transactions.bill_no,
transactions.amount AS base_value,
transactions.original_amount AS quote_value,
transactions.currency_id AS base_currency_id,
transactions.original_currency_id AS quote_currency_id,
transactions.currency_rate AS base_to_quote_currency_exchange_rate,
transactions.tax AS base_tax,
transactions.service_charge AS base_service_charge,
transactions.expires_on AS expired_at,
transactions.status,
transactions.deleted_at,
transactions.created_at,
transactions.updated_at,
current_timestamp() AS _dbt_ran_at
FROM transactions
),
-- FINAL
final__base_exchange__transactions AS (
SELECT
id,
owner_type,
owner_id,
transaction_type,
issued_by,
received_by,
recipient_bank_account_id,
payment_method,
payment_reference,
bill_no,
base_value,
quote_value,
base_currency_id,
quote_currency_id,
base_to_quote_currency_exchange_rate,
base_tax,
base_service_charge,
expired_at,
status,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM transactions_rename
)
SELECT * FROM final__base_exchange__transactions
@@ -0,0 +1,43 @@
-- IMPORTS
WITH user_email_verifications AS (
SELECT * FROM {{ source('src_exchange_mysql', 'user_email_verifications') }}
),
-- LOGIC
user_email_verifications_rename AS (
SELECT
user_email_verifications.id,
user_email_verifications.email,
user_email_verifications.token,
user_email_verifications.is_complete AS is_completed,
user_email_verifications.is_active AS is_activated,
user_email_verifications.is_sent,
user_email_verifications.created_at,
user_email_verifications.updated_at,
current_timestamp() AS _dbt_ran_at
FROM user_email_verifications
),
-- FINAL
final__base_exchange__user_email_verifications AS (
SELECT
id,
email,
token,
is_completed,
is_activated,
is_sent,
created_at,
updated_at,
_dbt_ran_at
FROM user_email_verifications_rename
)
SELECT * FROM final__base_exchange__user_email_verifications
@@ -0,0 +1,47 @@
-- IMPORTS
WITH users AS (
SELECT * FROM {{ source('src_exchange_mysql', 'users') }}
),
-- LOGIC
users_rename AS (
SELECT
users.id,
users.name,
users.email,
users.type AS user_type,
users.status,
users.remember_token,
users.active_at,
users.deleted_at,
users.created_at,
users.updated_at,
current_timestamp() AS _dbt_ran_at
FROM users
),
-- FINAL
final__base_exchange__users AS (
SELECT
id,
name,
email,
user_type,
status,
remember_token,
active_at,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM users_rename
)
SELECT * FROM final__base_exchange__users
@@ -0,0 +1,47 @@
-- IMPORTS
WITH wallet_logs AS (
SELECT * FROM {{ source('src_exchange_mysql', 'wallet_logs') }}
),
-- LOGIC
wallet_logs_rename AS (
SELECT
wallet_logs.id,
wallet_logs.wallet_id,
wallet_logs.owner_type,
wallet_logs.owner_id,
wallet_logs.code,
wallet_logs.currency_id,
wallet_logs.amount,
wallet_logs.deleted_at,
wallet_logs.created_at,
wallet_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM wallet_logs
),
-- FINAL
final__base_exchange__wallet_logs AS (
SELECT
id,
wallet_id,
owner_type,
owner_id,
code,
currency_id,
amount,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM wallet_logs_rename
)
SELECT * FROM final__base_exchange__wallet_logs
@@ -0,0 +1,45 @@
-- IMPORTS
WITH wallets AS (
SELECT * FROM {{ source('src_exchange_mysql', 'wallets') }}
),
-- LOGIC
wallets_rename AS (
SELECT
wallets.id,
wallets.owner_type,
wallets.owner_id,
wallets.code,
wallets.currency_id,
wallets.amount,
wallets.deleted_at,
wallets.created_at,
wallets.updated_at,
current_timestamp() AS _dbt_ran_at
FROM wallets
),
-- FINAL
final__base_exchange__wallets AS (
SELECT
id,
owner_type,
owner_id,
code,
currency_id,
amount,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM wallets_rename
)
SELECT * FROM final__base_exchange__wallets
@@ -0,0 +1,112 @@
-- IMPORTS
WITH booking_logs AS (
SELECT * FROM {{ ref('stg_exchange__booking_logs') }}
),
currency_rate_logs AS (
SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }}
),
-- LOGIC
booking_logs_calculate_value_join_currency_rate_logs AS (
SELECT
booking_logs.id,
booking_logs.booking_id,
booking_logs.company_id,
booking_logs.user_id,
booking_logs.marking_id,
booking_logs.service_type,
booking_logs.service_type_name,
booking_logs.bank_id,
booking_logs.fix_value,
booking_logs.fix_currency_id,
booking_logs.fix_currency_name,
booking_logs.fix_currency_short_code,
currency_rate_logs.payment_method AS estimate_payment_method,
currency_rate_logs.payment_method_name AS estimate_payment_method_name,
currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate,
booking_logs.quote_currency_id,
booking_logs.quote_currency_name,
booking_logs.quote_currency_short_code,
IFF(booking_logs.quote_currency_id = booking_logs.fix_currency_id,
fix_value,
ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value,
booking_logs.base_currency_id,
booking_logs.base_currency_name,
booking_logs.base_currency_short_code,
IFF(booking_logs.base_currency_id = booking_logs.fix_currency_id,
fix_value,
ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value,
-- If we have quote & base value, both of them are not contain 1 (MYR), then need to redo this function
CASE
WHEN (booking_logs.base_currency_id = '1')
THEN ROUND(estimate_base_value,2)
WHEN (booking_logs.quote_currency_id = '1')
THEN ROUND(estimate_quote_value,2)
ELSE
NULL
END AS estimate_value_rm,
booking_logs.status,
booking_logs.status_name,
booking_logs.deleted_at,
booking_logs.created_at,
booking_logs.updated_at,
current_timestamp() AS _dbt_ran_at
FROM booking_logs
LEFT JOIN currency_rate_logs
ON (booking_logs.created_at >= currency_rate_logs.created_date_from)
AND (booking_logs.created_at < currency_rate_logs.created_date_to)
AND (booking_logs.service_type = currency_rate_logs.service_type)
AND (booking_logs.quote_currency_id = currency_rate_logs.quote_currency_id)
AND (currency_rate_logs.payment_method = '1')
),
-- FINAL
final__int_booking_logs_get_estimate_rates__exchange AS (
SELECT
id,
booking_id,
company_id,
user_id,
marking_id,
service_type,
service_type_name,
bank_id,
fix_value,
fix_currency_id,
fix_currency_name,
fix_currency_short_code,
estimate_payment_method,
estimate_payment_method_name,
estimate_base_to_quote_currency_exchange_rate,
base_currency_id,
base_currency_name,
base_currency_short_code,
estimate_base_value,
estimate_value_rm,
quote_currency_id,
quote_currency_name,
quote_currency_short_code,
estimate_quote_value,
status,
status_name,
deleted_at,
created_at,
updated_at,
_dbt_ran_at
FROM booking_logs_calculate_value_join_currency_rate_logs
)
SELECT * FROM final__int_booking_logs_get_estimate_rates__exchange
@@ -0,0 +1,82 @@
-- IMPORTS
WITH bookings AS (
SELECT * FROM {{ ref('stg_exchange__bookings') }}
),
currency_rate_logs AS (
SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }}
),
-- LOGIC
bookings_calculate_value_join_currency_rate_logs AS (
SELECT
bookings.id,
bookings.company_id,
bookings.user_id,
bookings.marking_id,
bookings.service_type,
bookings.service_type_name,
bookings.bank_id,
bookings.fix_value,
bookings.fix_currency_id,
bookings.fix_currency_name,
bookings.fix_currency_short_code,
currency_rate_logs.payment_method AS estimate_payment_method,
currency_rate_logs.payment_method_name AS estimate_payment_method_name,
currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate,
bookings.quote_currency_id,
bookings.quote_currency_name,
bookings.quote_currency_short_code,
IFF(bookings.quote_currency_id = bookings.fix_currency_id,
fix_value,
ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value,
bookings.base_currency_id,
bookings.base_currency_name,
bookings.base_currency_short_code,
IFF(bookings.base_currency_id = bookings.fix_currency_id,
fix_value,
ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value,
-- If we have quote & base value, both of them are not contain 1 (MYR), then need to redo this function
CASE
WHEN (bookings.base_currency_id = '1')
THEN ROUND(estimate_base_value,2)
WHEN (bookings.quote_currency_id = '1')
THEN ROUND(estimate_quote_value,2)
ELSE
NULL
END AS estimate_value_rm,
bookings.status,
bookings.status_name,
bookings.booking_deleted_datetime,
bookings.booking_created_datetime,
bookings.booking_updated_datetime,
current_timestamp() AS _dbt_ran_at
FROM bookings
LEFT JOIN currency_rate_logs
ON (bookings.booking_created_datetime >= currency_rate_logs.created_date_from)
AND (bookings.booking_created_datetime < currency_rate_logs.created_date_to)
AND (bookings.service_type = currency_rate_logs.service_type)
AND (bookings.quote_currency_id = currency_rate_logs.quote_currency_id)
AND (currency_rate_logs.payment_method = '1')
),
-- FINAL
final__int_bookings_get_estimate_rates__exchange AS (
SELECT
*
FROM bookings_calculate_value_join_currency_rate_logs
)
SELECT * FROM final__int_bookings_get_estimate_rates__exchange
@@ -0,0 +1,81 @@
-- Import
WITH companies AS (
SELECT * FROM {{ ref('stg_exchange__companies') }}
),
segments AS (
SELECT * FROM {{ ref('stg_exchange__segments') }}
),
companies_and_segments_relations AS (
SELECT * FROM {{ ref('stg_exchange__companies_and_segments_relations') }}
),
-- LOGIC
get_is_migrated_companies AS (
SELECT
segment_id,
company_id
FROM companies_and_segments_relations
WHERE
segment_id = 5
),
-- Because we do not have timestamp to get maximum segment_id
get_segment_row_created_sequences AS (
SELECT
*,
ROW_NUMBER() OVER (ORDER BY NULL) AS row_created_sequence
FROM companies_and_segments_relations
WHERE
deleted_at IS NULL
AND
segment_id IN (1,2,3,4)
AND
dbt_valid_to IS NULL
),
get_company_latest_segments AS (
SELECT
*,
row_number() OVER
(PARTITION BY company_id
ORDER BY row_created_sequence DESC) AS is_latest_segment
FROM get_segment_row_created_sequences
QUALIFY
is_latest_segment = 1
),
-- FINAL
final__int_exchange__companies_level_segment AS (
SELECT
companies.*,
get_company_latest_segments.segment_id,
segments.name AS segment_name,
IFF(get_is_migrated_companies.company_id IS NOT NULL, 1, 0) AS is_migrated_company
FROM companies
LEFT JOIN get_company_latest_segments
ON (companies.id = get_company_latest_segments.company_id)
LEFT JOIN segments
ON (get_company_latest_segments.segment_id = segments.id)
LEFT JOIN get_is_migrated_companies
ON (companies.id = get_is_migrated_companies.company_id)
)
SELECT * FROM final__int_exchange__companies_level_segment
-- select * from get_is_migrated_company
@@ -0,0 +1,45 @@
-- IMPORTS
WITH user_email_verifications AS (
SELECT * FROM {{ ref('stg_exchange__user_email_verifications')}}
),
-- LOGIC
dedupe_user_email_verifications AS (
SELECT
user_id,
-- Function below search for '1' in is_complete groupconcat string, if found then true, if null than null, else false
IFF(CONTAINS( LISTAGG(is_completed, ',') WITHIN GROUP(ORDER BY is_completed), '1'),'1','0') AS is_completed,
IFF(CONTAINS( LISTAGG(is_activated, ',') WITHIN GROUP(ORDER BY is_activated), '1'),'1','0') AS is_activated,
IFF(CONTAINS( LISTAGG(is_sent, ',') WITHIN GROUP(ORDER BY is_sent), '1'), '1', '0') AS is_sent,
COUNT(*) AS email_sent,
MIN(created_at) AS first_email_created_at,
MAX(created_at) AS last_email_created_at,
current_timestamp() AS _dbt_ran_at
FROM user_email_verifications
GROUP BY
user_id
),
-- FINAL
final__int_exchange__dedupe_user_email_verifications AS (
SELECT
user_id,
is_completed,
is_activated,
is_sent,
email_sent,
first_email_created_at,
last_email_created_at,
_dbt_ran_at
FROM
dedupe_user_email_verifications
)
SELECT * FROM final__int_exchange__dedupe_user_email_verifications
@@ -0,0 +1,38 @@
--IMPORT
WITH contacts AS (
SELECT * FROM {{ ref('stg_exchange__contacts') }}
),
--LOGIC
get_latest_company_contacts AS (
SELECT
*,
row_number() OVER
(PARTITION BY company_id
ORDER BY contact_created_datetime DESC) AS last_row_number
FROM contacts
QUALIFY
last_row_number = 1
),
--FINAL
final__int_exchange__latest_company_contacts AS (
SELECT
id,
company_id,
country_id,
country_name,
reference,
phone,
email,
wechat_id,
contact_deleted_datetime,
contact_created_datetime,
contact_updated_datetime
FROM get_latest_company_contacts
)
SELECT * FROM final__int_exchange__latest_company_contacts
@@ -0,0 +1,72 @@
-- IMPORT
WITH transaction_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_costs') }}
),
-- LOGIC
duplicate_id_need_remove AS (
SELECT
ID,
TRANSACTION_ORDER_ID,
BASE_VALUE,
QUOTE_VALUE,
BASE_TO_QUOTE_CURRENCY_EXCHANGE_RATE,
STATUS_NAME,
CREATED_AT,
UPDATED_AT,
row_number() OVER
(PARTITION BY transaction_costs.transaction_order_id
ORDER BY transaction_costs.updated_at DESC) AS row_index
FROM
transaction_costs
WHERE
deleted_at IS NULL
QUALIFY
row_index = 2
),
-- duplicate_id_need_remove AS (
-- SELECT
-- id
-- FROM mark_duplicate_is_not_deleted_id
-- WHERE
-- row_index = 2
-- ),
removed_duplicate_id_table AS (
SELECT
transaction_costs.*
FROM transaction_costs
LEFT JOIN duplicate_id_need_remove
ON (transaction_costs.id = duplicate_id_need_remove.id)
WHERE
duplicate_id_need_remove.id IS NULL
),
--FINAL
final__int_exchange__transaction_costs_remove_duplicate_row AS (
SELECT * FROM removed_duplicate_id_table
)
SELECT * FROM final__int_exchange__transaction_costs_remove_duplicate_row
-- SELECT * FROM duplicate_transaction_order_id_which_not_deleted
-- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at
-- SELECT * FROM duplicate_id_need_remove
-- SELECT * FROM removed_duplicate_id_table WHERE deleted_at is not null
-- SELECT * FROM mark_duplicate_is_not_deleted_id WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at
-- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at
@@ -0,0 +1,28 @@
-- IMPORTS
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_log_orders_remove_undo_status') }}
),
bookings AS (
SELECT * FROM {{ ref('stg_exchange__bookings') }}
),
-- LOGICS
transaction_log_orders_join_bookings AS (
SELECT
transaction_log_orders.*,
bookings.user_id
FROM transaction_log_orders
LEFT JOIN bookings
ON (transaction_log_orders.booking_id = bookings.id)
),
-- FINAL
final__int_exchange__transaction_log_orders_get_user_company_ids AS (
SELECT * FROM transaction_log_orders_join_bookings
)
SELECT * FROM final__int_exchange__transaction_log_orders_get_user_company_ids
@@ -0,0 +1,26 @@
-- IMPORT
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('stg_exchange__transaction_log_orders') }}
),
-- LOGIC
transaction_log_orders_mark_undo_status AS (
SELECT
*
FROM transaction_log_orders
QUALIFY
row_number() OVER
(PARTITION BY transaction_log_orders.status, transaction_log_orders.transaction_order_id
ORDER BY transaction_log_orders.updated_at DESC) = 1
),
-- FINAL
final__int_exchange__transaction_log_orders_remove_undo_status AS (
SELECT * FROM transaction_log_orders_mark_undo_status
)
SELECT * FROM final__int_exchange__transaction_log_orders_remove_undo_status
@@ -0,0 +1,30 @@
-- IMPORTS
WITH transaction_orders AS (
SELECT * FROM {{ ref('stg_exchange__transaction_orders') }}
),
bookings AS (
SELECT * FROM {{ ref('stg_exchange__bookings') }}
),
-- LOGICS
transaction_orders_join_bookings AS (
SELECT
transaction_orders.*,
bookings.user_id
FROM transaction_orders
LEFT JOIN bookings
ON (transaction_orders.booking_id = bookings.id)
),
-- FINAL
final__int_exchange__transaction_orders_get_user_company_ids AS (
SELECT * FROM transaction_orders_join_bookings
)
SELECT * FROM final__int_exchange__transaction_orders_get_user_company_ids
@@ -0,0 +1,234 @@
-- IMPORT
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_log_orders') }}
),
latest_transaction_orders AS (
SELECT * FROM {{ ref('fct_exchange__transaction_orders') }}
),
booking_logs AS (
SELECT * FROM {{ ref('fct_exchange__booking_logs') }}
),
latest_bookings AS (
SELECT * FROM {{ ref('fct_exchange__bookings') }}
),
latest_transaction_costs AS (
SELECT * FROM {{ ref('fct_exchange__transaction_costs') }}
),
transaction_log_costs AS (
SELECT * FROM {{ ref('fct_exchange__transaction_log_costs') }}
),
companies AS (
SELECT * FROM {{ ref('dim_exchange__companies') }}
),
dates AS (
SELECT * FROM {{ ref('dim_exchange__dates') }}
),
--LOGIC
transaction_order_logs_status_datetime AS (
SELECT
transaction_order_id,
MIN(created_at) AS created_datetime,
MAX(CASE WHEN status = '0' THEN updated_at END) AS pending_submission_datetime,
MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime,
MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime,
MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime,
MAX(CASE WHEN status = '4' THEN updated_at END) AS rejected_datetime,
MAX(CASE WHEN status = '5' THEN updated_at END) AS suspended_datetime
FROM transaction_log_orders
GROUP BY
transaction_order_id
),
cost_logs_status_datetime AS (
SELECT
transaction_order_id,
MIN(created_at) AS created_datetime,
MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime,
MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime,
MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime
FROM transaction_log_costs
GROUP BY
transaction_order_id
),
latest_transaction_costs_remove_deleted AS (
SELECT
*
FROM
latest_transaction_costs
WHERE
deleted_at IS NULL
),
previous_order_date_row_rank AS (
SELECT
*,
row_number() OVER
(PARTITION BY latest_transaction_orders.company_id
ORDER BY latest_transaction_orders.created_at) AS row_rank_index
FROM
latest_transaction_orders
),
get_previous_order_date AS (
SELECT
previous_order_date_row_rank.id,
previous_order_date_row_rank_clone.created_at AS previous_order_created_datetime
FROM previous_order_date_row_rank
LEFT JOIN previous_order_date_row_rank AS previous_order_date_row_rank_clone
ON previous_order_date_row_rank.row_rank_index = previous_order_date_row_rank_clone.row_rank_index + 1
AND previous_order_date_row_rank.company_id = previous_order_date_row_rank_clone.company_id
),
--FINAL
final__rep_exchange__daily_transactions AS (
SELECT
latest_transaction_orders.id,
latest_transaction_orders.booking_id,
latest_bookings.marking_id AS booking_marking_id,
latest_bookings.service_type,
latest_bookings.service_type_name,
latest_bookings.booking_created_datetime,
latest_transaction_orders.seller_company_id,
latest_transaction_orders.bank_id,
latest_transaction_orders.user_id,
latest_transaction_orders.company_id,
companies.marking_id AS company_marking_id,
companies.company_type,
companies.company_type_name,
companies.business_type AS company_business_type,
companies.business_type_name AS company_business_type_name,
companies.segment_id AS company_segment_id,
companies.segment_name AS company_segment_name,
companies.is_migrated_company AS company_is_migrated_company,
companies.country_id AS company_country_id,
companies.country_name AS company_country_name,
companies.state_id AS company_state_id,
companies.state_name AS company_state_name,
companies.district_id AS company_district_id,
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.wallet_id AS company_wallet_id,
companies.have_wallet AS company_have_wallet,
-- i stay here
latest_bookings.is_first_time_user AS is_first_time_booking_user,
latest_bookings.is_first_time_company AS is_first_time_booking_company,
latest_transaction_orders.is_first_time_user,
latest_transaction_orders.is_first_time_company,
latest_transaction_orders.status,
latest_transaction_orders.status_name,
latest_transaction_orders.transaction_type,
latest_transaction_orders.transaction_type_name,
latest_transaction_orders.payment_method,
latest_transaction_orders.payment_method_name,
latest_transaction_orders.base_currency_id,
latest_transaction_orders.base_currency_name,
latest_transaction_orders.quote_currency_id,
latest_transaction_orders.quote_currency_name,
latest_transaction_orders.base_to_quote_currency_exchange_rate,
latest_transaction_orders.base_value,
latest_transaction_orders.quote_value,
latest_transaction_orders.transaction_value_rm,
latest_transaction_orders.transaction_base_tax,
latest_transaction_orders.transaction_tax_rm,
latest_transaction_orders.base_service_charge,
latest_transaction_orders.transaction_service_charge_rm,
get_previous_order_date.previous_order_created_datetime AS previous_order_created_datetime,
transaction_order_logs_status_datetime.created_datetime AS order_created_datetime,
order_created_dates.date_actual AS order_created_date,
order_created_dates.first_day_of_week AS order_created_week,
transaction_order_logs_status_datetime.pending_submission_datetime AS order_pending_submission_datetime,
transaction_order_logs_status_datetime.pending_verification_datetime AS order_pending_verification_datetime,
transaction_order_logs_status_datetime.approved_datetime AS order_approved_datetime,
transaction_order_logs_status_datetime.completed_datetime AS order_completed_datetime,
transaction_order_logs_status_datetime.rejected_datetime AS order_rejected_datetime,
transaction_order_logs_status_datetime.suspended_datetime AS order_suspended_datetime,
latest_transaction_costs_remove_deleted.id AS cost_id,
latest_transaction_costs_remove_deleted.supplier_company_id AS cost_supplier_company_id,
latest_transaction_costs_remove_deleted.status AS cost_status,
latest_transaction_costs_remove_deleted.status_name AS cost_status_name,
latest_transaction_costs_remove_deleted.base_currency_id AS cost_base_currency_id,
latest_transaction_costs_remove_deleted.base_currency_name AS cost_base_currency_name,
latest_transaction_costs_remove_deleted.quote_currency_id AS cost_quote_currency_id,
latest_transaction_costs_remove_deleted.quote_currency_name AS cost_quote_currency_name,
latest_transaction_costs_remove_deleted.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate,
latest_transaction_costs_remove_deleted.base_value AS cost_base_value,
latest_transaction_costs_remove_deleted.quote_value AS cost_quote_value,
latest_transaction_costs_remove_deleted.transaction_value_rm AS cost_transaction_value_rm,
latest_transaction_costs_remove_deleted.transaction_base_tax AS cost_transaction_base_tax,
latest_transaction_costs_remove_deleted.transaction_tax_rm AS cost_transaction_tax_rm,
latest_transaction_costs_remove_deleted.base_service_charge AS cost_base_service_charge,
latest_transaction_costs_remove_deleted.transaction_service_charge_rm AS cost_transaction_service_charge_rm,
cost_logs_status_datetime.created_datetime AS cost_created_datetime,
cost_logs_status_datetime.pending_verification_datetime AS cost_pending_verification_datetime,
cost_logs_status_datetime.approved_datetime AS cost_approved_datetime,
cost_logs_status_datetime.completed_datetime AS cost_completed_datetime
FROM latest_transaction_orders
LEFT JOIN latest_bookings
ON (latest_transaction_orders.booking_id = latest_bookings.id)
LEFT JOIN transaction_order_logs_status_datetime
ON (latest_transaction_orders.id = transaction_order_logs_status_datetime.transaction_order_id)
LEFT JOIN latest_transaction_costs_remove_deleted
ON (latest_transaction_orders.id = latest_transaction_costs_remove_deleted.transaction_order_id)
LEFT JOIN cost_logs_status_datetime
ON (latest_transaction_orders.id = cost_logs_status_datetime.transaction_order_id)
LEFT JOIN companies
ON (latest_transaction_orders.company_id = companies.id)
LEFT JOIN get_previous_order_date
ON(latest_transaction_orders.id = get_previous_order_date.id)
LEFT JOIN dates AS order_created_dates
ON (date(transaction_order_logs_status_datetime.created_datetime) = order_created_dates.date_day )
ORDER BY latest_transaction_orders.updated_at ASC
)
SELECT * FROM final__rep_exchange__daily_transactions
-- @YAM: not unique order id
--(2007, 40808, 2013, 40920, 41101, 125169, 18604, 18552, 18550, 35795, 117444, 72332, 40798, 51174, 18553, 2010, 66195, 89413, 1678, 71516, 5673, 78366, 66208, 18546, 1672, 1684, 18579, 66202, 132074, 18554, 43249, 40804, 86259, 4881, 2311, 103084, 18549, 41097, 106321, 17634, 66207, 18582, 1661, 63970) and so on....
-- (OMAIR)
-- SELECT booking_id, count(*) FROM booking_logs WHERE status= 3 GROUP BY booking_id ORDER BY count(*) desc
-- SELECT booking_id, company_id, user_id, fix_value, status_name, updated_at FROM booking_logs WHERE booking_id IN (18901,18618,22960) ORDER BY booking_id, updated_at
-- SELECT * FROM booking_logs WHERE booking_id = 22173 ORDER BY updated_at
@@ -0,0 +1,211 @@
version: 2
models:
- name: rep_exchange__daily_orders
columns:
- name: id
description: transaction orders id
tests:
- unique
- not_null
- name: booking_id
description: transaction orders id
tests:
- not_null
- relationships:
to: ref('base_exchange__bookings')
field: id
- name: service_type
description: type of service
tests:
- not_null
- name: service_type_name
description: name of type of service
tests:
- not_null
- name: booking_created_datetime
description: date of boking created
tests:
- not_null
- name: seller_company_id
description: seller company id
tests:
- not_null
- name: bank_id
description: bank id
tests:
- not_null
- name: company_id
description: user id
tests:
- not_null
- name: company_marking_id
description: company marking id
tests:
- relationships:
to: ref('base_exchange__companies')
field: marking_id
- name: company_type
description: type of company
tests:
- not_null
- name: company_type_name
description: name type of company
tests:
- accepted_values:
values:
- PERSONAL
- CORPORATE
- name: company_business_type
description: company business type
tests:
- not_null
- name: company_business_type_name
description: company business type name
tests:
- accepted_values:
values:
- IMPORTER
- name: company_segment_id
description: company segment id
tests:
- not_null
- name: company_segment_name
description: company segment name
tests:
- accepted_values:
values:
- PLATINIUM MEMBER
- GOLD MEMBER
- STANDARD SEGMENT
- name: cost_id
description: cost id
tests:
- relationships:
to: ref('base_exchange__transactions')
field: id
- name: base_value
description: test base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: quote_value
description: test quote value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_value_rm
description: transaction value rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_base_tax
description: test transaction base tax combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: base_service_charge
description: test base service charge combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_tax_rm
description: test transaction tax rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_value
description: test cost base value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_to_quote_currency_exchange_rate
description: test cost base to quote currency exchange rate combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_quote_value
description: test cost quote value combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_value_rm
description: test COST TRANSACTION VALUE RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_tax_rm
description: test COST TRANSACTION TAX RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_base_service_charge
description: test COST BASE SERVICE CHARGE combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: cost_transaction_service_charge_rm
description: test COST TRANSACTION SERVICE CHARGE_RM combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
- name: transaction_service_charge_rm
description: test transaction service charge rm combine with dbt_utils
tests:
- not_null
- dbt_utils.expression_is_true:
expression: " >= 0"
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: order_completed_datetime
column_B: order_created_datetime
or_equal: True
tests:
- dbt_utils.expression_is_true:
expression: "cost_transaction_value_rm + cost_transaction_tax_rm + cost_transaction_service_charge_rm < 2*(transaction_value_rm + transaction_base_tax + transaction_tax_rm)"
severity: warn
# FRESHNESS SOURCE
@@ -0,0 +1,178 @@
-- IMPORTS
WITH companies AS (
SELECT * FROM {{ ref('int_exchange__companies_level_segment') }}
),
company_addresses AS (
SELECT * FROM {{ ref('stg_exchange__company_addresses') }}
),
current_wallets AS (
SELECT * FROM {{ ref('stg_exchange__current_wallets') }}
),
companies_and_users_relations AS (
SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }}
),
company_contacts AS (
SELECT * FROM {{ ref('int_exchange__latest_company_contacts') }}
),
documents AS (
SELECT * FROM {{ ref('stg_exchange__documents') }}
),
-- LOGICS
latest_billing_company_addresses AS (
SELECT
*,
-- 1 will be latest updated address, rank by updated_at date
row_number() OVER
(PARTITION BY company_addresses.company_id
ORDER BY company_addresses.updated_at DESC) AS lastest_address_rank
FROM company_addresses
WHERE
is_billing_address = 1
QUALIFY
lastest_address_rank = 1
),
company_current_wallets AS (
SELECT
id,
owner_id AS company_id,
amount,
currency_id,
currency_name,
created_at,
updated_at
FROM current_wallets
WHERE
owner_type = 'App\\Models\\Company'
),
count_company_user AS (
SELECT
company_id,
COUNT(user_id) AS count_user
FROM companies_and_users_relations
GROUP BY company_id
),
latest_company_upload_identity_documents AS (
SELECT
id,
owner_type,
owner_id AS company_id,
document_type AS identity_document_type_name,
reference AS identity_number,
status,
status_name,
approved_at,
created_at,
row_number() OVER
(PARTITION BY company_id
ORDER BY created_at DESC) AS latest_row_number
FROM documents
WHERE
owner_type = 'App\\Models\\Company'
AND
document_type IN ('IDENTITY_CARD','SSM_REGISTRATION')
AND
status = '2'
QUALIFY
latest_row_number = '1'
),
-- FINAL
final__dim_exchange__companies AS (
SELECT
companies.id,
companies.name,
companies.marking_id,
companies.account_software_id,
companies.company_type,
companies.company_type_name,
companies.business_type,
companies.business_type_name,
companies.segment_id,
companies.segment_name,
companies.is_migrated_company,
companies.status,
companies.status_name,
companies.created_at,
company_contacts.reference AS company_contact_reference,
company_contacts.phone AS company_contact_phone,
company_contacts.email AS company_contact_email,
company_contacts.wechat_id AS company_contact_wechat_id,
latest_billing_company_addresses.id AS address_id,
latest_billing_company_addresses.country_id,
latest_billing_company_addresses.country_name,
latest_billing_company_addresses.country_short_code,
latest_billing_company_addresses.state_id,
latest_billing_company_addresses.state_name,
latest_billing_company_addresses.district_id,
latest_billing_company_addresses.district_name,
latest_billing_company_addresses.postcode,
latest_billing_company_addresses.address_line_one,
latest_billing_company_addresses.address_line_two,
latest_billing_company_addresses.is_billing_address,
latest_billing_company_addresses.deleted_at AS address_deleted_at,
latest_billing_company_addresses.created_at AS address_created_at,
latest_billing_company_addresses.updated_at AS address_updated_at,
company_current_wallets.id AS wallet_id,
IFF(company_current_wallets.id IS NOT NULL, 1, 0) AS have_wallet,
company_current_wallets.amount AS wallet_balance,
company_current_wallets.currency_id AS wallet_currency_id,
company_current_wallets.currency_name AS wallet_currency_name,
company_current_wallets.created_at AS wallet_created_at,
company_current_wallets.updated_at AS wallet_updated_at,
latest_company_upload_identity_documents.identity_document_type_name,
latest_company_upload_identity_documents.identity_number,
latest_company_upload_identity_documents.status AS identity_status,
latest_company_upload_identity_documents.status_name AS identity_status_name,
latest_company_upload_identity_documents.approved_at AS identity_approved_datetime,
latest_company_upload_identity_documents.created_at AS identity_created_datetime,
count_company_user.count_user AS number_of_user
FROM companies
LEFT JOIN latest_billing_company_addresses
ON (companies.id = latest_billing_company_addresses.company_id)
LEFT JOIN company_current_wallets
ON (companies.id = company_current_wallets.company_id)
LEFT JOIN count_company_user
ON (companies.id = count_company_user.company_id)
LEFT JOIN company_contacts
ON (companies.id = company_contacts.company_id)
LEFT JOIN latest_company_upload_identity_documents
ON (companies.id = latest_company_upload_identity_documents.company_id)
)
SELECT * FROM final__dim_exchange__companies
@@ -0,0 +1,202 @@
-- Code and forum link refer: https://gitlab.com/gitlab-data/analytics/-/blob/master/transform/snowflake-dbt/models/sources/date/date_details_source.sql https://stackoverflow.com/questions/71016585/dbt-use-dbt-modeling-to-insert-rows-in-a-table-like-date-dimension-table-in-az
-- Import
-- @NOTE TO GETSON, join this into calculated
WITH holiday_and_non_working_day_lists AS (
SELECT * FROM {{ ref('seed_exchange__holiday_and_non_working_day_lists') }}
),
-- LOGIC
date_spine AS (
{{ dbt_utils.date_spine(
start_date="to_date('01/01/2015', 'mm/dd/yyyy')",
datepart="day",
end_date="dateadd(year, 40, current_date)"
)
}}
),
calculated as (
SELECT
date_day,
date_day AS date_actual,
DAYNAME(date_day) AS day_name,
DATE_PART('month', date_day) AS month_actual,
DATE_PART('year', date_day) AS year_actual,
DATE_PART(quarter, date_day) AS quarter_actual,
CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7
ELSE DATE_PART(dayofweek, date_day) END AS day_of_week,
CASE WHEN day_name = 'Mon' THEN date_day
ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week,
CASE WHEN day_name = 'Mon' THEN WEEK(date_day)
ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column
CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1'
THEN '1'
ELSE week_of_year_temp END AS week_of_year,
DATE_PART('day', date_day) AS day_of_month,
ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter,
ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year,
-- @TODO @GERSON ,done
-- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023)
CASE WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual
when DATE_PART('month', DATEADD( month,-1, date_day) ) =12 THEN year_actual-1
ELSE (year_actual+1) END AS fiscal_year,
-- @TODO @GERSON , done
-- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4)
CASE WHEN month_actual in (2,3,4) THEN '1'
WHEN month_actual in (5,6,7) THEN '2'
WHEN month_actual in (8,9,10) THEN '3'
WHEN month_actual in (11,12) THEN '4'
WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4'
-- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4'
ELSE '4' END AS fiscal_quarter,
-- @TODO @GERSON ,done
ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter,
ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year,
TO_CHAR(date_day, 'MMMM') AS month_name,
TRUNC(date_day, 'Month') AS first_day_of_month,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month,
FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year,
FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter,
-- @TODO @GERSON ,done
-- Change this 2 too
FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter,
LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter,
-- @TODO @GERSON,done
FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year,
LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year,
-- @TODO @GERSON ,done
DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year,
-- @TODO @GERSON ,done
CASE WHEN EXTRACT('month', date_day) = 1 THEN 12
ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year,
LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week,
(year_actual || '-Q' || fiscal_quarter) AS quarter_name,
-- @TODO @GERSON,done
(fiscal_year || '-' || DECODE(fiscal_quarter,
1, 'Q1',
2, 'Q2',
3, 'Q3',
4, 'Q4')) AS fiscal_quarter_name,
('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy,
DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute,
--here check
fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name,
('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy,
-- @TODO @GERSON
-- Use the Seed, we will not doing the hard code in SQL, we use CSV to refer the holiday and desc
h.holiday_desc,
h.is_holiday,
h.holiday_race,
h.holiday_country,
h.holiday_state,
h.is_non_working_day,
-- @TODO @GERSON
DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter,
IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter,
DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year,
IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year,
DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa,
DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings
FROM date_spine s
LEFT JOIN holiday_and_non_working_day_lists h
ON s.date_day = to_date(h.date, 'dd/mm/yyyy')
QUALIFY date_actual >= '2016-01-01'
ORDER BY date_day
),
final AS (
SELECT
date_day,
date_actual,
day_name,
month_actual,
year_actual,
quarter_actual,
day_of_week,
first_day_of_week,
week_of_year,
day_of_month,
day_of_quarter,
day_of_year,
fiscal_year,
fiscal_quarter,
day_of_fiscal_quarter,
day_of_fiscal_year,
month_name,
first_day_of_month,
last_day_of_month,
first_day_of_year,
last_day_of_year,
first_day_of_quarter,
last_day_of_quarter,
first_day_of_fiscal_quarter,
last_day_of_fiscal_quarter,
first_day_of_fiscal_year,
last_day_of_fiscal_year,
week_of_fiscal_year,
month_of_fiscal_year,
last_day_of_week,
quarter_name,
fiscal_quarter_name,
fiscal_quarter_name_fy,
fiscal_quarter_number_absolute,
fiscal_month_name,
fiscal_month_name_fy,
holiday_desc,
is_holiday,
holiday_race,
holiday_country,
holiday_state,
is_non_working_day,
last_month_of_fiscal_quarter,
is_first_day_of_last_month_of_fiscal_quarter,
last_month_of_fiscal_year,
is_first_day_of_last_month_of_fiscal_year,
snapshot_date_fpa,
snapshot_date_billings,
current_timestamp() as _dbt_run_at
FROM calculated
)
SELECT * FROM final
@@ -0,0 +1,45 @@
-- IMPORTS
WITH users AS (
SELECT * FROM {{ ref('stg_exchange__users') }}
),
dedupe_user_email_verifications AS (
SELECT * FROM {{ ref('int_exchange__dedupe_user_email_verifications') }}
),
-- LOGICS
users_join_dedupe_user_email_verifications AS (
SELECT
users.id,
users.name,
users.email,
users.user_type,
users.user_type_name,
users.status,
users.status_name,
users.company_id,
users.created_at,
users.deleted_at,
users.updated_at,
dedupe_user_email_verifications.is_sent AS email_verification_is_sent,
dedupe_user_email_verifications.is_completed AS email_verification_is_completed,
dedupe_user_email_verifications.is_activated AS email_verification_is_activated,
dedupe_user_email_verifications.email_sent AS email_verification_email_sent,
first_email_created_at AS email_verification_first_email_created_at,
last_email_created_at AS email_verification_last_email_created_at
FROM users
JOIN dedupe_user_email_verifications
ON (users.id = dedupe_user_email_verifications.user_id)
),
-- FINAL
final__dim_exchange__users AS (
SELECT * FROM users_join_dedupe_user_email_verifications
)
SELECT *
FROM final__dim_exchange__users
@@ -0,0 +1,81 @@
-- IMPORTS
WITH booking_logs AS (
SELECT * FROM {{ ref('int_exchange__booking_logs_get_estimate_rates') }}
),
-- LOGICS
first_time_user_booking_id AS (
SELECT
booking_id,
user_id,
created_at,
row_number() OVER
(PARTITION BY booking_logs.user_id
ORDER BY booking_logs.created_at) AS cummulative_count_booking
FROM
booking_logs
GROUP BY
booking_id,
user_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
first_time_company_booking_id AS (
SELECT
booking_id,
company_id,
created_at,
row_number() OVER
(PARTITION BY booking_logs.company_id
ORDER BY booking_logs.created_at) AS cummulative_count_booking
FROM
booking_logs
GROUP BY
booking_id,
company_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
booking_logs_lists AS (
SELECT
booking_logs.*,
IFF(first_time_user_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_user,
IFF(first_time_company_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_company
FROM
booking_logs
LEFT JOIN first_time_user_booking_id
ON (booking_logs.booking_id = first_time_user_booking_id.booking_id)
LEFT JOIN first_time_company_booking_id
ON (booking_logs.booking_id = first_time_company_booking_id.booking_id)
ORDER BY
created_at
),
-- FINAL
final_fct_exchange__booking_logs AS (
SELECT
*
FROM booking_logs_lists
)
SELECT * FROM final_fct_exchange__booking_logs
@@ -0,0 +1,60 @@
-- IMPORTS
WITH bookings AS (
SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_rates') }}
),
-- LOGICS
bookings_lists AS (
SELECT
bookings.*,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY bookings.company_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1
ELSE 0
END AS is_first_time_company,
CASE
WHEN bookings.status = '3'
AND
MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL))
OVER (PARTITION BY bookings.company_id)
= bookings.booking_created_datetime
THEN 1
ELSE 0
END AS is_first_time_company_completed,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY bookings.user_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1
ELSE 0
END AS is_first_time_user,
CASE
WHEN bookings.status = '3'
AND
MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL))
OVER (PARTITION BY bookings.user_id)
= bookings.booking_created_datetime
THEN 1
ELSE 0
END AS is_first_time_user_completed
FROM
bookings
ORDER BY
booking_created_datetime
),
-- FINAL
final_fct_exchange__bookings AS (
SELECT
*
FROM bookings_lists
)
SELECT * FROM final_fct_exchange__bookings
@@ -0,0 +1,39 @@
--IMPORT
With documents AS (
SELECT * FROM {{ ref('stg_exchange__documents') }}
),
--LOGIC
company_identity_documents AS (
SELECT
*
FROM documents
WHERE
owner_type = 'App\\Models\\Company'
AND
document_type IN ('IDENTITY_CARD','SSM_REGISTRATION')
),
--FINAL
final__fct_exchange__company_identity_documents AS (
SELECT
id,
owner_type,
owner_id AS company_id,
document_type AS identity_document_type_name,
reference AS identity_number,
status AS identity_status,
status_name AS identity_status_name,
approved_by AS identity_approved_by,
deleted_at AS identity_deleted_datetine,
approved_at AS identity_approved_datetine,
created_at AS identity_created_datetime
FROM
company_identity_documents
)
SELECT * FROM final__fct_exchange__company_identity_documents
@@ -0,0 +1,48 @@
--IMPORT
WITH user_email_verifications AS (
SELECT * FROM {{ ref('stg_exchange__user_email_verifications') }}
),
companies_and_users_relations AS (
SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }}
),
--LOGIC
user_email_verifications_get_company_id AS (
SELECT
user_email_verifications.id,
user_email_verifications.user_id,
companies_and_users_relations.company_id,
user_email_verifications.email,
user_email_verifications.is_sent,
user_email_verifications.is_completed,
user_email_verifications.is_activated,
user_email_verifications.created_at,
user_email_verifications.updated_at
FROM user_email_verifications
LEFT JOIN companies_and_users_relations
ON (user_email_verifications.user_id = companies_and_users_relations.user_id)
),
--FINAL
final__fct_exchange__company_email_verifications AS (
SELECT
id,
user_id,
company_id,
email,
is_sent,
is_completed,
is_activated,
created_at AS verification_email_created_datetime,
updated_at AS verification_email_updated_datetime
FROM user_email_verifications_get_company_id
)
SELECT * FROM final__fct_exchange__company_email_verifications
@@ -0,0 +1,60 @@
-- IMPORTS
WITH transaction_costs AS (
SELECT * FROM {{ ref('int_exchange__transaction_costs_remove_duplicate_row') }}
),
-- LOGICS
transaction_costs_lists AS (
SELECT
id,
transaction_order_id,
supplier_company_id,
bank_id,
buyer_company_id,
status,
status_name,
transaction_type,
transaction_type_name,
payment_method,
payment_method_name,
base_currency_id,
base_currency_name,
quote_currency_id,
quote_currency_name,
base_to_quote_currency_exchange_rate,
base_value,
quote_value,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_value,
99999999) AS transaction_value_rm,
base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_tax,
99999999) AS transaction_tax_rm,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_service_charge,
99999999) AS transaction_service_charge_rm,
deleted_at,
created_at,
updated_at
FROM transaction_costs
),
-- FINAL
final__fct_exchange__transaction_costs AS (
SELECT * FROM transaction_costs_lists
)
SELECT * FROM final__fct_exchange__transaction_costs
@@ -0,0 +1,61 @@
-- IMPORTS
WITH transaction_log_costs AS (
SELECT * FROM {{ ref('stg_exchange__transaction_log_costs') }}
),
-- LOGICS
transaction_log_costs_lists AS (
SELECT
id,
transaction_cost_id,
transaction_order_id,
supplier_company_id,
bank_id,
buyer_company_id,
status,
status_name,
transaction_type,
transaction_type_name,
payment_method,
payment_method_name,
base_currency_id,
base_currency_name,
quote_currency_id,
quote_currency_name,
base_to_quote_currency_exchange_rate,
base_value,
quote_value,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_value,
99999999) AS transaction_value_rm,
base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_tax,
99999999) AS transaction_tax_rm,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(base_currency_id = '1',
base_service_charge,
99999999) AS transaction_service_charge_rm,
deleted_at,
created_at,
updated_at
FROM transaction_log_costs
),
-- FINAL
final__fct_exchange__transaction_log_costs AS (
SELECT * FROM transaction_log_costs_lists
)
SELECT * FROM final__fct_exchange__transaction_log_costs
@@ -0,0 +1,116 @@
-- IMPORTS
WITH transaction_log_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_log_orders_get_user_ids') }}
),
-- LOGICS
first_time_user_transaction_order_id AS (
SELECT
transaction_order_id,
user_id,
created_at,
row_number() OVER
(PARTITION BY transaction_log_orders.user_id
ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking
FROM
transaction_log_orders
GROUP BY
transaction_order_id,
user_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
first_time_company_transaction_order_id AS (
SELECT
transaction_order_id,
company_id,
created_at,
row_number() OVER
(PARTITION BY transaction_log_orders.company_id
ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking
FROM
transaction_log_orders
GROUP BY
transaction_order_id,
company_id,
created_at
QUALIFY
cummulative_count_booking = 1
),
transaction_log_orders_lists AS (
SELECT
transaction_log_orders.id,
transaction_log_orders.transaction_order_id,
transaction_log_orders.booking_id,
transaction_log_orders.seller_company_id,
transaction_log_orders.bank_id,
transaction_log_orders.company_id,
transaction_log_orders.user_id,
transaction_log_orders.status,
transaction_log_orders.status_name,
transaction_log_orders.transaction_type,
transaction_log_orders.transaction_type_name,
transaction_log_orders.payment_method,
transaction_log_orders.payment_method_name,
transaction_log_orders.base_currency_id,
transaction_log_orders.base_currency_name,
transaction_log_orders.quote_currency_id,
transaction_log_orders.quote_currency_name,
transaction_log_orders.base_to_quote_currency_exchange_rate,
transaction_log_orders.base_value,
transaction_log_orders.quote_value,
-- use for future if base currency id other from '1'
IFF(transaction_log_orders.base_currency_id = '1',
transaction_log_orders.base_value,
99999999) AS transaction_value_rm,
transaction_log_orders.base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(transaction_log_orders.base_currency_id = '1',
transaction_log_orders.base_tax,
99999999) AS transaction_tax_rm,
base_service_charge,
-- use for future if base currency id other from '1'
IFF(transaction_log_orders.base_currency_id = '1',
transaction_log_orders.base_service_charge,
99999999) AS transaction_service_charge_rm,
transaction_log_orders.deleted_at,
transaction_log_orders.created_at,
transaction_log_orders.updated_at,
IFF(first_time_user_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_user,
IFF(first_time_company_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_company
FROM transaction_log_orders
LEFT JOIN first_time_user_transaction_order_id
ON (transaction_log_orders.transaction_order_id = first_time_user_transaction_order_id.transaction_order_id)
LEFT JOIN first_time_company_transaction_order_id
ON (transaction_log_orders.transaction_order_id = first_time_company_transaction_order_id.transaction_order_id)
),
-- FINAL
final__fct_exchange__transaction_log_orders AS (
SELECT * FROM transaction_log_orders_lists
)
SELECT * FROM final__fct_exchange__transaction_log_orders
@@ -0,0 +1,92 @@
-- IMPORTS
WITH transaction_orders AS (
SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }}
),
-- LOGICS
transaction_orders_lists AS (
SELECT
transaction_orders.id,
transaction_orders.booking_id,
transaction_orders.seller_company_id,
transaction_orders.bank_id,
transaction_orders.user_id,
transaction_orders.company_id,
transaction_orders.status,
transaction_orders.status_name,
transaction_orders.transaction_type,
transaction_orders.transaction_type_name,
transaction_orders.payment_method,
transaction_orders.payment_method_name,
transaction_orders.base_currency_id,
transaction_orders.base_currency_name,
transaction_orders.quote_currency_id,
transaction_orders.quote_currency_name,
transaction_orders.base_to_quote_currency_exchange_rate,
transaction_orders.base_value,
transaction_orders.quote_value,
-- use for future if base currency id other from '1'
IFF(transaction_orders.base_currency_id = '1',
transaction_orders.base_value,
99999999) AS transaction_value_rm,
transaction_orders.base_tax AS transaction_base_tax,
-- use for future if base currency id other from '1'
IFF(transaction_orders.base_currency_id = '1',
transaction_orders.base_tax,
99999999) AS transaction_tax_rm,
transaction_orders.base_service_charge,
-- use for future if base currency id other from '1'
IFF(transaction_orders.base_currency_id = '1',
transaction_orders.base_service_charge,
99999999) AS transaction_service_charge_rm,
transaction_orders.deleted_at,
transaction_orders.created_at,
transaction_orders.updated_at,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.company_id ORDER BY transaction_orders.created_at) = 1 THEN 1
ELSE 0
END AS is_first_time_company,
CASE
WHEN transaction_orders.status = '3'
AND
MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL))
OVER (PARTITION BY transaction_orders.company_id)
= transaction_orders.created_at
THEN 1
ELSE 0
END AS is_first_time_company_completed,
CASE
WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.user_id ORDER BY transaction_orders.created_at) = 1 THEN 1
ELSE 0
END AS is_first_time_user,
CASE
WHEN transaction_orders.status = '3'
AND
MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL))
OVER (PARTITION BY transaction_orders.user_id)
= transaction_orders.created_at
THEN 1
ELSE 0
END AS is_first_time_user_completed
FROM transaction_orders
),
-- FINAL
final__fct_exchange__transaction_orders AS (
SELECT * FROM transaction_orders_lists
)
SELECT * FROM final__fct_exchange__transaction_orders
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: activity_log
description: Raw `activity_log` table from Snowflake
columns:
- name: id
description: Primary key for 'activity_log'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: addresses
description: Raw `addresses` table from Snowflake
columns:
- name: id
description: Primary key for 'addresses'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: announcement_segment
description: Raw `announcement_segment` table from Snowflake
columns:
- name: announcement_id
description: Primary key for 'announcement_segment'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: announcements
description: Raw `announcements` table from Snowflake
columns:
- name: id
description: Primary key for 'announcements'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}
@@ -0,0 +1,24 @@
version: 2
sources:
- name: src_exchange_mysql
description: Source data from snowflake 'DEV_CIEF_RAW_DB'
database: |
{%- if target.name == "prod" -%} PROD_CIEF_RAW_DB
{%- else -%} DEV_CIEF_RAW_DB
{%- endif -%}
schema: EXCHANGE_MYSQL
tables:
- name: bank_logs
description: Raw `bank_logs` table from Snowflake
columns:
- name: id
description: Primary key for 'bank_logs'
tests:
- unique
- not_null
loaded_at_field: _source_loaded_at
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 24, period: hour}

Some files were not shown because too many files have changed in this diff Show More