diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49f147c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/README.md b/README.md index e69de29..13040c0 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,15 @@ +Welcome to your new dbt project! + +### Using the starter project + +Try running the following commands: +- dbt run +- dbt test + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [dbt community](http://community.getbdt.com/) to learn from other analytics engineers +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/analyses/.gitkeep b/analyses/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/analyses/exchange/blog_ger.sql b/analyses/exchange/blog_ger.sql new file mode 100644 index 0000000..a1e029f --- /dev/null +++ b/analyses/exchange/blog_ger.sql @@ -0,0 +1,3 @@ + +SELECT distinct QUOTE_CURRENCY_ID ,QUOTE_CURRENCY_NAME + FROM {{ ref('fct_exchange__bookings') }} diff --git a/analyses/exchange/company_conversion_v2.sql b/analyses/exchange/company_conversion_v2.sql new file mode 100644 index 0000000..af2afe3 --- /dev/null +++ b/analyses/exchange/company_conversion_v2.sql @@ -0,0 +1,341 @@ +-- MARCRO for count the cummulative_activity_number +{% set funnel_activity = ['register','email_verified','identity_documents_uploaded','identity_documents_verified','first_time_booking','first_time_order','first_time_order_completed','repeat_completed_order_after_30days','repeat_completed_order_after_60days']%} + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + +company_identity_documents AS ( + SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }} +), + +email_verifications AS ( + SELECT * FROM {{ ref('fct_exchange__email_verifications') }} +), + +bookings AS ( + SELECT * FROM {{ ref('fct_exchange__bookings') }} +), + + +-- LOGIC +remove_supplier_company AS ( + SELECT + * + FROM companies + + WHERE business_type NOT IN ('1','3') +), + +clean_company_transaction_orders AS ( + SELECT + transaction_orders.* + + FROM transaction_orders + + -- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders + INNER JOIN remove_supplier_company + ON (transaction_orders.company_id = remove_supplier_company.id) +), + +clean_company_bookings AS ( + SELECT + bookings.* + + FROM bookings + + -- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders + INNER JOIN remove_supplier_company + ON (bookings.company_id = remove_supplier_company.id) +), + + +company_first_time_order_date AS ( + SELECT + id, + company_id, + created_at + + FROM clean_company_transaction_orders + + WHERE + is_first_time_company_completed = '1' +), + + + +--start defind funnel +repeat_completed_order_after_60days_activities AS ( + SELECT + clean_company_transaction_orders.company_id, + 0 AS is_data_cleansing_generate_row, + 'repeat_completed_order_after_60days' AS activity, + MIN(clean_company_transaction_orders.created_at) AS activity_datetime + + FROM clean_company_transaction_orders + + LEFT JOIN company_first_time_order_date + ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id) + + WHERE + DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 60 + + GROUP BY + clean_company_transaction_orders.company_id +), + +repeat_completed_order_after_30days_activities AS ( + SELECT + clean_company_transaction_orders.company_id, + 0 AS is_data_cleansing_generate_row, + 'repeat_completed_order_after_30days' AS activity, + MIN(clean_company_transaction_orders.created_at) AS activity_datetime + + FROM clean_company_transaction_orders + + LEFT JOIN company_first_time_order_date + ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id) + + WHERE + DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 30 + + GROUP BY + clean_company_transaction_orders.company_id +), + +first_time_order_completed_activities AS ( + SELECT + clean_company_transaction_orders.company_id, + 0 AS is_data_cleansing_generate_row, + 'first_time_order_completed' AS activity, + MIN(clean_company_transaction_orders.created_at) AS activity_datetime + + FROM clean_company_transaction_orders + + WHERE + clean_company_transaction_orders.is_first_time_company_completed = 1 + + GROUP BY + clean_company_transaction_orders.company_id +), + +first_time_order_activities AS ( + SELECT + clean_company_transaction_orders.company_id, + 0 AS is_data_cleansing_generate_row, + 'first_time_order' AS activity, + MIN(clean_company_transaction_orders.created_at) AS activity_datetime + + FROM clean_company_transaction_orders + + WHERE + clean_company_transaction_orders.is_first_time_company = 1 + + GROUP BY + clean_company_transaction_orders.company_id +), + +first_time_booking_activities AS ( + SELECT + IFNULL( unclean_first_time_booking_activities.company_id , first_time_order_activities.company_id ) AS company_id, + IFF( unclean_first_time_booking_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'first_time_booking' AS activity, + IFNULL( unclean_first_time_booking_activities.activity_datetime , first_time_order_activities.activity_datetime ) AS activity_datetime + + FROM + ( + SELECT + clean_company_bookings.company_id, + 'first_time_booking' AS activity, + MIN(clean_company_bookings.booking_created_datetime) AS activity_datetime + + FROM clean_company_bookings + + WHERE + clean_company_bookings.is_first_time_company = 1 + + GROUP BY + clean_company_bookings.company_id + ) AS unclean_first_time_booking_activities + + FULL OUTER JOIN first_time_order_activities + ON (unclean_first_time_booking_activities.company_id = first_time_order_activities.company_id) +), + + +identity_documents_verified_activities AS ( + SELECT + IFNULL( unclean_identity_documents_verified_activities.company_id , first_time_booking_activities.company_id ) AS company_id, + IFF( unclean_identity_documents_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'identity_documents_verified' AS activity, + IFNULL( unclean_identity_documents_verified_activities.activity_datetime , first_time_booking_activities.activity_datetime ) AS activity_datetime + + FROM + ( + + SELECT + company_id, + 'identity_documents_verified' AS activity, + MIN(identity_created_datetime) AS activity_datetime + + FROM company_identity_documents + + WHERE + identity_status = '2' + + GROUP BY + company_id + + ) AS unclean_identity_documents_verified_activities + + + FULL OUTER JOIN first_time_booking_activities + ON (unclean_identity_documents_verified_activities.company_id = first_time_booking_activities.company_id) + +), + +identity_documents_uploaded_activities AS ( + SELECT + IFNULL( unclean_identity_documents_uploaded_activities.company_id , identity_documents_verified_activities.company_id ) AS company_id, + IFF( unclean_identity_documents_uploaded_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'identity_documents_uploaded' AS activity, + IFNULL( unclean_identity_documents_uploaded_activities.activity_datetime , identity_documents_verified_activities.activity_datetime ) AS activity_datetime + + FROM + ( + + SELECT + company_id, + 'identity_documents_uploaded' AS activity, + MIN(identity_created_datetime) AS activity_datetime + + FROM company_identity_documents + + GROUP BY + company_id + + ) AS unclean_identity_documents_uploaded_activities + + + FULL OUTER JOIN identity_documents_verified_activities + ON (unclean_identity_documents_uploaded_activities.company_id = identity_documents_verified_activities.company_id) +), + +email_verified_activities AS ( + SELECT + IFNULL( unclean_email_verified_activities.company_id , identity_documents_uploaded_activities.company_id ) AS company_id, + IFF( unclean_email_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'email_verified' AS activity, + IFNULL( unclean_email_verified_activities.activity_datetime , identity_documents_uploaded_activities.activity_datetime ) AS activity_datetime + + FROM + ( + SELECT + company_id, + 'email_verified' AS activity, + MIN(verification_email_created_datetime) AS activity_datetime + + FROM email_verifications + + WHERE + is_completed = '1' + + GROUP BY + company_id + ) AS unclean_email_verified_activities + + FULL OUTER JOIN identity_documents_uploaded_activities + ON (unclean_email_verified_activities.company_id = identity_documents_uploaded_activities.company_id) + +), + +register_activities AS ( + SELECT + IFNULL( unclean_register_activities.company_id , email_verified_activities.company_id ) AS company_id, + IFF( unclean_register_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'register' AS activity, + IFNULL( unclean_register_activities.activity_datetime , email_verified_activities.activity_datetime ) AS activity_datetime + + FROM + ( + SELECT + id AS company_id, + 'register' AS activity, + MIN(created_at) AS activity_datetime + + FROM remove_supplier_company + + GROUP BY + id + ) AS unclean_register_activities + + FULL OUTER JOIN email_verified_activities + ON (unclean_register_activities.company_id = email_verified_activities.company_id) + +), + + +--all activites Union into a table +union_all_activities AS ( + SELECT * + FROM register_activities + + {% for activity in funnel_activity[1:] %} + + UNION + SELECT * + FROM {{activity}}_activities + + {% endfor %} + +), + +union_all_activities_flag_migrated_company AS ( + SELECT + union_all_activities.*, + remove_supplier_company.is_migrated_company + + FROM union_all_activities + + LEFT JOIN remove_supplier_company + ON (union_all_activities.company_id = remove_supplier_company.id) + +), + + +--FINAL + +sem_exchange__company_funnels AS ( + SELECT + company_id, + is_data_cleansing_generate_row, + is_migrated_company, + activity_datetime, + + {% for activity in funnel_activity %} + + count(iff(activity = '{{activity}}', company_id, NULL)) + OVER (ORDER BY activity_datetime) AS cumulative_{{activity}}_activity, + + {% endfor %} + + {% for activity in funnel_activity %} + + count(iff(activity = '{{activity}}' AND is_migrated_company = 0, company_id, NULL)) + OVER (ORDER BY activity_datetime) AS exclude_migrated_cumulative_{{activity}}_activity, + + {% endfor %} + + '__macro_end' AS __macro_end + + FROM union_all_activities_flag_migrated_company + + ORDER BY activity_datetime +) + +SELECT * FROM sem_exchange__company_funnels \ No newline at end of file diff --git a/analyses/exchange/company_conversionrate.sql b/analyses/exchange/company_conversionrate.sql new file mode 100644 index 0000000..754845d --- /dev/null +++ b/analyses/exchange/company_conversionrate.sql @@ -0,0 +1,299 @@ +-- HOW TO USE IT? +-- MUST: company_migration_segment filter need to be choose + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + +company_identity_documents AS ( + SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }} +), + +email_verifications AS ( + SELECT * FROM {{ ref('fct_exchange__email_verifications') }} +), + + +-- LOGIC +remove_supplier_company AS ( + SELECT + * + FROM companies + + WHERE business_type = 2 +), + + + +company_register_events AS ( + SELECT + remove_supplier_company.id AS company_id, + 'Register' AS register_event, + + MIN(remove_supplier_company.created_at) AS event_datetime + + FROM remove_supplier_company + + GROUP BY + remove_supplier_company.id +), + +company_email_verification_send_events AS ( + SELECT + company_id, + 'Email Verification Send' AS email_verification_send_event, + verification_email_created_datetime AS event_datetime, + row_number() OVER + (PARTITION BY company_id + ORDER BY verification_email_created_datetime) AS first_row_number + + FROM email_verifications + + QUALIFY + first_row_number = 1 +), + +company_email_verification_complete_events AS ( + SELECT + company_id, + 'Email Verification Complete' AS email_verification_complete_event, + verification_email_updated_datetime AS event_datetime, + row_number() OVER + (PARTITION BY company_id + ORDER BY verification_email_updated_datetime) AS first_row_number + + FROM email_verifications + + WHERE + is_completed = '1' + + QUALIFY + first_row_number = 1 +), + +company_upload_identity_events AS ( + SELECT + company_id, + 'Upload Identity' AS upload_identity_event, + + identity_created_datetime AS event_datetime, + row_number() OVER + (PARTITION BY company_id + ORDER BY identity_created_datetime) AS first_row_number + + FROM company_identity_documents + + QUALIFY + first_row_number = 1 +), + +company_approve_identity_events AS ( + SELECT + company_id, + 'Approve Identity' AS approve_identity_event, + + identity_approved_datetine AS event_datetime, + row_number() OVER + (PARTITION BY company_id + ORDER BY identity_created_datetime) AS first_row_number + + FROM company_identity_documents + + WHERE + identity_status in ('2','3') + + QUALIFY + first_row_number = 1 +), + +company_first_time_order_events AS ( + SELECT + first_time_transaction_orders.company_id, + companies.id, + 'First Time Order' AS first_time_order_event, + first_time_transaction_orders.created_at AS event_datetime + + FROM + (SELECT + *, + row_number() OVER + (PARTITION BY transaction_orders.company_id + ORDER BY transaction_orders.created_at) AS row_rank_index + + FROM transaction_orders + + QUALIFY + row_rank_index = 1 + ) AS first_time_transaction_orders + + INNER JOIN companies + ON(first_time_transaction_orders.company_id = companies.id) +), + + + +company_funnel_events AS ( + SELECT + company_id, + register_event, + NULL AS email_verification_send_event, + NULL AS email_verification_complete_event, + NULL AS upload_identity_event, + NULL AS approve_identity_event, + NULL AS first_time_order_event, + event_datetime + + FROM company_register_events + WHERE register_event IS NOT NULL + + UNION + + SELECT + company_id, + NULL AS register_event, + email_verification_send_event, + NULL AS email_verification_complete_event, + NULL AS upload_identity_event, + NULL AS approve_identity_event, + NULL AS first_time_order_event, + event_datetime + + FROM company_email_verification_send_events + WHERE email_verification_send_event IS NOT NULL + + UNION + + SELECT + company_id, + NULL AS register_event, + NULL AS email_verification_send_event, + email_verification_complete_event, + NULL AS upload_identity_event, + NULL AS approve_identity_event, + NULL AS first_time_order_event, + event_datetime + + FROM company_email_verification_complete_events + WHERE email_verification_complete_event IS NOT NULL + + UNION + + SELECT + company_id, + NULL AS register_event, + NULL AS email_verification_send_event, + NULL AS email_verification_complete_event, + upload_identity_event, + NULL AS approve_identity_event, + NULL AS first_time_order_event, + event_datetime + + FROM company_upload_identity_events + WHERE upload_identity_event IS NOT NULL + + UNION + + SELECT + company_id, + NULL AS register_event, + NULL AS email_verification_send_event, + NULL AS email_verification_complete_event, + NULL AS upload_identity_event, + approve_identity_event, + NULL AS first_time_order_event, + event_datetime + + FROM company_approve_identity_events + WHERE approve_identity_event IS NOT NULL + + UNION + + SELECT + company_id, + NULL AS register_event, + NULL AS email_verification_send_event, + NULL AS email_verification_complete_event, + NULL AS upload_identity_event, + NULL AS approve_identity_event, + first_time_order_event, + event_datetime + + FROM company_first_time_order_events + WHERE first_time_order_event IS NOT NULL + +), + +--FINAL +sem_exchange__company_funnel_events AS ( + SELECT + 'all_company' AS company_migration_segment, + company_funnel_events.event_datetime, + company_funnel_events.company_id, + remove_supplier_company.is_migrated_company, + + count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS register_company, + + count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company, + + count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company, + + count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company, + + count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event, + + count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company + + FROM company_funnel_events + + INNER JOIN remove_supplier_company + ON (company_funnel_events.company_id = remove_supplier_company.id) + + UNION + + SELECT + 'exclude_migrated_company' AS company_migration_segment, + company_funnel_events.event_datetime, + company_funnel_events.company_id, + remove_supplier_company.is_migrated_company, + + count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS register_company, + + count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company, + + count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company, + + count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company, + + count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event, + + count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL)) + OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company + + FROM company_funnel_events + + INNER JOIN remove_supplier_company + ON (company_funnel_events.company_id = remove_supplier_company.id) + + WHERE remove_supplier_company.is_migrated_company = '0' + + +) + + +SELECT count(*) FROM sem_exchange__company_funnel_events \ No newline at end of file diff --git a/analyses/shipping/y2022_companies_ltv.sql b/analyses/shipping/y2022_companies_ltv.sql new file mode 100644 index 0000000..82f03d4 --- /dev/null +++ b/analyses/shipping/y2022_companies_ltv.sql @@ -0,0 +1,84 @@ +WITH container_packing_lists AS ( + SELECT * FROM {{ ref('base_shipping__container_packing_lists') }} +), + +containers AS ( + SELECT * FROM {{ ref('base_shipping__containers') }} +), + +packing_lists AS ( + SELECT * FROM {{ ref('base_shipping__packing_lists') }} +), + +packages AS ( + SELECT * FROM {{ ref('base_shipping__packages') }} +), + +company_modules AS ( + SELECT * FROM {{ ref('base_shipping__company_modules') }} +), + +company_connections AS ( + SELECT * FROM {{ ref('base_shipping__company_connections') }} +), + +orders AS ( + SELECT * FROM {{ ref('base_shipping__orders') }} +) +-- LOGIC +company_module_join_companies AS ( + SELECT + company_modules.id AS company_id, + company_modules.name AS company_name, + company_connections.invitee_reference AS marking + + FROM + company_modules + LEFT JOIN company_connections + ON (company_modules.id = company_connections.invitee_id) +), + +final AS ( + SELECT + container_packing_lists.*, + containers.*, + packing_lists.*, + packages.*, + packages.created_at AS packages_created_at, + ((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm + + FROM container_packing_lists + + JOIN containers + ON (container_packing_lists.container_id = containers.id) + JOIN packing_lists + ON (container_packing_lists.packing_list_id = packing_lists.id) + JOIN packages + ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null) + + WHERE packing_lists.deleted_at is null AND packages_created_at >= '2022-01-01') +-- ), + +-- final_group_by_company_module AS ( +-- SELECT +-- final.owner_id, +-- SUM(final.cbm) AS life_time_cbm + +-- FROM final + +-- GROUP BY +-- owner_id +-- ) + + + +SELECT * FROM final + +-- SELECT +-- containers.*, packing_lists.id as packing_list_id, SUM((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm +-- FROM +-- container_packing_lists as packed_containers +-- JOIN containers ON packed_containers.container_id = containers.id +-- JOIN packing_lists ON packed_containers.packing_list_id = packing_lists.id +-- JOIN packages ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null) +-- WHERE packing_lists.deleted_at is null GROUP BY packing_lists.id; \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..2a411e4 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,142 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'cief_dbt_cloud' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'default' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. +models: + cief_dbt_cloud: + # Applies to all files under models/example/ + exchange: + marts: + reporting: + description: data modeling use in Preset (Superset) + +materialized: view + +schema: exchange__reporting + + warehouse: + description: data modeling contain all fct & dim + +materialized: table + +schema: exchange__warehouse + + intermediate: + description: data modeling use for pipeline transformation, contain all int + +materialized: view + +schema: exchange__intermediate + + staging: + description: light_transformation & will be occur in staging, contain all stg + +materialized: view + +schema: exchange__staging + + base: + description: light_transformation such as rename and remove test or wrong input data + +materialized: view + +schema: exchange__base + + + shipping: + marts: + reporting: + description: data modeling use in Preset (Superset) + +materialized: view + +schema: shipping__reporting + + warehouse: + description: data modeling contain all fct & dim + +materialized: view + +schema: shipping__warehouse + + intermediate: + description: data modeling use for pipeline transformation, contain all int + +materialized: view + +schema: shipping__intermediate + + staging: + description: light_transformation & will be occur in staging, contain all stg + +materialized: view + +schema: shipping__staging + + base: + description: light_transformation such as rename and remove test or wrong input data + +materialized: view + +schema: shipping__base + + + crisp: + marts: + reporting: + description: data modeling use in Preset (Superset) + +materialized: view + +schema: crisp__reporting + + warehouse: + description: data modeling contain all fct & dim + +materialized: view + +schema: crisp__warehouse + + intermediate: + description: data modeling use for pipeline transformation, contain all int + +materialized: view + +schema: crisp__intermediate + + staging: + description: light_transformation & will be occur in staging, contain all stg + +materialized: view + +schema: crisp__staging + + base: + description: light_transformation such as rename and remove test or wrong input data + +materialized: view + +schema: crisp__base + + +seeds: + cief_dbt_cloud: + exchange: + +schema: exchange__seeds + shipping: + +schema: shipping__seeds + crisp: + +schema: crisp__seeds + +snapshots: + cief_dbt_cloud: + exchange: + +target_schema: "{% if target.name == 'prod' %}PROD_EXCHANGE__SNAPSHOTS{% else %}{{ target.schema }}_EXCHANGE__SNAPSHOTS{% endif %}" + shipping: + +target_schema: "{% if target.name == 'prod' %}PROD_SHIPPING__SNAPSHOTS{% else %}{{ target.schema }}_SHIPPING__SNAPSHOTS{% endif %}" + crisp: + +target_schema: "{% if target.name == 'prod' %}PROD_CRISP__SNAPSHOTS{% else %}{{ target.schema }}_CRISP__SNAPSHOTS{% endif %}" + +tests: + +severity: warn + + + diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/models/crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql b/models/crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql new file mode 100644 index 0000000..36b61a5 --- /dev/null +++ b/models/crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql @@ -0,0 +1,77 @@ +--IMPORT +WITH messages AS ( + SELECT * FROM {{ ref('stg_crisp__messages') }} + ORDER BY conversation_id, message_created_datetime, id +), + +conversations AS ( + SELECT * FROM {{ ref('stg_crisp__conversations') }} +), + +--LOGIC +mark_first_dialog_message AS ( + SELECT + CASE + WHEN LAG(event_type) OVER (ORDER BY conversation_id, message_created_datetime, id) = 'state:resolved' + THEN 1 + ELSE 0 + END AS first_dialog_message_flag, + * + + FROM messages +), + +message_with_dialog_id AS ( + SELECT + (SUM(first_dialog_message_flag) OVER (ORDER BY conversation_id, message_created_datetime, id ROWS UNBOUNDED PRECEDING))+1 AS dialog_id, + * + + FROM mark_first_dialog_message + ORDER BY conversation_id, message_created_datetime +), + +--FINAL +final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_type AS ( + SELECT + message_with_dialog_id.dialog_id, + message_with_dialog_id.id, + + message_with_dialog_id.conversation_id, + + CASE + WHEN message_with_dialog_id.message_from_side = 'operator' THEN message_with_dialog_id.user_id + WHEN message_with_dialog_id.message_from_side = 'user' and conversations.user_id IS NOT NULL THEN conversations.user_id + WHEN message_with_dialog_id.message_from_side = 'user' and conversations.user_id IS NULL THEN null + ELSE 'ERROR please contact DATA team' + END AS user_id, + + message_with_dialog_id.user_name, + message_with_dialog_id.message_from_side, + + CASE + WHEN message_with_dialog_id.message_from_side = 'user' AND message_with_dialog_id.user_id IS NOT NULL THEN 'user' + WHEN message_with_dialog_id.message_from_side = 'operator' AND message_with_dialog_id.user_id IS NOT NULL THEN 'operator' + WHEN message_with_dialog_id.message_from_side = 'operator' AND message_with_dialog_id.user_id IS NULL THEN 'bot' + ELSE 'ERROR please contact DATA team' + END AS message_generate_by, + + message_with_dialog_id.message_generate_platform, + message_with_dialog_id.message_deliverd_platform, + message_with_dialog_id.message_read_platform, + message_with_dialog_id.message_format, + message_with_dialog_id.event_type, + message_with_dialog_id.event_message, + message_with_dialog_id.message, + message_with_dialog_id.is_edited_message, + message_with_dialog_id.attachment_type, + message_with_dialog_id.attachment_name, + message_with_dialog_id.attachment_url, + message_with_dialog_id.message_created_datetime + + FROM message_with_dialog_id + LEFT JOIN conversations + ON (message_with_dialog_id.user_id = conversations.id) --The user_id will become conversation_id if message_from_side is user + +) + +SELECT * FROM final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_type \ No newline at end of file diff --git a/models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql b/models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql new file mode 100644 index 0000000..31d35d3 --- /dev/null +++ b/models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql @@ -0,0 +1,68 @@ +--IMPORT +WITH peoples AS ( + SELECT * FROM {{ ref('stg_crisp__peoples') }} +), + +people_data AS ( + SELECT * FROM {{ ref('stg_crisp__people_data') }} +), + +--LOGIC +people_get_marking_id AS ( + SELECT + peoples.id, + peoples.name, + peoples.phone, + peoples.email, + peoples.website, + peoples.score, + peoples.segment_list, + peoples.role, + peoples.seniority, + peoples.title, + peoples.gender, + peoples.timezone, + peoples.geo_country, + peoples.geo_city, + peoples.geo_region_code, + peoples.address, + peoples.locale_list, + peoples.geo_latitude, + peoples.geo_longitude, + peoples.last_activated_datetime, + peoples.people_created_datetime, + peoples.people_updated_datetime, + peoples.company_name, + peoples.company_legal_name, + peoples.company_description, + peoples.company_phone, + peoples.company_email_list, + peoples.company_website, + peoples.company_domain, + peoples.company_tag_list, + peoples.company_number_of_employees, + peoples.company_market_cap, + peoples.company_country, + peoples.company_city, + peoples.company_region, + peoples.company_latitude, + peoples.company_logitude, + peoples.company_timezone, + people_data.company_name AS marking_company_name, + people_data.whatsapp_number, + people_data.whatsapp_business_number, + people_data.facebook_page_name, + people_data.facebook_page_id, + people_data.exchange_marking_id, + people_data.izyim_marking_id, + people_data.lite_marking_id + + FROM peoples + + LEFT JOIN people_data + ON (peoples.id = people_data.people_id) +) + + +--FINAL +select * from people_get_marking_id \ No newline at end of file diff --git a/models/crisp/marts/warehouse/dim_crisp__companies.sql b/models/crisp/marts/warehouse/dim_crisp__companies.sql new file mode 100644 index 0000000..98751bc --- /dev/null +++ b/models/crisp/marts/warehouse/dim_crisp__companies.sql @@ -0,0 +1,86 @@ +--IMPORT +WITH users_and_companies AS ( + SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }} +), + +--LOGIC +split_company_data AS ( + SELECT + exchange_marking_id, + izyim_marking_id, + lite_marking_id, + marking_company_name, + whatsapp_number, + whatsapp_business_number, + facebook_page_name, + facebook_page_id, + company_name, + company_legal_name, + company_description, + company_phone, + company_email_list, + company_website, + company_domain, + company_tag_list, + company_number_of_employees, + company_market_cap, + company_country, + company_city, + company_region, + company_latitude, + company_logitude, + company_timezone + + FROM users_and_companies +), + +companies_remove_duplicate AS ( + SELECT + DISTINCT * + + FROM split_company_data +), + +--FINAL +final_dim_crisp__companies AS ( + SELECT + exchange_marking_id, + izyim_marking_id, + lite_marking_id, + IFF(COUNT(DISTINCT(marking_company_name)) > 1, LISTAGG(marking_company_name, ','), MAX(marking_company_name) ) AS marking_company_name, + IFF(COUNT(DISTINCT(whatsapp_number)) > 1, LISTAGG(whatsapp_number, ','), MAX(whatsapp_number) ) AS whatsapp_number, + IFF(COUNT(DISTINCT(whatsapp_business_number)) > 1, LISTAGG(whatsapp_business_number, ','), MAX(whatsapp_business_number) ) AS whatsapp_business_number, + IFF(COUNT(DISTINCT(facebook_page_name)) > 1, LISTAGG(facebook_page_name, ','), MAX(facebook_page_name) ) AS facebook_page_name, + IFF(COUNT(DISTINCT(facebook_page_id)) > 1, LISTAGG(facebook_page_id, ','), MAX(facebook_page_id) ) AS facebook_page_id, + IFF(COUNT(DISTINCT(company_name)) > 1, LISTAGG(company_name, ','), MAX(company_name) ) AS company_name, + IFF(COUNT(DISTINCT(company_legal_name)) > 1, LISTAGG(company_legal_name, ','), MAX(company_legal_name) ) AS company_legal_name, + IFF(COUNT(DISTINCT(company_description)) > 1, LISTAGG(company_description, ','), MAX(company_description) ) AS company_description, + IFF(COUNT(DISTINCT(company_phone)) > 1, LISTAGG(company_phone, ','), MAX(company_phone) ) AS company_phone, + IFF(COUNT(DISTINCT(company_email_list)) > 1, LISTAGG(company_email_list, ','), MAX(company_email_list) ) AS company_email_list, + IFF(COUNT(DISTINCT(company_website)) > 1, LISTAGG(company_website, ','), MAX(company_website) ) AS company_website, + IFF(COUNT(DISTINCT(company_domain)) > 1, LISTAGG(company_domain, ','), MAX(company_domain) ) AS company_domain, + IFF(COUNT(DISTINCT(company_tag_list)) > 1, LISTAGG(company_tag_list, ','), MAX(company_tag_list) ) AS company_tag_list, + IFF(COUNT(DISTINCT(company_number_of_employees)) > 1, LISTAGG(company_number_of_employees, ','), MAX(company_number_of_employees) ) AS company_number_of_employees, + IFF(COUNT(DISTINCT(company_market_cap)) > 1, LISTAGG(company_market_cap, ','), MAX(company_market_cap) ) AS company_market_cap, + IFF(COUNT(DISTINCT(company_country)) > 1, LISTAGG(company_country, ','), MAX(company_country) ) AS company_country, + IFF(COUNT(DISTINCT(company_city)) > 1, LISTAGG(company_city, ','), MAX(company_city) ) AS company_city, + IFF(COUNT(DISTINCT(company_region)) > 1, LISTAGG(company_region, ','), MAX(company_region) ) AS company_region, + IFF(COUNT(DISTINCT(company_latitude)) > 1, LISTAGG(company_latitude, ','), MAX(company_latitude) ) AS company_latitude, + IFF(COUNT(DISTINCT(company_logitude)) > 1, LISTAGG(company_logitude, ','), MAX(company_logitude) ) AS company_logitude, + IFF(COUNT(DISTINCT(company_timezone)) > 1, LISTAGG(company_timezone, ','), MAX(company_timezone) ) AS company_timezone + + FROM companies_remove_duplicate + + WHERE + (IFF(exchange_marking_id IS NULL, 1, 0) + + IFF(izyim_marking_id IS NULL, 1, 0) + + IFF(lite_marking_id IS NULL, 1, 0)) < 3 + + GROUP BY + exchange_marking_id, + izyim_marking_id, + lite_marking_id + +) +-- SELECT EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID, count(*) FROM final_dim_crisp__companies GROUP BY EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID ORDER BY count(*) desc +SELECT * FROM final_dim_crisp__companies \ No newline at end of file diff --git a/models/crisp/marts/warehouse/dim_crisp__users.sql b/models/crisp/marts/warehouse/dim_crisp__users.sql new file mode 100644 index 0000000..75878e2 --- /dev/null +++ b/models/crisp/marts/warehouse/dim_crisp__users.sql @@ -0,0 +1,48 @@ +--IMPORT +WITH users_and_companies AS ( + SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }} +), + +--LOGIC +split_user_data AS ( + SELECT + id, + name, + phone, + email, + website, + score, + segment_list, + role, + seniority, + title, + gender, + timezone, + geo_country, + geo_city, + geo_region_code, + address, + locale_list, + geo_latitude, + geo_longitude, + last_activated_datetime, + people_created_datetime, + people_updated_datetime, + exchange_marking_id AS company_exchange_marking_id, + izyim_marking_id AS company_izyim_marking_id, + lite_marking_id AS company_lite_marking_id + + FROM users_and_companies +), + +--FINAL +final_dim_crisp__users AS ( + SELECT + * + FROM split_user_data +) +SELECT * FROM final_dim_crisp__users +-- SELECT company_lite_marking_id, count(distinct(company_exchange_marking_id)) AS t1, count(distinct(company_izyim_marking_id)) AS t2 +-- FROM final_dim_crisp__users GROUP BY company_lite_marking_id HAVING t1 >1 or t2 > 1 +-- SELECT company_IZYIM_MARKING_ID, company_exchange_marking_id FROM final_dim_crisp__users +-- WHERE company_IZYIM_MARKING_ID IN ('1324FEN','3791YBE', '1291NSC', '2698LXA', '2431HEB', '2634PLT', '8299LIE', '1882KAI', '1889LBH', '913DLT', '1650GTB', '5903CSB', '6631GEB', '1656HBP', '1905JYW', '8265LSB', '7260UUT', '5989HSY') ORDER BY company_IZYIM_MARKING_ID \ No newline at end of file diff --git a/models/crisp/marts/warehouse/fct_crisp__dialogs.sql b/models/crisp/marts/warehouse/fct_crisp__dialogs.sql new file mode 100644 index 0000000..e844829 --- /dev/null +++ b/models/crisp/marts/warehouse/fct_crisp__dialogs.sql @@ -0,0 +1,137 @@ +--IMPORT +WITH messages AS ( + SELECT * FROM {{ ref('int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by') }} + ORDER BY conversation_id, message_created_datetime, id +), + +--LOGIC +messages_with_ftu_id AS ( + SELECT + *, + FIRST_VALUE(CASE WHEN messages.message_generate_by = 'user' THEN messages.user_id END) + IGNORE NULLS + OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS first_user_id, + LAST_VALUE(CASE WHEN messages.message_generate_by = 'user' THEN messages.user_id END) + IGNORE NULLS + OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS last_user_id, + + FIRST_VALUE(CASE WHEN messages.message_generate_by = 'operator' THEN messages.user_id END) + IGNORE NULLS + OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS first_operator_id, + LAST_VALUE(CASE WHEN messages.message_generate_by = 'operator' THEN messages.user_id END) + IGNORE NULLS + OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS last_operator_id + + from messages +), + +dialog_message_last_row AS ( + SELECT + RANK() OVER (PARTITION BY dialog_id ORDER BY message_created_datetime DESC) AS last_row_desc_rank, + * + + FROM messages + QUALIFY last_row_desc_rank = 1 +), + +--FINAL + +final_fct_crisp__dialogs AS ( + SELECT + messages_with_ftu_id.dialog_id AS dialog_id, + max(messages_with_ftu_id.conversation_id) AS conversation_id, + array_agg(messages_with_ftu_id.id) AS message_ids, + + CASE + WHEN max(dialog_message_last_row.event_type) = 'state:resolved' THEN 'close' + WHEN max(dialog_message_last_row.event_type) IS NULL THEN 'open' + ELSE 'ERROR, Please contact DATA team' + END AS dialog_status, + + min(messages_with_ftu_id.message_created_datetime) AS dialog_created_datetime, + + min(IFF(messages_with_ftu_id.message_generate_by = 'user', + messages_with_ftu_id.message_created_datetime, + null)) AS first_user_reply_datetime, + + min(IFF(messages_with_ftu_id.message_generate_by = 'operator', + messages_with_ftu_id.message_created_datetime, + null)) AS first_operator_reply_datetime, + + min(IFF(messages_with_ftu_id.message_generate_by = 'bot', + messages_with_ftu_id.message_created_datetime, + null)) AS first_bot_reply_datetime, + + max(IFF(messages_with_ftu_id.message_generate_by = 'user', + messages_with_ftu_id.message_created_datetime, + null)) AS last_user_reply_datetime, + + max(IFF(messages_with_ftu_id.message_generate_by = 'operator', + messages_with_ftu_id.message_created_datetime, + null)) AS last_operator_reply_datetime, + + max(IFF(messages_with_ftu_id.message_generate_by = 'bot', + messages_with_ftu_id.message_created_datetime, + null)) AS last_bot_reply_datetime, + + max(IFF((messages_with_ftu_id.event_type = 'state:resolved'), + messages_with_ftu_id.message_created_datetime, + null)) AS dialog_solved_datetime, + + + array_agg(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'user' , + messages_with_ftu_id.user_id, + NULL))) AS user_ids, + + count(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'user', + messages_with_ftu_id.user_id, + null))) AS number_of_user_involved, + + max(messages_with_ftu_id.first_user_id) AS first_user_id, + max(messages_with_ftu_id.last_user_id) AS last_user_id, + + + array_agg(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'operator' , + messages_with_ftu_id.user_id, + NULL))) AS operator_ids, + + count(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'operator', + messages_with_ftu_id.user_id, + null))) AS number_of_operator_involved, + + max(messages_with_ftu_id.first_operator_id) AS first_operator_id, + max(messages_with_ftu_id.last_operator_id) AS last_operator_id, + + + count(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'user', + messages_with_ftu_id.id, + null))) AS number_of_user_chat, + + count(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'operator', + messages_with_ftu_id.id, + null))) AS number_of_operator_chat, + + count(distinct( + IFF(messages_with_ftu_id.message_generate_by = 'bot', + messages_with_ftu_id.id, + null))) AS number_of_bot_chat + + + FROM messages_with_ftu_id + + LEFT JOIN dialog_message_last_row + ON (dialog_message_last_row.dialog_id = messages_with_ftu_id.dialog_id) + + GROUP BY + messages_with_ftu_id.dialog_id +) + +SELECT * FROM final_fct_crisp__dialogs + +--Check 'K Huan Huan' why result number_of_user_id = 2 session_id = 'session_75a3a65d-390b-4ed9-90a7-f98a8a39b768'; \ No newline at end of file diff --git a/models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql b/models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql new file mode 100644 index 0000000..9685016 --- /dev/null +++ b/models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql @@ -0,0 +1,15 @@ +--IMPORT +WITH snap_user_email_subscriptions AS ( + SELECT * FROM {{ ref('stg_crisp__people_email_subscriptions') }} +), + + +--FINAL +final_fct_crisp__user_email_subscriptions AS ( + SELECT + * + + FROM snap_user_email_subscriptions +) + +SELECT * FROM snap_user_email_subscriptions \ No newline at end of file diff --git a/models/crisp/source/src_crisp__get_message_in_conversations.yml b/models/crisp/source/src_crisp__get_message_in_conversations.yml new file mode 100644 index 0000000..7a5f001 --- /dev/null +++ b/models/crisp/source/src_crisp__get_message_in_conversations.yml @@ -0,0 +1,26 @@ +version: 2 + +sources: + - name: src_crisp_api + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + + tables: + - name: get_message_in_conversations + description: Raw `get_people_subscription_status` table from Snowflake + columns: + - name: fingerprint + description: Primary key for 'get_message_in_conversations' + tests: + - unique + - not_null + - name: delivered + description: the platform which after the message is output from origin, and delivered to the user/operator. When an operator send a message but the user is not online anymore, we send it via email only after 2min during these 2min, delivered is empty + loaded_at_field: api_called_at + freshness: + warn_after: {count: 1, period: day} + error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__get_people_data.yml b/models/crisp/source/src_crisp__get_people_data.yml new file mode 100644 index 0000000..b691274 --- /dev/null +++ b/models/crisp/source/src_crisp__get_people_data.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_crisp_api + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + + tables: + - name: get_people_data + description: Raw `get_people_data` table from Snowflake + columns: + - name: people_id + description: Primary key for 'get_people_data' + tests: + - unique + - not_null + loaded_at_field: api_called_at + freshness: + warn_after: {count: 1, period: day} + error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__get_people_subscription_status.yml b/models/crisp/source/src_crisp__get_people_subscription_status.yml new file mode 100644 index 0000000..0ea03ca --- /dev/null +++ b/models/crisp/source/src_crisp__get_people_subscription_status.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_crisp_api + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + + tables: + - name: get_people_subscription_status + description: Raw `get_people_subscription_status` table from Snowflake + columns: + - name: people_id + description: Primary key for 'get_people_subscription_status' + tests: + - unique + - not_null + loaded_at_field: api_called_at + freshness: + warn_after: {count: 1, period: day} + error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__list_conversations.yml b/models/crisp/source/src_crisp__list_conversations.yml new file mode 100644 index 0000000..3af0c3c --- /dev/null +++ b/models/crisp/source/src_crisp__list_conversations.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_crisp_api + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + + tables: + - name: list_conversations + description: Raw `get_people_subscription_status` table from Snowflake + columns: + - name: SESSION_ID + description: Primary key for 'list_conversations' + tests: + - unique + - not_null + loaded_at_field: api_called_at + freshness: + warn_after: {count: 1, period: day} + error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__list_people_profiles.yml b/models/crisp/source/src_crisp__list_people_profiles.yml new file mode 100644 index 0000000..a7e7f69 --- /dev/null +++ b/models/crisp/source/src_crisp__list_people_profiles.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_crisp_api + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + + tables: + - name: list_people_profiles + description: Raw `list_people_profiles` table from Snowflake + columns: + - name: people_id + description: Primary key for 'list_people_profiles' + tests: + - unique + - not_null + loaded_at_field: api_called_at + freshness: + warn_after: {count: 1, period: day} + error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__list_website_operator.yml b/models/crisp/source/src_crisp__list_website_operator.yml new file mode 100644 index 0000000..b185736 --- /dev/null +++ b/models/crisp/source/src_crisp__list_website_operator.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_crisp_api + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + + tables: + - name: list_website_operator + description: Raw `list_website_operator` table from Snowflake + columns: + - name: details__user_id + description: Primary key for 'list_website_operator' + tests: + - unique + - not_null + loaded_at_field: api_called_at + freshness: + warn_after: {count: 1, period: day} + error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__conversations.sql b/models/crisp/staging/stg_crisp__conversations.sql new file mode 100644 index 0000000..39b9c6c --- /dev/null +++ b/models/crisp/staging/stg_crisp__conversations.sql @@ -0,0 +1,22 @@ +-- IMPORT +WITH list_conversations AS ( + SELECT * FROM {{ source('src_crisp_api', 'list_conversations') }} +), + +--LOGIC + +--FINAL +final_stg_crisp__conversations AS ( + SELECT + session_id AS id, + people_id AS user_id, + state, + status, + IFF(is_blocked = TRUE, 1, 0) AS is_blocked, + assigned__user_id AS operator_id, + convert_timezone('Asia/Kuala_Lumpur', created_at) AS conversation_created_datetime, + convert_timezone('Asia/Kuala_Lumpur', updated_at) AS conversation_updated_datetime + + FROM list_conversations +) +SELECT * FROM final_stg_crisp__conversations \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__messages.sql b/models/crisp/staging/stg_crisp__messages.sql new file mode 100644 index 0000000..6fa4992 --- /dev/null +++ b/models/crisp/staging/stg_crisp__messages.sql @@ -0,0 +1,32 @@ +-- IMPORT +WITH get_message_in_conversations AS ( + SELECT * FROM {{ source('src_crisp_api', 'get_message_in_conversations') }} +), + +-- LOGIC + +-- FINAL +final_stg_crisp__messages AS ( +SELECT + fingerprint AS id, + session_id AS conversation_id, + user__user_id AS user_id, + user__nickname AS user_name, + from_user AS message_from_side, + origin AS message_generate_platform, + delivered AS message_deliverd_platform, + read AS message_read_platform, + type AS message_format, + content__namespace AS event_type, + content__text AS event_message, + content AS message, + edited AS is_edited_message, + content__type AS attachment_type, + content__name AS attachment_name, + content__url AS attachment_url, + convert_timezone('Asia/Kuala_Lumpur', timestamp) AS message_created_datetime + +FROM get_message_in_conversations ORDER BY session_id, message_created_datetime +) + +SELECT * FROM final_stg_crisp__messages \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__operators.sql b/models/crisp/staging/stg_crisp__operators.sql new file mode 100644 index 0000000..cf9fae8 --- /dev/null +++ b/models/crisp/staging/stg_crisp__operators.sql @@ -0,0 +1,20 @@ +-- IMPORT +WITH list_website_operator AS ( + SELECT * FROM {{ source('src_crisp_api', 'list_website_operator') }} +), + +--LOGIC + +--FINAL +final_stg_crisp__operators AS ( + SELECT + details__user_id AS id, + details__title AS title, + details__first_name AS first_name, + details__last_name AS last, + details__email AS email + + FROM list_website_operator +) + +SELECT * FROM final_stg_crisp__operators \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__people_data.sql b/models/crisp/staging/stg_crisp__people_data.sql new file mode 100644 index 0000000..8454886 --- /dev/null +++ b/models/crisp/staging/stg_crisp__people_data.sql @@ -0,0 +1,23 @@ +-- IMPORT +WITH get_people_data AS ( + SELECT * FROM {{ source('src_crisp_api', 'get_people_data') }} +), + +--LOGIC + +--FINAL +final_stg_crisp__get_people_data AS ( + SELECT + people_id, + data__company_name AS company_name, + data__wa_number AS whatsapp_number, + data__whatsapp_business_number AS whatsapp_business_number, + data__fb_page AS facebook_page_name, + data__fb_page_id AS facebook_page_id, + data__exchange_marking AS exchange_marking_id, + data__izyim_marking AS izyim_marking_id, + data__lite_marking AS lite_marking_id + + FROM get_people_data +) +SELECT * FROM final_stg_crisp__get_people_data \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__people_email_subscriptions.sql b/models/crisp/staging/stg_crisp__people_email_subscriptions.sql new file mode 100644 index 0000000..9f8135f --- /dev/null +++ b/models/crisp/staging/stg_crisp__people_email_subscriptions.sql @@ -0,0 +1,20 @@ +-- IMPORT +WITH get_people_subscription_status AS ( + SELECT * FROM {{ ref('snap_crisp__get_people_subscription_status') }} +), + +--LOGIC + +--FINAL +final_stg_crisp__people_email_subscriptions AS ( + SELECT + dbt_scd_id, + people_id, + IFF(email = TRUE, 1, 0) AS is_subscribed, + DBT_VALID_FROM AS dbt_valid_from_datetime, + DBT_VALID_TO AS dbt_valid_to_datetime + + FROM get_people_subscription_status +) + +SELECT * FROM final_stg_crisp__people_email_subscriptions \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__peoples.sql b/models/crisp/staging/stg_crisp__peoples.sql new file mode 100644 index 0000000..f8cb577 --- /dev/null +++ b/models/crisp/staging/stg_crisp__peoples.sql @@ -0,0 +1,54 @@ +-- IMPORT +WITH list_people_profiles AS ( + SELECT * FROM {{ source('src_crisp_api', 'list_people_profiles') }} +), + +--LOGIC + +--FINAL +final_stg_crisp__peoples AS ( + SELECT + people_id AS id, + person__nickname AS name, + person__phone AS phone, + email AS email, + person__website AS website, + score AS score, + segments AS segment_list, + person__employment__role AS role, + person__employment__seniority AS seniority, + person__employment__title AS title, + person__gender AS gender, + person__timezone AS timezone, + person__geolocation__country AS geo_country, + person__geolocation__city AS geo_city, + person__geolocation__region AS geo_region_code, + person__address AS address, + person__locales AS locale_list, + person__geolocation__coordinates__latitude AS geo_latitude, + person__geolocation__coordinates__longitude AS geo_longitude, + convert_timezone('Asia/Kuala_Lumpur', active__last) AS last_activated_datetime, + convert_timezone('Asia/Kuala_Lumpur', created_at) AS people_created_datetime, + convert_timezone('Asia/Kuala_Lumpur', updated_at) AS people_updated_datetime, + + company__name AS company_name, + company__legal_name AS company_legal_name, + company__description AS company_description, + company__phones AS company_phone, + company__emails AS company_email_list, + company__url AS company_website, + company__domain AS company_domain, + company__tags AS company_tag_list, + company__metrics__employees AS company_number_of_employees, + company__metrics__market_cap AS company_market_cap, + company__geolocation__country AS company_country, + company__geolocation__city AS company_city, + company__geolocation__region AS company_region, + company__geolocation__coordinates__latitude AS company_latitude, + company__geolocation__coordinates__longitude AS company_logitude, + company__timezone AS company_timezone + + FROM list_people_profiles +) + +SELECT * FROM final_stg_crisp__peoples \ No newline at end of file diff --git a/models/exchange/base/activity_logs/base_exchange__activity_logs.sql b/models/exchange/base/activity_logs/base_exchange__activity_logs.sql new file mode 100644 index 0000000..fed1f5d --- /dev/null +++ b/models/exchange/base/activity_logs/base_exchange__activity_logs.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH activity_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'activity_log') }} +), + + +-- LOGIC +activity_logs_rename AS ( + + SELECT + activity_logs.id, + activity_logs.log_name, + activity_logs.description, + activity_logs.subject_id, + activity_logs.subject_type, + activity_logs.causer_id, + activity_logs.causer_type, + activity_logs.properties, + activity_logs.created_at, + activity_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM activity_logs +), + + +-- FINAL +final__base_exchange__activity_logs AS ( + + SELECT + id, + log_name, + description, + subject_id, + subject_type, + causer_id, + causer_type, + properties, + created_at, + updated_at, + _dbt_ran_at + + FROM activity_logs_rename +) + +SELECT * FROM final__base_exchange__activity_logs \ No newline at end of file diff --git a/models/exchange/base/addresses/base_exchange__addresses.sql b/models/exchange/base/addresses/base_exchange__addresses.sql new file mode 100644 index 0000000..19c0b1a --- /dev/null +++ b/models/exchange/base/addresses/base_exchange__addresses.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH addresses AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'addresses') }} +), + +-- LOGIC +addresses_rename AS ( + + SELECT + addresses.id, + addresses.company_id, + addresses.country_id, + addresses.state_id, + addresses.district_id, + addresses.postcode, + addresses.street_one AS address_line_one, + addresses.street_two AS address_line_two, + addresses.billing AS is_billing_address, + addresses.deleted_at, + addresses.created_at, + addresses.updated_At, + current_timestamp() as _dbt_ran_at + + FROM addresses +), + +-- FINAL +final__base_exchange__addresses AS ( + + SELECT + id, + company_id, + country_id, + state_id, + district_id, + postcode, + address_line_one, + address_line_two, + is_billing_address, + deleted_at, + created_at, + updated_At, + _dbt_ran_at + + FROM addresses_rename +) + +SELECT * FROM final__base_exchange__addresses \ No newline at end of file diff --git a/models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql b/models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql new file mode 100644 index 0000000..4418be9 --- /dev/null +++ b/models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql @@ -0,0 +1,34 @@ +-- IMPORTS +WITH announcement_segments AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'announcement_segment') }} +), + + +-- LOGIC +announcement_segments_rename AS ( + + SELECT + announcement_segments.announcement_id, + announcement_segments.segment_id, + announcement_segments.created_at, + announcement_segments.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM announcement_segments +), + + +-- FINAL +final__base_exchange__announcement_segments AS ( + + SELECT + announcement_id, + segment_id, + created_at, + updated_at, + _dbt_ran_at + + FROM announcement_segments_rename +) + +SELECT * FROM final__base_exchange__announcement_segments \ No newline at end of file diff --git a/models/exchange/base/announcements/base_exchange__announcements.sql b/models/exchange/base/announcements/base_exchange__announcements.sql new file mode 100644 index 0000000..db52a8b --- /dev/null +++ b/models/exchange/base/announcements/base_exchange__announcements.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH announcements AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'announcements') }} +), + + +-- LOGIC +announcements_rename AS ( + + SELECT + id, + title, + description, + starting_on AS started_at, + ending_on AS ended_at, + is_all_day, + duration, + is_recurring, + recurrence_pattern, + deleted_at, + created_at, + updated_at, + current_timestamp() as _dbt_ran_at + + FROM announcements +), + + +-- FINAL +final__base_exchange__announcements AS ( + + SELECT + id, + title, + description, + started_at, + ended_at, + is_all_day, + duration, + is_recurring, + recurrence_pattern, + deleted_at, + created_at, + updated_at, + current_timestamp() as _dbt_ran_at + + FROM announcements_rename +) + +SELECT * FROM final__base_exchange__announcements \ No newline at end of file diff --git a/models/exchange/base/bank_logs/base_exchange__bank_logs.sql b/models/exchange/base/bank_logs/base_exchange__bank_logs.sql new file mode 100644 index 0000000..ae9755b --- /dev/null +++ b/models/exchange/base/bank_logs/base_exchange__bank_logs.sql @@ -0,0 +1,61 @@ +-- IMPORTS +WITH bank_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'bank_logs') }} +), + + +-- LOGIC +bank_logs_rename AS ( + + SELECT + bank_logs.id, + bank_logs.bank_id, + bank_logs.company_id, + bank_logs.reference, + bank_logs.bank_name, + bank_logs.holder_name, + bank_logs.account_no, + bank_logs.bank_branch, + bank_logs.swift AS swift_code, + bank_logs.snap AS cnaps_code, + bank_logs.type AS bank_type, + bank_logs.default AS is_default, + bank_logs.status, + bank_logs.country_id, + bank_logs.deleted_at, + bank_logs.created_at, + bank_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM bank_logs + +), + + +-- FINAL +final__base_exchange__bank_logs AS ( + + SELECT + id, + bank_id, + company_id, + reference, + bank_name, + holder_name, + account_no, + bank_branch, + swift_code, + cnaps_code, + bank_type, + is_default, + status, + country_id, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM bank_logs_rename +) + +SELECT * FROM final__base_exchange__bank_logs \ No newline at end of file diff --git a/models/exchange/base/banks/base_exchange__banks.sql b/models/exchange/base/banks/base_exchange__banks.sql new file mode 100644 index 0000000..0b5ae1f --- /dev/null +++ b/models/exchange/base/banks/base_exchange__banks.sql @@ -0,0 +1,58 @@ +-- IMPORTS +WITH banks AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'banks') }} +), + + +-- LOGIC +banks_rename AS ( + + SELECT + banks.id, + banks.company_id, + banks.reference, + banks.bank_name, + banks.holder_name, + banks.account_no, + banks.bank_branch, + banks.swift AS swift_code, + banks.snap AS cnaps_code, + banks.type AS bank_type, + banks.default AS is_default, + banks.status, + banks.country_id, + banks.deleted_at, + banks.created_at, + banks.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM banks +), + + +-- FINAL +final__base_exchange__banks AS ( + + SELECT + id, + company_id, + reference, + bank_name, + holder_name, + account_no, + bank_branch, + swift_code, + cnaps_code, + bank_type, + is_default, + status, + country_id, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM banks_rename +) + +SELECT * FROM final__base_exchange__banks \ No newline at end of file diff --git a/models/exchange/base/booking_logs/base_exchange__booking_logs.sql b/models/exchange/base/booking_logs/base_exchange__booking_logs.sql new file mode 100644 index 0000000..e5800b1 --- /dev/null +++ b/models/exchange/base/booking_logs/base_exchange__booking_logs.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH booking_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'booking_logs') }} +), + + +-- LOGIC +booking_logs_rename AS ( + + SELECT + booking_logs.id, + booking_logs.booking_id, + booking_logs.company_id, + booking_logs.marking AS marking_id, + booking_logs.service_id AS service_type, + booking_logs.bank_id, + booking_logs.fix_amount AS fix_value, + booking_logs.fix_currency_id, + booking_logs.convertible_currency_id AS quote_currency_id, + booking_logs.conversion_currency_id AS base_currency_id, + booking_logs.status, + booking_logs.deleted_at, + booking_logs.created_at, + booking_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM booking_logs + +), + +-- FINAL +final_base_exchange__booking_logs AS ( + + SELECT + id, + booking_id, + company_id, + marking_id, + service_type, + bank_id, + fix_value, + fix_currency_id, + quote_currency_id, + base_currency_id, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM booking_logs_rename + +) + +SELECT * FROM final_base_exchange__booking_logs \ No newline at end of file diff --git a/models/exchange/base/bookings/base_exchange__bookings.sql b/models/exchange/base/bookings/base_exchange__bookings.sql new file mode 100644 index 0000000..f5a3072 --- /dev/null +++ b/models/exchange/base/bookings/base_exchange__bookings.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH bookings AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'bookings') }} +), + + +-- LOGIC +bookings_rename AS ( + + SELECT + bookings.id, + bookings.company_id, + bookings.marking AS marking_id, + bookings.service_id AS service_type, + bookings.bank_id, + bookings.fix_amount AS fix_value, + bookings.fix_currency_id, + bookings.convertible_currency_id AS quote_currency_id, + bookings.conversion_currency_id AS base_currency_id, + bookings.status, + bookings.deleted_at, + bookings.created_at, + bookings.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM bookings + +), + +-- FINAL +final_base_exchange__bookings AS ( + + SELECT + id, + company_id, + marking_id, + service_type, + bank_id, + fix_value, + fix_currency_id, + quote_currency_id, + base_currency_id, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM bookings_rename + +) + +SELECT * FROM final_base_exchange__bookings \ No newline at end of file diff --git a/models/exchange/base/companies/base_exchange__companies.sql b/models/exchange/base/companies/base_exchange__companies.sql new file mode 100644 index 0000000..c6566fc --- /dev/null +++ b/models/exchange/base/companies/base_exchange__companies.sql @@ -0,0 +1,47 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'companies') }} +), + + +-- LOGIC +companies_rename AS ( + + SELECT + companies.id, + companies.name, + companies.reference AS marking_id, + companies.debtor AS account_software_id, + companies.type AS company_type, + companies.business_type, + companies.status, + companies.deleted_at, + companies.created_at, + companies.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM companies +), + + +-- FINAL +final__base_exchange__companies AS ( + + SELECT + id, + name, + marking_id, + account_software_id, + company_type, + business_type, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM companies_rename + +) + +SELECT * FROM final__base_exchange__companies \ No newline at end of file diff --git a/models/exchange/base/contacts/base_exchange__contacts.sql b/models/exchange/base/contacts/base_exchange__contacts.sql new file mode 100644 index 0000000..78421fe --- /dev/null +++ b/models/exchange/base/contacts/base_exchange__contacts.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH contacts AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'contacts') }} +), + + +-- LOGIC +companies_rename AS ( + + SELECT + contacts.id, + contacts.company_id, + contacts.country_id, + contacts.reference, + contacts.phone, + contacts.email, + contacts.wechat_id, + contacts.deleted_at, + contacts.created_at, + contacts.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM contacts + +), + + +-- FINAL +final__base_exchange__contacts AS ( + + SELECT + id, + company_id, + country_id, + reference, + phone, + email, + wechat_id, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM companies_rename + +) + +SELECT * FROM final__base_exchange__contacts \ No newline at end of file diff --git a/models/exchange/base/countries/base_exchange__countries.sql b/models/exchange/base/countries/base_exchange__countries.sql new file mode 100644 index 0000000..c60a99b --- /dev/null +++ b/models/exchange/base/countries/base_exchange__countries.sql @@ -0,0 +1,41 @@ +-- IMPORTS +WITH countries AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'countries') }} +), + +-- LOGIC +countries_rename AS ( + + SELECT + countries.id, + countries.name, + countries.short_code, + countries.phone_code, + countries.deleted_at, + countries.created_at, + countries.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM countries + +), + + +-- FINAL +final__base_exchange__countries AS ( + + SELECT + id, + name, + short_code, + phone_code, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM countries_rename + +) + +SELECT * FROM final__base_exchange__countries \ No newline at end of file diff --git a/models/exchange/base/currencies/base_exchange__currencies.sql b/models/exchange/base/currencies/base_exchange__currencies.sql new file mode 100644 index 0000000..72c670d --- /dev/null +++ b/models/exchange/base/currencies/base_exchange__currencies.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH currencies AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'currencies') }} +), + + +-- LOGIC +currencies_rename AS ( + + SELECT + currencies.id, + currencies.country_id, + currencies.name, + currencies.short_code, + currencies.symbol, + currencies.deleted_at, + currencies.created_at, + currencies.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM currencies +), + + +-- FINAL +final__base_exchange__currencies AS ( + + SELECT + id, + country_id, + name, + short_code, + symbol, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM currencies_rename + +) + +SELECT * FROM final__base_exchange__currencies \ No newline at end of file diff --git a/models/exchange/base/currency_logs/base_exchange__currency_logs.sql b/models/exchange/base/currency_logs/base_exchange__currency_logs.sql new file mode 100644 index 0000000..9a549a2 --- /dev/null +++ b/models/exchange/base/currency_logs/base_exchange__currency_logs.sql @@ -0,0 +1,39 @@ +-- IMPORTS +WITH currency_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'currency_logs') }} +), + + +-- LOGIC +currency_logs_rename AS ( + + SELECT + currency_logs.id, + currency_logs.currency_id, + currency_logs.created_by, + currency_logs.deleted_at, + currency_logs.created_at, + currency_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM currency_logs +), + + +-- FINAL +final__base_exchange__currency_logs AS ( + + SELECT + id, + currency_id, + created_by, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM currency_logs_rename + +) + +SELECT * FROM final__base_exchange__currency_logs \ No newline at end of file diff --git a/models/exchange/base/currency_rate_logs/base_exchange__currency_rate_logs.sql b/models/exchange/base/currency_rate_logs/base_exchange__currency_rate_logs.sql new file mode 100644 index 0000000..f1f39e0 --- /dev/null +++ b/models/exchange/base/currency_rate_logs/base_exchange__currency_rate_logs.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH currency_rate_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'currency_rate_logs') }} +), +-- LOGIC +currency_rate_logs_clean_list AS ( + + SELECT + id, + round(1/selling, 5) AS selling + FROM + currency_rate_logs + WHERE + id IN ('579','580','581','582') +), + +currency_rate_logs_rename_clean_wrong_rate AS ( + + SELECT + currency_rate_logs.id, + currency_rate_logs.currency_rate_id, + COALESCE(currency_rate_logs_clean_list.selling, currency_rate_logs.selling) AS base_to_quote_currency_exchange_rate, + currency_rate_logs.created_by, + currency_rate_logs.created_at, + currency_rate_logs.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM currency_rate_logs + + LEFT JOIN currency_rate_logs_clean_list + ON (currency_rate_logs.id = currency_rate_logs_clean_list.id) +), + + +-- FINAL +final__base_exchange__currency_rate_logs AS ( + + SELECT + id, + currency_rate_id, + base_to_quote_currency_exchange_rate, + created_by, + created_at, + updated_at, + _dbt_ran_at + + FROM currency_rate_logs_rename_clean_wrong_rate +) + +SELECT * FROM final__base_exchange__currency_rate_logs \ No newline at end of file diff --git a/models/exchange/base/currency_rates/base_exchange__currency_rates.sql b/models/exchange/base/currency_rates/base_exchange__currency_rates.sql new file mode 100644 index 0000000..488d104 --- /dev/null +++ b/models/exchange/base/currency_rates/base_exchange__currency_rates.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH currency_rates AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'currency_rates') }} +), + + +-- LOGIC +currency_rates_rename AS ( + + SELECT + currency_rates.id, + currency_rates.currency_id AS quote_currency_id, + currency_rates.selling AS base_to_quote_currency_exchange_rate, + currency_rates.payment_method_type AS payment_method, + currency_rates.service_id AS service_type, + currency_rates.deleted_at, + currency_rates.created_at, + currency_rates.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM currency_rates +), + + +-- FINAL +final__base_exchange__currency_rates AS ( + + SELECT + id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + payment_method, + service_type, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM currency_rates_rename + +) + +SELECT * FROM final__base_exchange__currency_rates \ No newline at end of file diff --git a/models/exchange/base/districts/base_exchange__districts.sql b/models/exchange/base/districts/base_exchange__districts.sql new file mode 100644 index 0000000..c193cb6 --- /dev/null +++ b/models/exchange/base/districts/base_exchange__districts.sql @@ -0,0 +1,44 @@ +-- IMPORTS +WITH districts AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'districts') }} +), + + +-- LOGIC +districts_rename AS ( + + SELECT + districts.id, + districts.country_id, + districts.state_id, + districts.name, + districts.postcode, + districts.status, + districts.deleted_at, + districts.created_at, + districts.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM districts +), + + +-- FINAL +final__base_exchange__districts AS ( + + SELECT + id, + country_id, + state_id, + name, + postcode, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM districts_rename +) + +SELECT * FROM final__base_exchange__districts \ No newline at end of file diff --git a/models/exchange/base/documents/base_exchange__documents.sql b/models/exchange/base/documents/base_exchange__documents.sql new file mode 100644 index 0000000..59e6f5f --- /dev/null +++ b/models/exchange/base/documents/base_exchange__documents.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'documents') }} +), + + +-- LOGIC +documents_rename AS ( + + SELECT + documents.id, + documents.owner_type, + documents.owner_id, + documents.document_type, + documents.reference, + documents.status, + documents.issued_date AS issued_at, + documents.expired_date AS expired_at, + documents.approver AS approved_by, + documents.approval_date AS approved_at, + documents.deleted_at, + documents.created_at, + documents.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM documents +), + + +-- FINAL +final__base_exchange__documents AS ( + + SELECT + id, + owner_type, + owner_id, + document_type, + reference, + status, + issued_at, + expired_at, + approved_by, + approved_at, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM documents_rename + +) + +SELECT * FROM final__base_exchange__documents \ No newline at end of file diff --git a/models/exchange/base/employees/base_exchange__employees.sql b/models/exchange/base/employees/base_exchange__employees.sql new file mode 100644 index 0000000..b895732 --- /dev/null +++ b/models/exchange/base/employees/base_exchange__employees.sql @@ -0,0 +1,32 @@ +-- IMPORTS +WITH employees AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'employees') }} +), + + +-- LOGIC +employees_rename AS ( + + SELECT + employees.company_id, + employees.user_id, + employees.status, + current_timestamp() AS _dbt_ran_at + + FROM employees +), + + +-- FINAL +final__base_exchange__employees AS ( + + SELECT + company_id, + user_id, + status, + _dbt_ran_at + + FROM employees_rename +) + +SELECT * FROM final__base_exchange__employees \ No newline at end of file diff --git a/models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql b/models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql new file mode 100644 index 0000000..aa6b770 --- /dev/null +++ b/models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH failed_jobs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'failed_jobs') }} +), + + +-- LOGIC +failed_jobs_rename AS ( + + SELECT + failed_jobs.id, + failed_jobs.connection, + failed_jobs.queue, + failed_jobs.payload, + failed_jobs.exception, + failed_jobs.failed_at, + current_timestamp() AS _dbt_ran_at + + FROM failed_jobs + +), + + +-- FINAL +final__stg_failed_jobs__exchange AS ( + + SELECT + id, + connection, + queue, + payload, + exception, + failed_at, + _dbt_ran_at + + FROM failed_jobs_rename + +) + +SELECT * FROM final__stg_failed_jobs__exchange \ No newline at end of file diff --git a/models/exchange/base/files/base_exchange__files.sql b/models/exchange/base/files/base_exchange__files.sql new file mode 100644 index 0000000..3cc140e --- /dev/null +++ b/models/exchange/base/files/base_exchange__files.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH files AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'files') }} +), + + +-- LOGIC +files_rename AS ( + + SELECT + files.id, + files.document_id, + files.file, + files.file_type, + files.deleted_at, + files.created_at, + files.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM files + +), + + +-- FINAL +final__base_exchange__files AS ( + + SELECT + id, + document_id, + file, + file_type, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM files_rename + +) + +SELECT * FROM final__base_exchange__files \ No newline at end of file diff --git a/models/exchange/base/group_transactions/base_exchange__group_transactions.sql b/models/exchange/base/group_transactions/base_exchange__group_transactions.sql new file mode 100644 index 0000000..628b543 --- /dev/null +++ b/models/exchange/base/group_transactions/base_exchange__group_transactions.sql @@ -0,0 +1,34 @@ +-- IMPORTS +WITH group_transactions AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'group_transactions') }} +), + + +-- LOGIC +group_transactions_rename AS ( + + SELECT + group_transactions.id, + group_transactions.group_id, + group_transactions.transaction_id, + current_timestamp() AS _dbt_ran_at + + FROM group_transactions + +), + + +-- FINAL +final__base_exchange__group_transactions AS ( + + SELECT + id, + group_id, + transaction_id, + _dbt_ran_at + + FROM group_transactions_rename + +) + +SELECT * FROM final__base_exchange__group_transactions \ No newline at end of file diff --git a/models/exchange/base/groups/base_exchange__groups.sql b/models/exchange/base/groups/base_exchange__groups.sql new file mode 100644 index 0000000..e4dbdd8 --- /dev/null +++ b/models/exchange/base/groups/base_exchange__groups.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH groups AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'groups') }} +), + + +-- LOGIC +groups_rename AS ( + + SELECT + groups.id, + groups.reference AS marking_id, + groups.issuer AS issued_by, + groups.receiver AS received_by, + groups.amount AS base_amount, + groups.original_amount AS quote_amount, + groups.currency_id AS base_currency_id, + groups.original_currency_id AS quote_currency_id, + groups.currency_rate AS base_to_quote_currency_exchange_rate, + groups.tax AS base_tax, + groups.service_charge AS base_service_charge, + groups.status, + groups.created_at, + groups.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM groups +), + + +-- FINAL +final__base_exchange__groups AS ( + + SELECT + id, + marking_id, + issued_by, + received_by, + base_amount, + quote_amount, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + status, + created_at, + updated_at, + _dbt_ran_at + + FROM groups_rename + +) + +SELECT * FROM final__base_exchange__groups \ No newline at end of file diff --git a/models/exchange/base/jobs/base_exchange__jobs.sql b/models/exchange/base/jobs/base_exchange__jobs.sql new file mode 100644 index 0000000..4ae12ee --- /dev/null +++ b/models/exchange/base/jobs/base_exchange__jobs.sql @@ -0,0 +1,41 @@ +-- IMPORTS +WITH jobs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'jobs') }} +), + + +-- LOGIC +jobs_rename AS ( + + SELECT + jobs.id, + jobs.queue, + jobs.payload, + jobs.attempts, + TO_TIMESTAMP(jobs.reserved_at) AS reserved_at, + TO_TIMESTAMP(jobs.available_at) AS available_at, + TO_TIMESTAMP(jobs.created_at) AS created_at, + current_timestamp() AS _dbt_ran_at + + FROM jobs +), + + +-- FINAL +final__base_exchange__jobs AS ( + + SELECT + id, + queue, + payload, + attempts, + reserved_at, + available_at, + created_at, + _dbt_ran_at + + FROM jobs_rename + +) + +SELECT * FROM final__base_exchange__jobs \ No newline at end of file diff --git a/models/exchange/base/migrations/base_exchange__migrations.sql b/models/exchange/base/migrations/base_exchange__migrations.sql new file mode 100644 index 0000000..c7e3231 --- /dev/null +++ b/models/exchange/base/migrations/base_exchange__migrations.sql @@ -0,0 +1,33 @@ +-- IMPORTS +WITH migrations AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'migrations') }} +), + + +-- LOGIC +migrations_rename AS ( + + SELECT + migrations.id, + migrations.migration, + migrations.batch, + current_timestamp() AS _dbt_ran_at + + FROM migrations +), + + +-- FINAL +final__base_exchange__migrations AS ( + + SELECT + id, + migration, + batch, + _dbt_ran_at + + FROM migrations_rename + +) + +SELECT * FROM final__base_exchange__migrations \ No newline at end of file diff --git a/models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql b/models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql new file mode 100644 index 0000000..a5b25da --- /dev/null +++ b/models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql @@ -0,0 +1,34 @@ +-- IMPORTS +WITH model_has_roles AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'model_has_roles') }} +), + + +-- LOGIC +migrations_rename AS ( + + SELECT + model_has_roles.role_id, + model_has_roles.model_type, + model_has_roles.model_id, + current_timestamp() AS _dbt_ran_at + + FROM model_has_roles + +), + + +-- FINAL +final__base_exchange__model_has_roles AS ( + + SELECT + role_id, + model_type, + model_id, + _dbt_ran_at + + FROM migrations_rename + +) + +SELECT * FROM final__base_exchange__model_has_roles \ No newline at end of file diff --git a/models/exchange/base/notifications/base_exchange__notifications.sql b/models/exchange/base/notifications/base_exchange__notifications.sql new file mode 100644 index 0000000..620b4c1 --- /dev/null +++ b/models/exchange/base/notifications/base_exchange__notifications.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH notifications AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'notifications') }} +), + + +-- LOGIC +notifications_rename AS ( + + SELECT + notifications.id, + notifications.title, + notifications.description, + notifications.subject_type, + notifications.subject_id, + notifications.target_type, + notifications.target_id, + notifications.causer_type, + notifications.causer_id, + notifications.status, + notifications.deleted_at, + notifications.created_at, + notifications.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM notifications +), + + +-- FINAL +final__base_exchange__notifications AS ( + + SELECT + id, + title, + description, + subject_type, + subject_id, + target_type, + target_id, + causer_type, + causer_id, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM notifications_rename + +) + +SELECT * FROM final__base_exchange__notifications \ No newline at end of file diff --git a/models/exchange/base/password_resets/base_exchange__password_resets.sql b/models/exchange/base/password_resets/base_exchange__password_resets.sql new file mode 100644 index 0000000..d3c5505 --- /dev/null +++ b/models/exchange/base/password_resets/base_exchange__password_resets.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH password_resets AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'password_resets') }} +), + + +-- LOGIC +password_resets_rename AS ( + + SELECT + password_resets.id, + password_resets.user_id, + password_resets.token, + password_resets.is_expired, + password_resets.is_complete, + password_resets.created_at, + password_resets.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM password_resets + +), + + +-- FINAL +final__base_exchange__password_resets AS ( + + SELECT + id, + user_id, + token, + is_expired, + is_complete, + created_at, + updated_at, + _dbt_ran_at + + FROM password_resets_rename + +) + +SELECT * FROM final__base_exchange__password_resets \ No newline at end of file diff --git a/models/exchange/base/permissions/base_exchange__permissions.sql b/models/exchange/base/permissions/base_exchange__permissions.sql new file mode 100644 index 0000000..0d1ff93 --- /dev/null +++ b/models/exchange/base/permissions/base_exchange__permissions.sql @@ -0,0 +1,38 @@ +-- IMPORTS +WITH permissions AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'permissions') }} +), + + +-- LOGIC +permissions_rename AS ( + + SELECT + permissions.id, + permissions.name, + permissions.guard_name, + permissions.created_at, + permissions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM permissions + +), + + +-- FINAL +final__base_exchange__permissions AS ( + + SELECT + id, + name, + guard_name, + created_at, + updated_at, + _dbt_ran_at + + FROM permissions_rename + +) + +SELECT * FROM final__base_exchange__permissions \ No newline at end of file diff --git a/models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql b/models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql new file mode 100644 index 0000000..2867cf1 --- /dev/null +++ b/models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql @@ -0,0 +1,32 @@ +-- IMPORTS +WITH role_has_permissions AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'role_has_permissions') }} +), + + +-- LOGIC +role_has_permissions_rename AS ( + + SELECT + role_has_permissions.permission_id, + role_has_permissions.role_id, + current_timestamp() AS _dbt_ran_at + + FROM role_has_permissions + +), + + +-- FINAL +final__exchange_base__role_has_permissions AS ( + + SELECT + permission_id, + role_id, + _dbt_ran_at + + FROM role_has_permissions_rename + +) + +SELECT * FROM final__exchange_base__role_has_permissions \ No newline at end of file diff --git a/models/exchange/base/roles/base_exchange__roles.sql b/models/exchange/base/roles/base_exchange__roles.sql new file mode 100644 index 0000000..0f7c8db --- /dev/null +++ b/models/exchange/base/roles/base_exchange__roles.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH roles AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'roles') }} +), + + +-- LOGIC +roles_rename AS ( + + SELECT + roles.id, + roles.name, + roles.guard_name, + roles.type, + roles.created_at, + roles.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM roles + +), + + +-- FINAL +final__base_exchange__roles AS ( + + SELECT + id, + name, + guard_name, + type, + created_at, + updated_at, + _dbt_ran_at + + FROM roles_rename + +) + +SELECT * FROM final__base_exchange__roles \ No newline at end of file diff --git a/models/exchange/base/segment_companies/base_exchange__segment_companies.sql b/models/exchange/base/segment_companies/base_exchange__segment_companies.sql new file mode 100644 index 0000000..61867d0 --- /dev/null +++ b/models/exchange/base/segment_companies/base_exchange__segment_companies.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH segment_companies AS ( + SELECT * FROM {{ ref('snap_exchange__segment_companies') }} +), + + +-- LOGIC +segment_companies_rename AS ( + + SELECT + segment_companies.concat_id, + segment_companies.segment_id, + segment_companies.company_id, + segment_companies.deleted_at, + segment_companies.created_at, + segment_companies.updated_at, + segment_companies.dbt_valid_from, + segment_companies.dbt_valid_to, + current_timestamp() AS _dbt_ran_at + + FROM segment_companies +), + + +-- FINAL +final__base_exchange__segment_companies AS ( + + SELECT + concat_id, + segment_id, + company_id, + deleted_at, + created_at, + updated_at, + dbt_valid_from, + dbt_valid_to, + _dbt_ran_at + + FROM segment_companies_rename + +) + +SELECT * FROM final__base_exchange__segment_companies \ No newline at end of file diff --git a/models/exchange/base/segment_constants/base_exchange__segment_constants.sql b/models/exchange/base/segment_constants/base_exchange__segment_constants.sql new file mode 100644 index 0000000..19adb06 --- /dev/null +++ b/models/exchange/base/segment_constants/base_exchange__segment_constants.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH segment_constants AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'segment_constants') }} +), + + +-- LOGIC +segment_constants_rename AS ( + + SELECT + segment_constants.id, + segment_constants.segment_id, + segment_constants.name, + segment_constants.reference AS marking_id, + segment_constants.detail, + segment_constants.deleted_at, + segment_constants.created_at, + segment_constants.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM segment_constants +), + + +-- FINAL +final_base_exchange__segment_constants AS ( + + SELECT + id, + segment_id, + name, + marking_id, + detail, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM segment_constants_rename + +) + +SELECT * FROM final_base_exchange__segment_constants \ No newline at end of file diff --git a/models/exchange/base/segments/base_exchange__segments.sql b/models/exchange/base/segments/base_exchange__segments.sql new file mode 100644 index 0000000..451517a --- /dev/null +++ b/models/exchange/base/segments/base_exchange__segments.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH segments AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'segments') }} +), + + +-- LOGIC +segments_rename AS ( + + SELECT + segments.id, + segments.name, + segments.type, + segments.status, + segments.deleted_at, + segments.created_at, + segments.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM segments +), + + +-- FINAL +final__base_exchange__segments AS ( + + SELECT + id, + name, + type, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM segments_rename +) + +SELECT * FROM final__base_exchange__segments \ No newline at end of file diff --git a/models/exchange/base/service_types/base_exchange__service_types.sql b/models/exchange/base/service_types/base_exchange__service_types.sql new file mode 100644 index 0000000..ce148a5 --- /dev/null +++ b/models/exchange/base/service_types/base_exchange__service_types.sql @@ -0,0 +1,39 @@ +-- IMPORTS +WITH service_types AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'service_types') }} +), + + +-- LOGIC +service_types_rename AS ( + + SELECT + service_types.id, + service_types.name, + service_types.status, + service_types.deleted_at, + service_types.created_at, + service_types.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM service_types +), + + +-- FINAL +final__base_exchange__service_types AS ( + + SELECT + id, + name, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM service_types_rename + +) + +SELECT * FROM final__base_exchange__service_types \ No newline at end of file diff --git a/models/exchange/base/states/base_exchange__states.sql b/models/exchange/base/states/base_exchange__states.sql new file mode 100644 index 0000000..202bc57 --- /dev/null +++ b/models/exchange/base/states/base_exchange__states.sql @@ -0,0 +1,41 @@ +-- IMPORTS +WITH states AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'states') }} +), + + +-- LOGIC +states_rename AS ( + + SELECT + states.id, + states.country_id, + states.name, + states.status, + states.deleted_at, + states.created_at, + states.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM states +), + + +-- FINAL +final__base_exchange__states AS ( + + SELECT + id, + country_id, + name, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM states_rename + +) + +SELECT * FROM final__base_exchange__states \ No newline at end of file diff --git a/models/exchange/base/transaction_details/base_exchange__transaction_details.sql b/models/exchange/base/transaction_details/base_exchange__transaction_details.sql new file mode 100644 index 0000000..edc6071 --- /dev/null +++ b/models/exchange/base/transaction_details/base_exchange__transaction_details.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH transaction_details AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'transaction_detail') }} +), + + +-- LOGIC +transaction_details_rename AS ( + + SELECT + transaction_details.id, + transaction_details.transaction_id, + transaction_details.product_code, + transaction_details.product_name, + transaction_details.quantity, + transaction_details.price, + transaction_details.amount, + transaction_details.deleted_at, + transaction_details.created_at, + transaction_details.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transaction_details + +), + + +-- FINAL +final__base_exchange__transaction_details AS ( + + SELECT + id, + transaction_id, + product_code, + product_name, + quantity, + price, + amount, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transaction_details_rename + +) + +SELECT * FROM final__base_exchange__transaction_details \ No newline at end of file diff --git a/models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql b/models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql new file mode 100644 index 0000000..edc4eef --- /dev/null +++ b/models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql @@ -0,0 +1,73 @@ +-- IMPORTS +WITH transaction_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'transaction_logs') }} +), + + +-- LOGIC +transaction_logs_rename AS ( + + SELECT + transaction_logs.id, + transaction_logs.transaction_id, + transaction_logs.owner_type, + transaction_logs.owner_id, + transaction_logs.type AS transaction_type, + transaction_logs.issuer AS issued_by, + transaction_logs.receiver AS received_by, + transaction_logs.recipient_bank_account_id, + transaction_logs.payment_method, + transaction_logs.payment_reference, + transaction_logs.bill_no, + transaction_logs.amount AS base_value, + transaction_logs.original_amount AS quote_value, + transaction_logs.currency_id AS base_currency_id, + transaction_logs.original_currency_id AS quote_currency_id, + transaction_logs.currency_rate AS base_to_quote_currency_exchange_rate, + transaction_logs.tax AS base_tax, + transaction_logs.service_charge AS base_service_charge, + transaction_logs.expires_on AS expired_at, + transaction_logs.status, + transaction_logs.deleted_at, + transaction_logs.created_at, + transaction_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transaction_logs +), + + +-- FINAL +final__base_exchange__transaction_logs AS ( + + SELECT + id, + transaction_id, + owner_type, + owner_id, + transaction_type, + issued_by, + received_by, + recipient_bank_account_id, + payment_method, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transaction_logs_rename + +) + +SELECT * FROM final__base_exchange__transaction_logs \ No newline at end of file diff --git a/models/exchange/base/transactions/base_exchange__transactions.sql b/models/exchange/base/transactions/base_exchange__transactions.sql new file mode 100644 index 0000000..59b3d4f --- /dev/null +++ b/models/exchange/base/transactions/base_exchange__transactions.sql @@ -0,0 +1,71 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'transactions') }} +), + + +-- LOGIC +transactions_rename AS ( + + SELECT + transactions.id, + transactions.owner_type, + transactions.owner_id, + transactions.type AS transaction_type, + transactions.issuer AS issued_by, + transactions.receiver AS received_by, + transactions.recipient_bank_account_id, + transactions.payment_method, + transactions.payment_reference, + transactions.bill_no, + transactions.amount AS base_value, + transactions.original_amount AS quote_value, + transactions.currency_id AS base_currency_id, + transactions.original_currency_id AS quote_currency_id, + transactions.currency_rate AS base_to_quote_currency_exchange_rate, + transactions.tax AS base_tax, + transactions.service_charge AS base_service_charge, + transactions.expires_on AS expired_at, + transactions.status, + transactions.deleted_at, + transactions.created_at, + transactions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transactions +), + + +-- FINAL +final__base_exchange__transactions AS ( + + SELECT + id, + owner_type, + owner_id, + transaction_type, + issued_by, + received_by, + recipient_bank_account_id, + payment_method, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions_rename + +) + +SELECT * FROM final__base_exchange__transactions \ No newline at end of file diff --git a/models/exchange/base/user_email_verifications/base_exchange__user_email_verifications.sql b/models/exchange/base/user_email_verifications/base_exchange__user_email_verifications.sql new file mode 100644 index 0000000..b5e2d38 --- /dev/null +++ b/models/exchange/base/user_email_verifications/base_exchange__user_email_verifications.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH user_email_verifications AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'user_email_verifications') }} +), + + +-- LOGIC +user_email_verifications_rename AS ( + + SELECT + user_email_verifications.id, + user_email_verifications.email, + user_email_verifications.token, + user_email_verifications.is_complete AS is_completed, + user_email_verifications.is_active AS is_activated, + user_email_verifications.is_sent, + user_email_verifications.created_at, + user_email_verifications.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM user_email_verifications +), + + +-- FINAL +final__base_exchange__user_email_verifications AS ( + + SELECT + id, + email, + token, + is_completed, + is_activated, + is_sent, + created_at, + updated_at, + _dbt_ran_at + + FROM user_email_verifications_rename + +) + +SELECT * FROM final__base_exchange__user_email_verifications \ No newline at end of file diff --git a/models/exchange/base/users/base_exchange__users.sql b/models/exchange/base/users/base_exchange__users.sql new file mode 100644 index 0000000..6d4caf1 --- /dev/null +++ b/models/exchange/base/users/base_exchange__users.sql @@ -0,0 +1,47 @@ +-- IMPORTS +WITH users AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'users') }} +), + + +-- LOGIC +users_rename AS ( + + SELECT + users.id, + users.name, + users.email, + users.type AS user_type, + users.status, + users.remember_token, + users.active_at, + users.deleted_at, + users.created_at, + users.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM users +), + + +-- FINAL +final__base_exchange__users AS ( + + SELECT + id, + name, + email, + user_type, + status, + remember_token, + active_at, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM users_rename + +) + +SELECT * FROM final__base_exchange__users \ No newline at end of file diff --git a/models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql b/models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql new file mode 100644 index 0000000..8f2c21b --- /dev/null +++ b/models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql @@ -0,0 +1,47 @@ +-- IMPORTS +WITH wallet_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'wallet_logs') }} +), + + +-- LOGIC +wallet_logs_rename AS ( + + SELECT + wallet_logs.id, + wallet_logs.wallet_id, + wallet_logs.owner_type, + wallet_logs.owner_id, + wallet_logs.code, + wallet_logs.currency_id, + wallet_logs.amount, + wallet_logs.deleted_at, + wallet_logs.created_at, + wallet_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM wallet_logs +), + + +-- FINAL +final__base_exchange__wallet_logs AS ( + + SELECT + id, + wallet_id, + owner_type, + owner_id, + code, + currency_id, + amount, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM wallet_logs_rename + +) + +SELECT * FROM final__base_exchange__wallet_logs \ No newline at end of file diff --git a/models/exchange/base/wallets/base_exchange__wallets.sql b/models/exchange/base/wallets/base_exchange__wallets.sql new file mode 100644 index 0000000..2f2cb9f --- /dev/null +++ b/models/exchange/base/wallets/base_exchange__wallets.sql @@ -0,0 +1,45 @@ +-- IMPORTS +WITH wallets AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'wallets') }} +), + + +-- LOGIC +wallets_rename AS ( + + SELECT + wallets.id, + wallets.owner_type, + wallets.owner_id, + wallets.code, + wallets.currency_id, + wallets.amount, + wallets.deleted_at, + wallets.created_at, + wallets.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM wallets +), + + +-- FINAL +final__base_exchange__wallets AS ( + + SELECT + id, + owner_type, + owner_id, + code, + currency_id, + amount, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM wallets_rename + +) + +SELECT * FROM final__base_exchange__wallets \ No newline at end of file diff --git a/models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql b/models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql new file mode 100644 index 0000000..7aaeab0 --- /dev/null +++ b/models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql @@ -0,0 +1,112 @@ +-- IMPORTS +WITH booking_logs AS ( + SELECT * FROM {{ ref('stg_exchange__booking_logs') }} +), + +currency_rate_logs AS ( + SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }} +), + + +-- LOGIC +booking_logs_calculate_value_join_currency_rate_logs AS ( + + SELECT + booking_logs.id, + booking_logs.booking_id, + booking_logs.company_id, + booking_logs.user_id, + booking_logs.marking_id, + booking_logs.service_type, + booking_logs.service_type_name, + booking_logs.bank_id, + booking_logs.fix_value, + booking_logs.fix_currency_id, + booking_logs.fix_currency_name, + booking_logs.fix_currency_short_code, + + currency_rate_logs.payment_method AS estimate_payment_method, + currency_rate_logs.payment_method_name AS estimate_payment_method_name, + currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate, + + booking_logs.quote_currency_id, + booking_logs.quote_currency_name, + booking_logs.quote_currency_short_code, + IFF(booking_logs.quote_currency_id = booking_logs.fix_currency_id, + fix_value, + ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value, + + booking_logs.base_currency_id, + booking_logs.base_currency_name, + booking_logs.base_currency_short_code, + IFF(booking_logs.base_currency_id = booking_logs.fix_currency_id, + fix_value, + ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value, + + -- If we have quote & base value, both of them are not contain 1 (MYR), then need to redo this function + CASE + WHEN (booking_logs.base_currency_id = '1') + THEN ROUND(estimate_base_value,2) + WHEN (booking_logs.quote_currency_id = '1') + THEN ROUND(estimate_quote_value,2) + ELSE + NULL + END AS estimate_value_rm, + + booking_logs.status, + booking_logs.status_name, + booking_logs.deleted_at, + booking_logs.created_at, + booking_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM booking_logs + + LEFT JOIN currency_rate_logs + ON (booking_logs.created_at >= currency_rate_logs.created_date_from) + AND (booking_logs.created_at < currency_rate_logs.created_date_to) + AND (booking_logs.service_type = currency_rate_logs.service_type) + AND (booking_logs.quote_currency_id = currency_rate_logs.quote_currency_id) + AND (currency_rate_logs.payment_method = '1') +), + + +-- FINAL +final__int_booking_logs_get_estimate_rates__exchange AS ( + + SELECT + id, + booking_id, + company_id, + user_id, + marking_id, + service_type, + service_type_name, + bank_id, + fix_value, + fix_currency_id, + fix_currency_name, + fix_currency_short_code, + estimate_payment_method, + estimate_payment_method_name, + estimate_base_to_quote_currency_exchange_rate, + base_currency_id, + base_currency_name, + base_currency_short_code, + estimate_base_value, + estimate_value_rm, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + estimate_quote_value, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM booking_logs_calculate_value_join_currency_rate_logs +) + +SELECT * FROM final__int_booking_logs_get_estimate_rates__exchange \ No newline at end of file diff --git a/models/exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql b/models/exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql new file mode 100644 index 0000000..3360294 --- /dev/null +++ b/models/exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql @@ -0,0 +1,82 @@ +-- IMPORTS +WITH bookings AS ( + SELECT * FROM {{ ref('stg_exchange__bookings') }} +), + +currency_rate_logs AS ( + SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }} +), + + +-- LOGIC +bookings_calculate_value_join_currency_rate_logs AS ( + + SELECT + bookings.id, + bookings.company_id, + bookings.user_id, + bookings.marking_id, + bookings.service_type, + bookings.service_type_name, + bookings.bank_id, + bookings.fix_value, + bookings.fix_currency_id, + bookings.fix_currency_name, + bookings.fix_currency_short_code, + + currency_rate_logs.payment_method AS estimate_payment_method, + currency_rate_logs.payment_method_name AS estimate_payment_method_name, + currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate, + + bookings.quote_currency_id, + bookings.quote_currency_name, + bookings.quote_currency_short_code, + IFF(bookings.quote_currency_id = bookings.fix_currency_id, + fix_value, + ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value, + + bookings.base_currency_id, + bookings.base_currency_name, + bookings.base_currency_short_code, + IFF(bookings.base_currency_id = bookings.fix_currency_id, + fix_value, + ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value, + + -- If we have quote & base value, both of them are not contain 1 (MYR), then need to redo this function + CASE + WHEN (bookings.base_currency_id = '1') + THEN ROUND(estimate_base_value,2) + WHEN (bookings.quote_currency_id = '1') + THEN ROUND(estimate_quote_value,2) + ELSE + NULL + END AS estimate_value_rm, + + bookings.status, + bookings.status_name, + bookings.booking_deleted_datetime, + bookings.booking_created_datetime, + bookings.booking_updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM bookings + + LEFT JOIN currency_rate_logs + ON (bookings.booking_created_datetime >= currency_rate_logs.created_date_from) + AND (bookings.booking_created_datetime < currency_rate_logs.created_date_to) + AND (bookings.service_type = currency_rate_logs.service_type) + AND (bookings.quote_currency_id = currency_rate_logs.quote_currency_id) + AND (currency_rate_logs.payment_method = '1') +), + + +-- FINAL +final__int_bookings_get_estimate_rates__exchange AS ( + + SELECT + * + + FROM bookings_calculate_value_join_currency_rate_logs +) + +SELECT * FROM final__int_bookings_get_estimate_rates__exchange \ No newline at end of file diff --git a/models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql b/models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql new file mode 100644 index 0000000..4dafd07 --- /dev/null +++ b/models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql @@ -0,0 +1,81 @@ +-- Import +WITH companies AS ( + SELECT * FROM {{ ref('stg_exchange__companies') }} +), + +segments AS ( + SELECT * FROM {{ ref('stg_exchange__segments') }} +), + +companies_and_segments_relations AS ( + SELECT * FROM {{ ref('stg_exchange__companies_and_segments_relations') }} +), + + +-- LOGIC +get_is_migrated_companies AS ( + SELECT + segment_id, + company_id + + FROM companies_and_segments_relations + + WHERE + segment_id = 5 +), + + +-- Because we do not have timestamp to get maximum segment_id +get_segment_row_created_sequences AS ( + SELECT + *, + ROW_NUMBER() OVER (ORDER BY NULL) AS row_created_sequence + + FROM companies_and_segments_relations + + WHERE + deleted_at IS NULL + AND + segment_id IN (1,2,3,4) + AND + dbt_valid_to IS NULL +), + + +get_company_latest_segments AS ( + SELECT + *, + row_number() OVER + (PARTITION BY company_id + ORDER BY row_created_sequence DESC) AS is_latest_segment + + FROM get_segment_row_created_sequences + + QUALIFY + is_latest_segment = 1 +), + + +-- FINAL +final__int_exchange__companies_level_segment AS ( + SELECT + companies.*, + get_company_latest_segments.segment_id, + segments.name AS segment_name, + IFF(get_is_migrated_companies.company_id IS NOT NULL, 1, 0) AS is_migrated_company + + FROM companies + + LEFT JOIN get_company_latest_segments + ON (companies.id = get_company_latest_segments.company_id) + + LEFT JOIN segments + ON (get_company_latest_segments.segment_id = segments.id) + + LEFT JOIN get_is_migrated_companies + ON (companies.id = get_is_migrated_companies.company_id) + +) + +SELECT * FROM final__int_exchange__companies_level_segment +-- select * from get_is_migrated_company \ No newline at end of file diff --git a/models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql b/models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql new file mode 100644 index 0000000..c102d88 --- /dev/null +++ b/models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql @@ -0,0 +1,45 @@ +-- IMPORTS +WITH user_email_verifications AS ( + SELECT * FROM {{ ref('stg_exchange__user_email_verifications')}} +), + + +-- LOGIC +dedupe_user_email_verifications AS ( + SELECT + user_id, + + -- Function below search for '1' in is_complete groupconcat string, if found then true, if null than null, else false + IFF(CONTAINS( LISTAGG(is_completed, ',') WITHIN GROUP(ORDER BY is_completed), '1'),'1','0') AS is_completed, + IFF(CONTAINS( LISTAGG(is_activated, ',') WITHIN GROUP(ORDER BY is_activated), '1'),'1','0') AS is_activated, + IFF(CONTAINS( LISTAGG(is_sent, ',') WITHIN GROUP(ORDER BY is_sent), '1'), '1', '0') AS is_sent, + COUNT(*) AS email_sent, + MIN(created_at) AS first_email_created_at, + MAX(created_at) AS last_email_created_at, + current_timestamp() AS _dbt_ran_at + + FROM user_email_verifications + + GROUP BY + user_id + +), + + +-- FINAL +final__int_exchange__dedupe_user_email_verifications AS ( + + SELECT + user_id, + is_completed, + is_activated, + is_sent, + email_sent, + first_email_created_at, + last_email_created_at, + _dbt_ran_at + FROM + dedupe_user_email_verifications +) + +SELECT * FROM final__int_exchange__dedupe_user_email_verifications \ No newline at end of file diff --git a/models/exchange/intermediate/int_exchange__latest_company_contacts.sql b/models/exchange/intermediate/int_exchange__latest_company_contacts.sql new file mode 100644 index 0000000..23b3b7e --- /dev/null +++ b/models/exchange/intermediate/int_exchange__latest_company_contacts.sql @@ -0,0 +1,38 @@ +--IMPORT +WITH contacts AS ( + SELECT * FROM {{ ref('stg_exchange__contacts') }} +), + +--LOGIC +get_latest_company_contacts AS ( + SELECT + *, + row_number() OVER + (PARTITION BY company_id + ORDER BY contact_created_datetime DESC) AS last_row_number + + FROM contacts + + QUALIFY + last_row_number = 1 +), + +--FINAL +final__int_exchange__latest_company_contacts AS ( + SELECT + id, + company_id, + country_id, + country_name, + reference, + phone, + email, + wechat_id, + contact_deleted_datetime, + contact_created_datetime, + contact_updated_datetime + + FROM get_latest_company_contacts +) + +SELECT * FROM final__int_exchange__latest_company_contacts diff --git a/models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql b/models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql new file mode 100644 index 0000000..2c8054e --- /dev/null +++ b/models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql @@ -0,0 +1,72 @@ +-- IMPORT +WITH transaction_costs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} +), + + +-- LOGIC +duplicate_id_need_remove AS ( + SELECT + ID, + TRANSACTION_ORDER_ID, + BASE_VALUE, + QUOTE_VALUE, + BASE_TO_QUOTE_CURRENCY_EXCHANGE_RATE, + STATUS_NAME, + CREATED_AT, + UPDATED_AT, + row_number() OVER + (PARTITION BY transaction_costs.transaction_order_id + ORDER BY transaction_costs.updated_at DESC) AS row_index + + FROM + transaction_costs + + WHERE + deleted_at IS NULL + + QUALIFY + row_index = 2 +), + + +-- duplicate_id_need_remove AS ( +-- SELECT +-- id + +-- FROM mark_duplicate_is_not_deleted_id + +-- WHERE +-- row_index = 2 + +-- ), + + +removed_duplicate_id_table AS ( + SELECT + transaction_costs.* + + FROM transaction_costs + + LEFT JOIN duplicate_id_need_remove + ON (transaction_costs.id = duplicate_id_need_remove.id) + WHERE + duplicate_id_need_remove.id IS NULL +), + + +--FINAL +final__int_exchange__transaction_costs_remove_duplicate_row AS ( + SELECT * FROM removed_duplicate_id_table +) + +SELECT * FROM final__int_exchange__transaction_costs_remove_duplicate_row + + + +-- SELECT * FROM duplicate_transaction_order_id_which_not_deleted +-- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at +-- SELECT * FROM duplicate_id_need_remove +-- SELECT * FROM removed_duplicate_id_table WHERE deleted_at is not null +-- SELECT * FROM mark_duplicate_is_not_deleted_id WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at +-- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at diff --git a/models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql b/models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql new file mode 100644 index 0000000..51134fc --- /dev/null +++ b/models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql @@ -0,0 +1,28 @@ +-- IMPORTS +WITH transaction_log_orders AS ( + SELECT * FROM {{ ref('int_exchange__transaction_log_orders_remove_undo_status') }} +), + +bookings AS ( + SELECT * FROM {{ ref('stg_exchange__bookings') }} +), + +-- LOGICS +transaction_log_orders_join_bookings AS ( + SELECT + transaction_log_orders.*, + bookings.user_id + + FROM transaction_log_orders + + LEFT JOIN bookings + ON (transaction_log_orders.booking_id = bookings.id) +), + + +-- FINAL +final__int_exchange__transaction_log_orders_get_user_company_ids AS ( + SELECT * FROM transaction_log_orders_join_bookings +) + +SELECT * FROM final__int_exchange__transaction_log_orders_get_user_company_ids \ No newline at end of file diff --git a/models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql b/models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql new file mode 100644 index 0000000..0e6d5cc --- /dev/null +++ b/models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql @@ -0,0 +1,26 @@ +-- IMPORT +WITH transaction_log_orders AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_log_orders') }} +), + + +-- LOGIC +transaction_log_orders_mark_undo_status AS ( + SELECT + * + + FROM transaction_log_orders + + QUALIFY + row_number() OVER + (PARTITION BY transaction_log_orders.status, transaction_log_orders.transaction_order_id + ORDER BY transaction_log_orders.updated_at DESC) = 1 +), + + +-- FINAL +final__int_exchange__transaction_log_orders_remove_undo_status AS ( + SELECT * FROM transaction_log_orders_mark_undo_status +) + +SELECT * FROM final__int_exchange__transaction_log_orders_remove_undo_status \ No newline at end of file diff --git a/models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql b/models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql new file mode 100644 index 0000000..6e0ec76 --- /dev/null +++ b/models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql @@ -0,0 +1,30 @@ +-- IMPORTS +WITH transaction_orders AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_orders') }} +), + +bookings AS ( + SELECT * FROM {{ ref('stg_exchange__bookings') }} +), + +-- LOGICS +transaction_orders_join_bookings AS ( + SELECT + transaction_orders.*, + bookings.user_id + + + + FROM transaction_orders + + LEFT JOIN bookings + ON (transaction_orders.booking_id = bookings.id) +), + + +-- FINAL +final__int_exchange__transaction_orders_get_user_company_ids AS ( + SELECT * FROM transaction_orders_join_bookings +) + +SELECT * FROM final__int_exchange__transaction_orders_get_user_company_ids \ No newline at end of file diff --git a/models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql b/models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql new file mode 100644 index 0000000..1fbbb02 --- /dev/null +++ b/models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql @@ -0,0 +1,234 @@ +-- IMPORT +WITH transaction_log_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_log_orders') }} +), + +latest_transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + +booking_logs AS ( + SELECT * FROM {{ ref('fct_exchange__booking_logs') }} +), + +latest_bookings AS ( + SELECT * FROM {{ ref('fct_exchange__bookings') }} +), + +latest_transaction_costs AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_costs') }} +), + +transaction_log_costs AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_log_costs') }} +), + +companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +dates AS ( + SELECT * FROM {{ ref('dim_exchange__dates') }} +), + + +--LOGIC +transaction_order_logs_status_datetime AS ( + + SELECT + transaction_order_id, + MIN(created_at) AS created_datetime, + MAX(CASE WHEN status = '0' THEN updated_at END) AS pending_submission_datetime, + MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime, + MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime, + MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime, + MAX(CASE WHEN status = '4' THEN updated_at END) AS rejected_datetime, + MAX(CASE WHEN status = '5' THEN updated_at END) AS suspended_datetime + + FROM transaction_log_orders + + GROUP BY + transaction_order_id +), + + +cost_logs_status_datetime AS ( + + SELECT + transaction_order_id, + MIN(created_at) AS created_datetime, + MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime, + MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime, + MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime + + FROM transaction_log_costs + + GROUP BY + transaction_order_id +), + + +latest_transaction_costs_remove_deleted AS ( + SELECT + * + + FROM + latest_transaction_costs + + WHERE + deleted_at IS NULL +), + + +previous_order_date_row_rank AS ( + SELECT + *, + row_number() OVER + (PARTITION BY latest_transaction_orders.company_id + ORDER BY latest_transaction_orders.created_at) AS row_rank_index + + FROM + latest_transaction_orders +), + +get_previous_order_date AS ( + SELECT + previous_order_date_row_rank.id, + previous_order_date_row_rank_clone.created_at AS previous_order_created_datetime + + FROM previous_order_date_row_rank + + LEFT JOIN previous_order_date_row_rank AS previous_order_date_row_rank_clone + ON previous_order_date_row_rank.row_rank_index = previous_order_date_row_rank_clone.row_rank_index + 1 + AND previous_order_date_row_rank.company_id = previous_order_date_row_rank_clone.company_id +), + +--FINAL +final__rep_exchange__daily_transactions AS ( + SELECT + latest_transaction_orders.id, + latest_transaction_orders.booking_id, + + latest_bookings.marking_id AS booking_marking_id, + latest_bookings.service_type, + latest_bookings.service_type_name, + latest_bookings.booking_created_datetime, + + latest_transaction_orders.seller_company_id, + latest_transaction_orders.bank_id, + latest_transaction_orders.user_id, + latest_transaction_orders.company_id, + + companies.marking_id AS company_marking_id, + companies.company_type, + companies.company_type_name, + companies.business_type AS company_business_type, + companies.business_type_name AS company_business_type_name, + companies.segment_id AS company_segment_id, + companies.segment_name AS company_segment_name, + companies.is_migrated_company AS company_is_migrated_company, + companies.country_id AS company_country_id, + companies.country_name AS company_country_name, + companies.state_id AS company_state_id, + companies.state_name AS company_state_name, + companies.district_id AS company_district_id, + companies.district_name AS company_district_name, + companies.postcode AS company_postcode, + companies.wallet_id AS company_wallet_id, + companies.have_wallet AS company_have_wallet, + -- i stay here + latest_bookings.is_first_time_user AS is_first_time_booking_user, + latest_bookings.is_first_time_company AS is_first_time_booking_company, + latest_transaction_orders.is_first_time_user, + latest_transaction_orders.is_first_time_company, + + latest_transaction_orders.status, + latest_transaction_orders.status_name, + latest_transaction_orders.transaction_type, + latest_transaction_orders.transaction_type_name, + latest_transaction_orders.payment_method, + latest_transaction_orders.payment_method_name, + latest_transaction_orders.base_currency_id, + latest_transaction_orders.base_currency_name, + latest_transaction_orders.quote_currency_id, + latest_transaction_orders.quote_currency_name, + latest_transaction_orders.base_to_quote_currency_exchange_rate, + latest_transaction_orders.base_value, + latest_transaction_orders.quote_value, + latest_transaction_orders.transaction_value_rm, + latest_transaction_orders.transaction_base_tax, + latest_transaction_orders.transaction_tax_rm, + latest_transaction_orders.base_service_charge, + latest_transaction_orders.transaction_service_charge_rm, + + get_previous_order_date.previous_order_created_datetime AS previous_order_created_datetime, + transaction_order_logs_status_datetime.created_datetime AS order_created_datetime, + order_created_dates.date_actual AS order_created_date, + order_created_dates.first_day_of_week AS order_created_week, + + transaction_order_logs_status_datetime.pending_submission_datetime AS order_pending_submission_datetime, + transaction_order_logs_status_datetime.pending_verification_datetime AS order_pending_verification_datetime, + transaction_order_logs_status_datetime.approved_datetime AS order_approved_datetime, + transaction_order_logs_status_datetime.completed_datetime AS order_completed_datetime, + transaction_order_logs_status_datetime.rejected_datetime AS order_rejected_datetime, + transaction_order_logs_status_datetime.suspended_datetime AS order_suspended_datetime, + + latest_transaction_costs_remove_deleted.id AS cost_id, + latest_transaction_costs_remove_deleted.supplier_company_id AS cost_supplier_company_id, + latest_transaction_costs_remove_deleted.status AS cost_status, + latest_transaction_costs_remove_deleted.status_name AS cost_status_name, + latest_transaction_costs_remove_deleted.base_currency_id AS cost_base_currency_id, + latest_transaction_costs_remove_deleted.base_currency_name AS cost_base_currency_name, + latest_transaction_costs_remove_deleted.quote_currency_id AS cost_quote_currency_id, + latest_transaction_costs_remove_deleted.quote_currency_name AS cost_quote_currency_name, + latest_transaction_costs_remove_deleted.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate, + latest_transaction_costs_remove_deleted.base_value AS cost_base_value, + latest_transaction_costs_remove_deleted.quote_value AS cost_quote_value, + latest_transaction_costs_remove_deleted.transaction_value_rm AS cost_transaction_value_rm, + latest_transaction_costs_remove_deleted.transaction_base_tax AS cost_transaction_base_tax, + latest_transaction_costs_remove_deleted.transaction_tax_rm AS cost_transaction_tax_rm, + latest_transaction_costs_remove_deleted.base_service_charge AS cost_base_service_charge, + latest_transaction_costs_remove_deleted.transaction_service_charge_rm AS cost_transaction_service_charge_rm, + + cost_logs_status_datetime.created_datetime AS cost_created_datetime, + cost_logs_status_datetime.pending_verification_datetime AS cost_pending_verification_datetime, + cost_logs_status_datetime.approved_datetime AS cost_approved_datetime, + cost_logs_status_datetime.completed_datetime AS cost_completed_datetime + + FROM latest_transaction_orders + + LEFT JOIN latest_bookings + ON (latest_transaction_orders.booking_id = latest_bookings.id) + + LEFT JOIN transaction_order_logs_status_datetime + ON (latest_transaction_orders.id = transaction_order_logs_status_datetime.transaction_order_id) + + LEFT JOIN latest_transaction_costs_remove_deleted + ON (latest_transaction_orders.id = latest_transaction_costs_remove_deleted.transaction_order_id) + + LEFT JOIN cost_logs_status_datetime + ON (latest_transaction_orders.id = cost_logs_status_datetime.transaction_order_id) + + LEFT JOIN companies + ON (latest_transaction_orders.company_id = companies.id) + + LEFT JOIN get_previous_order_date + ON(latest_transaction_orders.id = get_previous_order_date.id) + + LEFT JOIN dates AS order_created_dates + ON (date(transaction_order_logs_status_datetime.created_datetime) = order_created_dates.date_day ) + + ORDER BY latest_transaction_orders.updated_at ASC +) + + +SELECT * FROM final__rep_exchange__daily_transactions +-- @YAM: not unique order id +--(2007, 40808, 2013, 40920, 41101, 125169, 18604, 18552, 18550, 35795, 117444, 72332, 40798, 51174, 18553, 2010, 66195, 89413, 1678, 71516, 5673, 78366, 66208, 18546, 1672, 1684, 18579, 66202, 132074, 18554, 43249, 40804, 86259, 4881, 2311, 103084, 18549, 41097, 106321, 17634, 66207, 18582, 1661, 63970) and so on.... + + +-- (OMAIR) +-- SELECT booking_id, count(*) FROM booking_logs WHERE status= 3 GROUP BY booking_id ORDER BY count(*) desc +-- SELECT booking_id, company_id, user_id, fix_value, status_name, updated_at FROM booking_logs WHERE booking_id IN (18901,18618,22960) ORDER BY booking_id, updated_at +-- SELECT * FROM booking_logs WHERE booking_id = 22173 ORDER BY updated_at + \ No newline at end of file diff --git a/models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml b/models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml new file mode 100644 index 0000000..f1aa2f9 --- /dev/null +++ b/models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml @@ -0,0 +1,211 @@ +version: 2 + +models: + - name: rep_exchange__daily_orders + columns: + - name: id + description: transaction orders id + tests: + - unique + - not_null + + - name: booking_id + description: transaction orders id + tests: + - not_null + - relationships: + to: ref('base_exchange__bookings') + field: id + + - name: service_type + description: type of service + tests: + - not_null + + - name: service_type_name + description: name of type of service + tests: + - not_null + + - name: booking_created_datetime + description: date of boking created + tests: + - not_null + + - name: seller_company_id + description: seller company id + tests: + - not_null + + - name: bank_id + description: bank id + tests: + - not_null + + - name: company_id + description: user id + tests: + - not_null + + - name: company_marking_id + description: company marking id + tests: + - relationships: + to: ref('base_exchange__companies') + field: marking_id + + - name: company_type + description: type of company + tests: + - not_null + + - name: company_type_name + description: name type of company + tests: + - accepted_values: + values: + - PERSONAL + - CORPORATE + + - name: company_business_type + description: company business type + tests: + - not_null + + - name: company_business_type_name + description: company business type name + tests: + - accepted_values: + values: + - IMPORTER + + - name: company_segment_id + description: company segment id + tests: + - not_null + + - name: company_segment_name + description: company segment name + tests: + - accepted_values: + values: + - PLATINIUM MEMBER + - GOLD MEMBER + - STANDARD SEGMENT + + - name: cost_id + description: cost id + tests: + - relationships: + to: ref('base_exchange__transactions') + field: id + + - name: base_value + description: test base value combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: quote_value + description: test quote value combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: transaction_value_rm + description: transaction value rm combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: transaction_base_tax + description: test transaction base tax combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: base_service_charge + description: test base service charge combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: transaction_tax_rm + description: test transaction tax rm combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_base_value + description: test cost base value combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_base_to_quote_currency_exchange_rate + description: test cost base to quote currency exchange rate combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_quote_value + description: test cost quote value combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_transaction_value_rm + description: test COST TRANSACTION VALUE RM combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_transaction_tax_rm + description: test COST TRANSACTION TAX RM combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_base_service_charge + description: test COST BASE SERVICE CHARGE combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: cost_transaction_service_charge_rm + description: test COST TRANSACTION SERVICE CHARGE_RM combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + - name: transaction_service_charge_rm + description: test transaction service charge rm combine with dbt_utils + tests: + - not_null + - dbt_utils.expression_is_true: + expression: " >= 0" + + tests: + - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: + column_A: order_completed_datetime + column_B: order_created_datetime + or_equal: True + tests: + - dbt_utils.expression_is_true: + expression: "cost_transaction_value_rm + cost_transaction_tax_rm + cost_transaction_service_charge_rm < 2*(transaction_value_rm + transaction_base_tax + transaction_tax_rm)" + severity: warn + +# FRESHNESS SOURCE diff --git a/models/exchange/marts/warehouse/dim_exchange__companies.sql b/models/exchange/marts/warehouse/dim_exchange__companies.sql new file mode 100644 index 0000000..f115879 --- /dev/null +++ b/models/exchange/marts/warehouse/dim_exchange__companies.sql @@ -0,0 +1,178 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ ref('int_exchange__companies_level_segment') }} +), + +company_addresses AS ( + SELECT * FROM {{ ref('stg_exchange__company_addresses') }} +), + +current_wallets AS ( + SELECT * FROM {{ ref('stg_exchange__current_wallets') }} +), + +companies_and_users_relations AS ( + SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }} +), + +company_contacts AS ( + SELECT * FROM {{ ref('int_exchange__latest_company_contacts') }} +), + +documents AS ( + SELECT * FROM {{ ref('stg_exchange__documents') }} +), + +-- LOGICS +latest_billing_company_addresses AS ( + SELECT + *, + + -- 1 will be latest updated address, rank by updated_at date + row_number() OVER + (PARTITION BY company_addresses.company_id + ORDER BY company_addresses.updated_at DESC) AS lastest_address_rank + + FROM company_addresses + + WHERE + is_billing_address = 1 + + QUALIFY + lastest_address_rank = 1 +), + + +company_current_wallets AS ( + SELECT + id, + owner_id AS company_id, + amount, + currency_id, + currency_name, + created_at, + updated_at + + FROM current_wallets + + WHERE + owner_type = 'App\\Models\\Company' +), + + +count_company_user AS ( + SELECT + company_id, + COUNT(user_id) AS count_user + + FROM companies_and_users_relations + + GROUP BY company_id +), + +latest_company_upload_identity_documents AS ( + SELECT + id, + owner_type, + owner_id AS company_id, + document_type AS identity_document_type_name, + reference AS identity_number, + status, + status_name, + approved_at, + created_at, + + row_number() OVER + (PARTITION BY company_id + ORDER BY created_at DESC) AS latest_row_number + + FROM documents + + WHERE + owner_type = 'App\\Models\\Company' + AND + document_type IN ('IDENTITY_CARD','SSM_REGISTRATION') + AND + status = '2' + + QUALIFY + latest_row_number = '1' + +), + +-- FINAL +final__dim_exchange__companies AS ( + SELECT + companies.id, + companies.name, + companies.marking_id, + companies.account_software_id, + companies.company_type, + companies.company_type_name, + companies.business_type, + companies.business_type_name, + companies.segment_id, + companies.segment_name, + companies.is_migrated_company, + companies.status, + companies.status_name, + companies.created_at, + + company_contacts.reference AS company_contact_reference, + company_contacts.phone AS company_contact_phone, + company_contacts.email AS company_contact_email, + company_contacts.wechat_id AS company_contact_wechat_id, + + latest_billing_company_addresses.id AS address_id, + latest_billing_company_addresses.country_id, + latest_billing_company_addresses.country_name, + latest_billing_company_addresses.country_short_code, + latest_billing_company_addresses.state_id, + latest_billing_company_addresses.state_name, + latest_billing_company_addresses.district_id, + latest_billing_company_addresses.district_name, + latest_billing_company_addresses.postcode, + latest_billing_company_addresses.address_line_one, + latest_billing_company_addresses.address_line_two, + latest_billing_company_addresses.is_billing_address, + latest_billing_company_addresses.deleted_at AS address_deleted_at, + latest_billing_company_addresses.created_at AS address_created_at, + latest_billing_company_addresses.updated_at AS address_updated_at, + + company_current_wallets.id AS wallet_id, + IFF(company_current_wallets.id IS NOT NULL, 1, 0) AS have_wallet, + company_current_wallets.amount AS wallet_balance, + company_current_wallets.currency_id AS wallet_currency_id, + company_current_wallets.currency_name AS wallet_currency_name, + company_current_wallets.created_at AS wallet_created_at, + company_current_wallets.updated_at AS wallet_updated_at, + + latest_company_upload_identity_documents.identity_document_type_name, + latest_company_upload_identity_documents.identity_number, + latest_company_upload_identity_documents.status AS identity_status, + latest_company_upload_identity_documents.status_name AS identity_status_name, + latest_company_upload_identity_documents.approved_at AS identity_approved_datetime, + latest_company_upload_identity_documents.created_at AS identity_created_datetime, + + count_company_user.count_user AS number_of_user + + FROM companies + + LEFT JOIN latest_billing_company_addresses + ON (companies.id = latest_billing_company_addresses.company_id) + + LEFT JOIN company_current_wallets + ON (companies.id = company_current_wallets.company_id) + + LEFT JOIN count_company_user + ON (companies.id = count_company_user.company_id) + + LEFT JOIN company_contacts + ON (companies.id = company_contacts.company_id) + + LEFT JOIN latest_company_upload_identity_documents + ON (companies.id = latest_company_upload_identity_documents.company_id) +) + + +SELECT * FROM final__dim_exchange__companies diff --git a/models/exchange/marts/warehouse/dim_exchange__dates.sql b/models/exchange/marts/warehouse/dim_exchange__dates.sql new file mode 100644 index 0000000..5ee656f --- /dev/null +++ b/models/exchange/marts/warehouse/dim_exchange__dates.sql @@ -0,0 +1,202 @@ +-- Code and forum link refer: https://gitlab.com/gitlab-data/analytics/-/blob/master/transform/snowflake-dbt/models/sources/date/date_details_source.sql https://stackoverflow.com/questions/71016585/dbt-use-dbt-modeling-to-insert-rows-in-a-table-like-date-dimension-table-in-az + +-- Import + +-- @NOTE TO GETSON, join this into calculated +WITH holiday_and_non_working_day_lists AS ( + SELECT * FROM {{ ref('seed_exchange__holiday_and_non_working_day_lists') }} +), + +-- LOGIC +date_spine AS ( + + {{ dbt_utils.date_spine( + start_date="to_date('01/01/2015', 'mm/dd/yyyy')", + datepart="day", + end_date="dateadd(year, 40, current_date)" + ) + }} + +), + +calculated as ( + + SELECT + date_day, + date_day AS date_actual, + + DAYNAME(date_day) AS day_name, + + DATE_PART('month', date_day) AS month_actual, + DATE_PART('year', date_day) AS year_actual, + + DATE_PART(quarter, date_day) AS quarter_actual, + + CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7 + ELSE DATE_PART(dayofweek, date_day) END AS day_of_week, + + CASE WHEN day_name = 'Mon' THEN date_day + ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week, + + CASE WHEN day_name = 'Mon' THEN WEEK(date_day) + ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column + + CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1' + THEN '1' + ELSE week_of_year_temp END AS week_of_year, + + DATE_PART('day', date_day) AS day_of_month, + + ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter, + ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year, + + -- @TODO @GERSON ,done + -- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023) + CASE WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual + when DATE_PART('month', DATEADD( month,-1, date_day) ) =12 THEN year_actual-1 + + ELSE (year_actual+1) END AS fiscal_year, + + -- @TODO @GERSON , done + -- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4) + CASE WHEN month_actual in (2,3,4) THEN '1' + WHEN month_actual in (5,6,7) THEN '2' + WHEN month_actual in (8,9,10) THEN '3' + WHEN month_actual in (11,12) THEN '4' + WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4' + -- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4' + ELSE '4' END AS fiscal_quarter, + + -- @TODO @GERSON ,done + ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter, + ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year, + + TO_CHAR(date_day, 'MMMM') AS month_name, + + TRUNC(date_day, 'Month') AS first_day_of_month, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month, + + FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year, + + FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter, + + -- @TODO @GERSON ,done + -- Change this 2 too + FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter, + LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter, + + -- @TODO @GERSON,done + FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year, + LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year, + + -- @TODO @GERSON ,done + DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year, + + -- @TODO @GERSON ,done + CASE WHEN EXTRACT('month', date_day) = 1 THEN 12 + ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year, + + LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week, + + (year_actual || '-Q' || fiscal_quarter) AS quarter_name, + + -- @TODO @GERSON,done + (fiscal_year || '-' || DECODE(fiscal_quarter, + 1, 'Q1', + 2, 'Q2', + 3, 'Q3', + 4, 'Q4')) AS fiscal_quarter_name, + ('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy, + DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute, + --here check + fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name, + ('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy, + + -- @TODO @GERSON + -- Use the Seed, we will not doing the hard code in SQL, we use CSV to refer the holiday and desc + h.holiday_desc, + h.is_holiday, + h.holiday_race, + h.holiday_country, + h.holiday_state, + h.is_non_working_day, + -- @TODO @GERSON + DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter, + IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter, + DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year, + IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year, + + DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa, + DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings + + FROM date_spine s + + LEFT JOIN holiday_and_non_working_day_lists h + ON s.date_day = to_date(h.date, 'dd/mm/yyyy') + + QUALIFY date_actual >= '2016-01-01' + + ORDER BY date_day + + +), + +final AS ( + + SELECT + date_day, + date_actual, + day_name, + month_actual, + year_actual, + quarter_actual, + day_of_week, + first_day_of_week, + week_of_year, + day_of_month, + day_of_quarter, + day_of_year, + fiscal_year, + fiscal_quarter, + day_of_fiscal_quarter, + day_of_fiscal_year, + month_name, + first_day_of_month, + last_day_of_month, + first_day_of_year, + last_day_of_year, + first_day_of_quarter, + last_day_of_quarter, + first_day_of_fiscal_quarter, + last_day_of_fiscal_quarter, + first_day_of_fiscal_year, + last_day_of_fiscal_year, + week_of_fiscal_year, + month_of_fiscal_year, + last_day_of_week, + quarter_name, + fiscal_quarter_name, + fiscal_quarter_name_fy, + fiscal_quarter_number_absolute, + fiscal_month_name, + fiscal_month_name_fy, + holiday_desc, + is_holiday, + holiday_race, + holiday_country, + holiday_state, + is_non_working_day, + last_month_of_fiscal_quarter, + is_first_day_of_last_month_of_fiscal_quarter, + last_month_of_fiscal_year, + is_first_day_of_last_month_of_fiscal_year, + snapshot_date_fpa, + snapshot_date_billings, + current_timestamp() as _dbt_run_at + FROM calculated + +) + +SELECT * FROM final \ No newline at end of file diff --git a/models/exchange/marts/warehouse/dim_exchange__users.sql b/models/exchange/marts/warehouse/dim_exchange__users.sql new file mode 100644 index 0000000..7275de2 --- /dev/null +++ b/models/exchange/marts/warehouse/dim_exchange__users.sql @@ -0,0 +1,45 @@ +-- IMPORTS +WITH users AS ( + SELECT * FROM {{ ref('stg_exchange__users') }} +), + +dedupe_user_email_verifications AS ( + SELECT * FROM {{ ref('int_exchange__dedupe_user_email_verifications') }} +), + +-- LOGICS +users_join_dedupe_user_email_verifications AS ( + SELECT + users.id, + users.name, + users.email, + users.user_type, + users.user_type_name, + users.status, + users.status_name, + users.company_id, + users.created_at, + users.deleted_at, + users.updated_at, + dedupe_user_email_verifications.is_sent AS email_verification_is_sent, + dedupe_user_email_verifications.is_completed AS email_verification_is_completed, + dedupe_user_email_verifications.is_activated AS email_verification_is_activated, + dedupe_user_email_verifications.email_sent AS email_verification_email_sent, + first_email_created_at AS email_verification_first_email_created_at, + last_email_created_at AS email_verification_last_email_created_at + + + FROM users + + JOIN dedupe_user_email_verifications + ON (users.id = dedupe_user_email_verifications.user_id) +), + + +-- FINAL +final__dim_exchange__users AS ( + SELECT * FROM users_join_dedupe_user_email_verifications +) + +SELECT * +FROM final__dim_exchange__users diff --git a/models/exchange/marts/warehouse/fct_exchange__booking_logs.sql b/models/exchange/marts/warehouse/fct_exchange__booking_logs.sql new file mode 100644 index 0000000..11537d6 --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__booking_logs.sql @@ -0,0 +1,81 @@ +-- IMPORTS +WITH booking_logs AS ( + SELECT * FROM {{ ref('int_exchange__booking_logs_get_estimate_rates') }} +), + + +-- LOGICS +first_time_user_booking_id AS ( + SELECT + booking_id, + user_id, + created_at, + row_number() OVER + (PARTITION BY booking_logs.user_id + ORDER BY booking_logs.created_at) AS cummulative_count_booking + + FROM + booking_logs + + GROUP BY + booking_id, + user_id, + created_at + + QUALIFY + cummulative_count_booking = 1 +), + + +first_time_company_booking_id AS ( + SELECT + booking_id, + company_id, + created_at, + row_number() OVER + (PARTITION BY booking_logs.company_id + ORDER BY booking_logs.created_at) AS cummulative_count_booking + + FROM + booking_logs + + GROUP BY + booking_id, + company_id, + created_at + + QUALIFY + cummulative_count_booking = 1 +), + + +booking_logs_lists AS ( + SELECT + booking_logs.*, + IFF(first_time_user_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_user, + IFF(first_time_company_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_company + + FROM + booking_logs + + LEFT JOIN first_time_user_booking_id + ON (booking_logs.booking_id = first_time_user_booking_id.booking_id) + + LEFT JOIN first_time_company_booking_id + ON (booking_logs.booking_id = first_time_company_booking_id.booking_id) + + ORDER BY + created_at +), + + +-- FINAL +final_fct_exchange__booking_logs AS ( + SELECT + * + + FROM booking_logs_lists +) + +SELECT * FROM final_fct_exchange__booking_logs + diff --git a/models/exchange/marts/warehouse/fct_exchange__bookings.sql b/models/exchange/marts/warehouse/fct_exchange__bookings.sql new file mode 100644 index 0000000..34d460e --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__bookings.sql @@ -0,0 +1,60 @@ +-- IMPORTS +WITH bookings AS ( + SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_rates') }} +), + + +-- LOGICS +bookings_lists AS ( + SELECT + bookings.*, + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY bookings.company_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1 + ELSE 0 + END AS is_first_time_company, + + CASE + WHEN bookings.status = '3' + AND + MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL)) + OVER (PARTITION BY bookings.company_id) + = bookings.booking_created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_company_completed, + + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY bookings.user_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1 + ELSE 0 + END AS is_first_time_user, + + CASE + WHEN bookings.status = '3' + AND + MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL)) + OVER (PARTITION BY bookings.user_id) + = bookings.booking_created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_user_completed + + FROM + bookings + + ORDER BY + booking_created_datetime +), + + +-- FINAL +final_fct_exchange__bookings AS ( + SELECT + * + + FROM bookings_lists +) + +SELECT * FROM final_fct_exchange__bookings + diff --git a/models/exchange/marts/warehouse/fct_exchange__company_identity_documents.sql b/models/exchange/marts/warehouse/fct_exchange__company_identity_documents.sql new file mode 100644 index 0000000..526c1af --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__company_identity_documents.sql @@ -0,0 +1,39 @@ +--IMPORT +With documents AS ( + SELECT * FROM {{ ref('stg_exchange__documents') }} +), + +--LOGIC +company_identity_documents AS ( + SELECT + * + + FROM documents + + WHERE + owner_type = 'App\\Models\\Company' + AND + document_type IN ('IDENTITY_CARD','SSM_REGISTRATION') +), + + +--FINAL +final__fct_exchange__company_identity_documents AS ( + SELECT + id, + owner_type, + owner_id AS company_id, + document_type AS identity_document_type_name, + reference AS identity_number, + status AS identity_status, + status_name AS identity_status_name, + approved_by AS identity_approved_by, + deleted_at AS identity_deleted_datetine, + approved_at AS identity_approved_datetine, + created_at AS identity_created_datetime + + FROM + company_identity_documents +) + +SELECT * FROM final__fct_exchange__company_identity_documents \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__email_verifications.sql b/models/exchange/marts/warehouse/fct_exchange__email_verifications.sql new file mode 100644 index 0000000..e53d802 --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__email_verifications.sql @@ -0,0 +1,48 @@ +--IMPORT +WITH user_email_verifications AS ( + SELECT * FROM {{ ref('stg_exchange__user_email_verifications') }} +), + +companies_and_users_relations AS ( + SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }} +), + + +--LOGIC +user_email_verifications_get_company_id AS ( + SELECT + user_email_verifications.id, + user_email_verifications.user_id, + companies_and_users_relations.company_id, + user_email_verifications.email, + user_email_verifications.is_sent, + user_email_verifications.is_completed, + user_email_verifications.is_activated, + user_email_verifications.created_at, + user_email_verifications.updated_at + + FROM user_email_verifications + + LEFT JOIN companies_and_users_relations + ON (user_email_verifications.user_id = companies_and_users_relations.user_id) +), + + +--FINAL +final__fct_exchange__company_email_verifications AS ( + SELECT + id, + user_id, + company_id, + email, + is_sent, + is_completed, + is_activated, + created_at AS verification_email_created_datetime, + updated_at AS verification_email_updated_datetime + + FROM user_email_verifications_get_company_id +) + + +SELECT * FROM final__fct_exchange__company_email_verifications \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql new file mode 100644 index 0000000..3a42556 --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql @@ -0,0 +1,60 @@ +-- IMPORTS +WITH transaction_costs AS ( + SELECT * FROM {{ ref('int_exchange__transaction_costs_remove_duplicate_row') }} +), + + +-- LOGICS +transaction_costs_lists AS ( + SELECT + id, + transaction_order_id, + supplier_company_id, + bank_id, + buyer_company_id, + status, + status_name, + transaction_type, + transaction_type_name, + payment_method, + payment_method_name, + base_currency_id, + base_currency_name, + quote_currency_id, + quote_currency_name, + base_to_quote_currency_exchange_rate, + + base_value, + quote_value, + -- use for future if base currency id other from '1' + IFF(base_currency_id = '1', + base_value, + 99999999) AS transaction_value_rm, + + base_tax AS transaction_base_tax, + -- use for future if base currency id other from '1' + IFF(base_currency_id = '1', + base_tax, + 99999999) AS transaction_tax_rm, + + base_service_charge, + -- use for future if base currency id other from '1' + IFF(base_currency_id = '1', + base_service_charge, + 99999999) AS transaction_service_charge_rm, + + + deleted_at, + created_at, + updated_at + + FROM transaction_costs +), + + +-- FINAL +final__fct_exchange__transaction_costs AS ( + SELECT * FROM transaction_costs_lists +) + +SELECT * FROM final__fct_exchange__transaction_costs \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql new file mode 100644 index 0000000..3bd04f1 --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql @@ -0,0 +1,61 @@ +-- IMPORTS +WITH transaction_log_costs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_log_costs') }} +), + + +-- LOGICS +transaction_log_costs_lists AS ( + SELECT + id, + transaction_cost_id, + transaction_order_id, + supplier_company_id, + bank_id, + buyer_company_id, + status, + status_name, + transaction_type, + transaction_type_name, + payment_method, + payment_method_name, + base_currency_id, + base_currency_name, + quote_currency_id, + quote_currency_name, + base_to_quote_currency_exchange_rate, + + base_value, + quote_value, + -- use for future if base currency id other from '1' + IFF(base_currency_id = '1', + base_value, + 99999999) AS transaction_value_rm, + + base_tax AS transaction_base_tax, + -- use for future if base currency id other from '1' + IFF(base_currency_id = '1', + base_tax, + 99999999) AS transaction_tax_rm, + + base_service_charge, + -- use for future if base currency id other from '1' + IFF(base_currency_id = '1', + base_service_charge, + 99999999) AS transaction_service_charge_rm, + + + deleted_at, + created_at, + updated_at + + FROM transaction_log_costs +), + + +-- FINAL +final__fct_exchange__transaction_log_costs AS ( + SELECT * FROM transaction_log_costs_lists +) + +SELECT * FROM final__fct_exchange__transaction_log_costs \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql new file mode 100644 index 0000000..47a0438 --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql @@ -0,0 +1,116 @@ +-- IMPORTS +WITH transaction_log_orders AS ( + SELECT * FROM {{ ref('int_exchange__transaction_log_orders_get_user_ids') }} +), + + +-- LOGICS +first_time_user_transaction_order_id AS ( + SELECT + transaction_order_id, + user_id, + created_at, + row_number() OVER + (PARTITION BY transaction_log_orders.user_id + ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking + + FROM + transaction_log_orders + + GROUP BY + transaction_order_id, + user_id, + created_at + + QUALIFY + cummulative_count_booking = 1 +), + + +first_time_company_transaction_order_id AS ( + SELECT + transaction_order_id, + company_id, + created_at, + row_number() OVER + (PARTITION BY transaction_log_orders.company_id + ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking + + FROM + transaction_log_orders + + GROUP BY + transaction_order_id, + company_id, + created_at + + QUALIFY + cummulative_count_booking = 1 +), + + +transaction_log_orders_lists AS ( + SELECT + transaction_log_orders.id, + transaction_log_orders.transaction_order_id, + transaction_log_orders.booking_id, + transaction_log_orders.seller_company_id, + transaction_log_orders.bank_id, + transaction_log_orders.company_id, + transaction_log_orders.user_id, + transaction_log_orders.status, + transaction_log_orders.status_name, + transaction_log_orders.transaction_type, + transaction_log_orders.transaction_type_name, + transaction_log_orders.payment_method, + transaction_log_orders.payment_method_name, + transaction_log_orders.base_currency_id, + transaction_log_orders.base_currency_name, + transaction_log_orders.quote_currency_id, + transaction_log_orders.quote_currency_name, + transaction_log_orders.base_to_quote_currency_exchange_rate, + + transaction_log_orders.base_value, + transaction_log_orders.quote_value, + -- use for future if base currency id other from '1' + IFF(transaction_log_orders.base_currency_id = '1', + transaction_log_orders.base_value, + 99999999) AS transaction_value_rm, + + transaction_log_orders.base_tax AS transaction_base_tax, + -- use for future if base currency id other from '1' + IFF(transaction_log_orders.base_currency_id = '1', + transaction_log_orders.base_tax, + 99999999) AS transaction_tax_rm, + + base_service_charge, + -- use for future if base currency id other from '1' + IFF(transaction_log_orders.base_currency_id = '1', + transaction_log_orders.base_service_charge, + 99999999) AS transaction_service_charge_rm, + + + transaction_log_orders.deleted_at, + transaction_log_orders.created_at, + transaction_log_orders.updated_at, + + IFF(first_time_user_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_user, + IFF(first_time_company_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_company + + + FROM transaction_log_orders + + LEFT JOIN first_time_user_transaction_order_id + ON (transaction_log_orders.transaction_order_id = first_time_user_transaction_order_id.transaction_order_id) + + LEFT JOIN first_time_company_transaction_order_id + ON (transaction_log_orders.transaction_order_id = first_time_company_transaction_order_id.transaction_order_id) +), + + +-- FINAL +final__fct_exchange__transaction_log_orders AS ( + SELECT * FROM transaction_log_orders_lists +) + +SELECT * FROM final__fct_exchange__transaction_log_orders \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql new file mode 100644 index 0000000..80b8521 --- /dev/null +++ b/models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql @@ -0,0 +1,92 @@ +-- IMPORTS +WITH transaction_orders AS ( + SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} +), + + +-- LOGICS +transaction_orders_lists AS ( + SELECT + transaction_orders.id, + transaction_orders.booking_id, + transaction_orders.seller_company_id, + transaction_orders.bank_id, + transaction_orders.user_id, + transaction_orders.company_id, + transaction_orders.status, + transaction_orders.status_name, + transaction_orders.transaction_type, + transaction_orders.transaction_type_name, + transaction_orders.payment_method, + transaction_orders.payment_method_name, + transaction_orders.base_currency_id, + transaction_orders.base_currency_name, + transaction_orders.quote_currency_id, + transaction_orders.quote_currency_name, + transaction_orders.base_to_quote_currency_exchange_rate, + + transaction_orders.base_value, + transaction_orders.quote_value, + -- use for future if base currency id other from '1' + IFF(transaction_orders.base_currency_id = '1', + transaction_orders.base_value, + 99999999) AS transaction_value_rm, + + transaction_orders.base_tax AS transaction_base_tax, + -- use for future if base currency id other from '1' + IFF(transaction_orders.base_currency_id = '1', + transaction_orders.base_tax, + 99999999) AS transaction_tax_rm, + + transaction_orders.base_service_charge, + -- use for future if base currency id other from '1' + IFF(transaction_orders.base_currency_id = '1', + transaction_orders.base_service_charge, + 99999999) AS transaction_service_charge_rm, + + + transaction_orders.deleted_at, + transaction_orders.created_at, + transaction_orders.updated_at, + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.company_id ORDER BY transaction_orders.created_at) = 1 THEN 1 + ELSE 0 + END AS is_first_time_company, + + CASE + WHEN transaction_orders.status = '3' + AND + MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL)) + OVER (PARTITION BY transaction_orders.company_id) + = transaction_orders.created_at + THEN 1 + ELSE 0 + END AS is_first_time_company_completed, + + + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.user_id ORDER BY transaction_orders.created_at) = 1 THEN 1 + ELSE 0 + END AS is_first_time_user, + + CASE + WHEN transaction_orders.status = '3' + AND + MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL)) + OVER (PARTITION BY transaction_orders.user_id) + = transaction_orders.created_at + THEN 1 + ELSE 0 + END AS is_first_time_user_completed + + FROM transaction_orders +), + + +-- FINAL +final__fct_exchange__transaction_orders AS ( + SELECT * FROM transaction_orders_lists +) + +SELECT * FROM final__fct_exchange__transaction_orders \ No newline at end of file diff --git a/models/exchange/marts/warehouse/test_exchange__bookings.yml b/models/exchange/marts/warehouse/test_exchange__bookings.yml new file mode 100644 index 0000000..e69de29 diff --git a/models/exchange/marts/warehouse/test_exchange__companies.yml b/models/exchange/marts/warehouse/test_exchange__companies.yml new file mode 100644 index 0000000..e69de29 diff --git a/models/exchange/marts/warehouse/test_exchange__transaction_orders.yml b/models/exchange/marts/warehouse/test_exchange__transaction_orders.yml new file mode 100644 index 0000000..e69de29 diff --git a/models/exchange/marts/warehouse/test_exchange__users.yml b/models/exchange/marts/warehouse/test_exchange__users.yml new file mode 100644 index 0000000..e69de29 diff --git a/models/exchange/source/src_activity_log__exchange.yml b/models/exchange/source/src_activity_log__exchange.yml new file mode 100644 index 0000000..2d2e5ca --- /dev/null +++ b/models/exchange/source/src_activity_log__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: activity_log + description: Raw `activity_log` table from Snowflake + columns: + - name: id + description: Primary key for 'activity_log' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_addresses__exchange.yml b/models/exchange/source/src_addresses__exchange.yml new file mode 100644 index 0000000..a92717e --- /dev/null +++ b/models/exchange/source/src_addresses__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: addresses + description: Raw `addresses` table from Snowflake + columns: + - name: id + description: Primary key for 'addresses' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_announcement_segment__exchange.yml b/models/exchange/source/src_announcement_segment__exchange.yml new file mode 100644 index 0000000..82e4e04 --- /dev/null +++ b/models/exchange/source/src_announcement_segment__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: announcement_segment + description: Raw `announcement_segment` table from Snowflake + columns: + - name: announcement_id + description: Primary key for 'announcement_segment' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_announcements__exchange.yml b/models/exchange/source/src_announcements__exchange.yml new file mode 100644 index 0000000..af6824a --- /dev/null +++ b/models/exchange/source/src_announcements__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: announcements + description: Raw `announcements` table from Snowflake + columns: + - name: id + description: Primary key for 'announcements' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_bank_logs__exchange.yml b/models/exchange/source/src_bank_logs__exchange.yml new file mode 100644 index 0000000..d0d9f03 --- /dev/null +++ b/models/exchange/source/src_bank_logs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: bank_logs + description: Raw `bank_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'bank_logs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_banks__exchange.yml b/models/exchange/source/src_banks__exchange.yml new file mode 100644 index 0000000..94e3888 --- /dev/null +++ b/models/exchange/source/src_banks__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: banks + description: Raw `banks` table from Snowflake + columns: + - name: id + description: Primary key for 'banks' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_booking_logs__exchange.yml b/models/exchange/source/src_booking_logs__exchange.yml new file mode 100644 index 0000000..09c48ad --- /dev/null +++ b/models/exchange/source/src_booking_logs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: booking_logs + description: Raw `bookings` table from Snowflake + columns: + - name: id + description: Primary key for 'booking_logs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_bookings__exchange.yml b/models/exchange/source/src_bookings__exchange.yml new file mode 100644 index 0000000..c53015c --- /dev/null +++ b/models/exchange/source/src_bookings__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: bookings + description: Raw `bookings` table from Snowflake + columns: + - name: id + description: Primary key for 'bookings' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_companies__exchange.yml b/models/exchange/source/src_companies__exchange.yml new file mode 100644 index 0000000..e269b49 --- /dev/null +++ b/models/exchange/source/src_companies__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: companies + description: Raw `companies` table from Snowflake + columns: + - name: id + description: Primary key for 'companies' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_contacts__exchange.yml b/models/exchange/source/src_contacts__exchange.yml new file mode 100644 index 0000000..1042e39 --- /dev/null +++ b/models/exchange/source/src_contacts__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: contacts + description: Raw `contacts` table from Snowflake + columns: + - name: id + description: Primary key for 'contacts' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_countries__exchange.yml b/models/exchange/source/src_countries__exchange.yml new file mode 100644 index 0000000..12131cf --- /dev/null +++ b/models/exchange/source/src_countries__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: countries + description: Raw `countries` table from Snowflake + columns: + - name: id + description: Primary key for 'countries' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currencies__exchange.yml b/models/exchange/source/src_currencies__exchange.yml new file mode 100644 index 0000000..7fe1842 --- /dev/null +++ b/models/exchange/source/src_currencies__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: currencies + description: Raw `currencies` table from Snowflake + columns: + - name: id + description: Primary key for 'currencies' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currency_logs__exchange.yml b/models/exchange/source/src_currency_logs__exchange.yml new file mode 100644 index 0000000..21ecaad --- /dev/null +++ b/models/exchange/source/src_currency_logs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: currency_logs + description: Raw `currency_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'currency_logs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currency_rate_logs.yml b/models/exchange/source/src_currency_rate_logs.yml new file mode 100644 index 0000000..3a25613 --- /dev/null +++ b/models/exchange/source/src_currency_rate_logs.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: currency_rate_logs + description: Raw `currency_rate_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'currency_rate_logs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currency_rates__exchange.yml b/models/exchange/source/src_currency_rates__exchange.yml new file mode 100644 index 0000000..82ce730 --- /dev/null +++ b/models/exchange/source/src_currency_rates__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: currency_rates + description: Raw `currency_rates` table from Snowflake + columns: + - name: id + description: Primary key for 'currency_rates' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_districts__exchange.yml b/models/exchange/source/src_districts__exchange.yml new file mode 100644 index 0000000..7b7754f --- /dev/null +++ b/models/exchange/source/src_districts__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: districts + description: Raw `districts` table from Snowflake + columns: + - name: id + description: Primary key for 'districts' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_documents__exchange.yml b/models/exchange/source/src_documents__exchange.yml new file mode 100644 index 0000000..b8fbfc3 --- /dev/null +++ b/models/exchange/source/src_documents__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: documents + description: Raw `documents` table from Snowflake + columns: + - name: id + description: Primary key for 'documents' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_employees__exchange.yml b/models/exchange/source/src_employees__exchange.yml new file mode 100644 index 0000000..05355a8 --- /dev/null +++ b/models/exchange/source/src_employees__exchange.yml @@ -0,0 +1,22 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: employees + description: Raw `employees` table from Snowflake + columns: + - name: company_id + description: Primary key for 'employees' + + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_failed_jobs__exchange.yml b/models/exchange/source/src_failed_jobs__exchange.yml new file mode 100644 index 0000000..0552305 --- /dev/null +++ b/models/exchange/source/src_failed_jobs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: failed_jobs + description: Raw `failed_jobs` table from Snowflake + columns: + - name: id + description: Primary key for 'failed_jobs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_files__exchange.yml b/models/exchange/source/src_files__exchange.yml new file mode 100644 index 0000000..20588e9 --- /dev/null +++ b/models/exchange/source/src_files__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: files + description: Raw `files` table from Snowflake + columns: + - name: id + description: Primary key for 'files' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_group_transactions__exchange.yml b/models/exchange/source/src_group_transactions__exchange.yml new file mode 100644 index 0000000..8585e51 --- /dev/null +++ b/models/exchange/source/src_group_transactions__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: group_transactions + description: Raw `group_transactions` table from Snowflake + columns: + - name: id + description: Primary key for 'group_transactions' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_groups__exchange.yml b/models/exchange/source/src_groups__exchange.yml new file mode 100644 index 0000000..9e398b2 --- /dev/null +++ b/models/exchange/source/src_groups__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: groups + description: Raw `groups` table from Snowflake + columns: + - name: id + description: Primary key for 'groups' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_jobs__exchange.yml b/models/exchange/source/src_jobs__exchange.yml new file mode 100644 index 0000000..6dc16f9 --- /dev/null +++ b/models/exchange/source/src_jobs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: jobs + description: Raw `jobs` table from Snowflake + columns: + - name: id + description: Primary key for 'jobs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_migrations__exchange.yml b/models/exchange/source/src_migrations__exchange.yml new file mode 100644 index 0000000..c9bc6c1 --- /dev/null +++ b/models/exchange/source/src_migrations__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: migrations + description: Raw `migrations` table from Snowflake + columns: + - name: id + description: Primary key for 'migrations' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_model_has_roles__exchange.yml b/models/exchange/source/src_model_has_roles__exchange.yml new file mode 100644 index 0000000..86b3736 --- /dev/null +++ b/models/exchange/source/src_model_has_roles__exchange.yml @@ -0,0 +1,22 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: model_has_roles + description: Raw `model_has_roles` table from Snowflake + columns: + - name: role_id + description: Primary key for 'model_has_roles' + + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_notifications__exchange.yml b/models/exchange/source/src_notifications__exchange.yml new file mode 100644 index 0000000..856a463 --- /dev/null +++ b/models/exchange/source/src_notifications__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: notifications + description: Raw `notifications` table from Snowflake + columns: + - name: id + description: Primary key for 'notifications' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_password_resets__exchange.yml b/models/exchange/source/src_password_resets__exchange.yml new file mode 100644 index 0000000..c1c3a4c --- /dev/null +++ b/models/exchange/source/src_password_resets__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: password_resets + description: Raw `password_resets` table from Snowflake + columns: + - name: id + description: Primary key for 'password_resets' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_permissions__exchange.yml b/models/exchange/source/src_permissions__exchange.yml new file mode 100644 index 0000000..201fa3d --- /dev/null +++ b/models/exchange/source/src_permissions__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: permissions + description: Raw `permissions` table from Snowflake + columns: + - name: id + description: Primary key for 'permissions' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_role_has_permissions__exchange.yml b/models/exchange/source/src_role_has_permissions__exchange.yml new file mode 100644 index 0000000..a0b1827 --- /dev/null +++ b/models/exchange/source/src_role_has_permissions__exchange.yml @@ -0,0 +1,22 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: role_has_permissions + description: Raw `role_has_permissions` table from Snowflake + columns: + - name: permission_id + description: Primary key for 'role_has_permissions' + + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_roles__exchange.yml b/models/exchange/source/src_roles__exchange.yml new file mode 100644 index 0000000..b4f4b0c --- /dev/null +++ b/models/exchange/source/src_roles__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: roles + description: Raw `roles` table from Snowflake + columns: + - name: id + description: Primary key for 'roles' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_segment_companies__exchange.yml b/models/exchange/source/src_segment_companies__exchange.yml new file mode 100644 index 0000000..accbcf0 --- /dev/null +++ b/models/exchange/source/src_segment_companies__exchange.yml @@ -0,0 +1,22 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: segment_companies + description: Raw `segment_companies` table from Snowflake + columns: + - name: segment_id + description: Primary key for 'segment_companies' + + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_segment_constants__exchange.yml b/models/exchange/source/src_segment_constants__exchange.yml new file mode 100644 index 0000000..9c2475e --- /dev/null +++ b/models/exchange/source/src_segment_constants__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: segment_constants + description: Raw `segment_constants` table from Snowflake + columns: + - name: id + description: Primary key for 'segment_constants' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_segments__exchange.yml b/models/exchange/source/src_segments__exchange.yml new file mode 100644 index 0000000..7323579 --- /dev/null +++ b/models/exchange/source/src_segments__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: segments + description: Raw `segments` table from Snowflake + columns: + - name: id + description: Primary key for 'segments' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_service_types__exchange.yml b/models/exchange/source/src_service_types__exchange.yml new file mode 100644 index 0000000..bbac54c --- /dev/null +++ b/models/exchange/source/src_service_types__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: service_types + description: Raw `service_types` table from Snowflake + columns: + - name: id + description: Primary key for 'service_types' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_states__exchange.yml b/models/exchange/source/src_states__exchange.yml new file mode 100644 index 0000000..881b421 --- /dev/null +++ b/models/exchange/source/src_states__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: states + description: Raw `states` table from Snowflake + columns: + - name: id + description: Primary key for 'states' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_transaction_detail__exchange.yml b/models/exchange/source/src_transaction_detail__exchange.yml new file mode 100644 index 0000000..8bdfe05 --- /dev/null +++ b/models/exchange/source/src_transaction_detail__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: transaction_detail + description: Raw `transaction_detail` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_detail' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_transaction_logs__exchange.yml b/models/exchange/source/src_transaction_logs__exchange.yml new file mode 100644 index 0000000..1802834 --- /dev/null +++ b/models/exchange/source/src_transaction_logs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: transaction_logs + description: Raw `transaction_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_logs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_transactions__exchange.yml b/models/exchange/source/src_transactions__exchange.yml new file mode 100644 index 0000000..4ae35ef --- /dev/null +++ b/models/exchange/source/src_transactions__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: transactions + description: Raw `transactions` table from Snowflake + columns: + - name: id + description: Primary key for 'transactions' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_user_email_verifications__exchange.yml b/models/exchange/source/src_user_email_verifications__exchange.yml new file mode 100644 index 0000000..7b6ed93 --- /dev/null +++ b/models/exchange/source/src_user_email_verifications__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: user_email_verifications + description: Raw `user_email_verifications` table from Snowflake + columns: + - name: id + description: Primary key for 'user_email_verifications' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_users__exchange.yml b/models/exchange/source/src_users__exchange.yml new file mode 100644 index 0000000..7b31631 --- /dev/null +++ b/models/exchange/source/src_users__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: users + description: Raw `users` table from Snowflake + columns: + - name: id + description: Primary key for 'users' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_wallet_logs__exchange.yml b/models/exchange/source/src_wallet_logs__exchange.yml new file mode 100644 index 0000000..c4da2bf --- /dev/null +++ b/models/exchange/source/src_wallet_logs__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: wallet_logs + description: Raw `wallet_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'wallet_logs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_wallets__exchange.yml b/models/exchange/source/src_wallets__exchange.yml new file mode 100644 index 0000000..62094bc --- /dev/null +++ b/models/exchange/source/src_wallets__exchange.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + + tables: + - name: wallets + description: Raw `wallets` table from Snowflake + columns: + - name: id + description: Primary key for 'wallets' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql b/models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql new file mode 100644 index 0000000..3e83c4a --- /dev/null +++ b/models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH activity_logs AS ( + SELECT * FROM {{ ref('base_exchange__activity_logs') }} +), + + +-- LOGIC +activity_logs_listout AS ( + + SELECT + activity_logs.id, + activity_logs.log_name, + activity_logs.description, + activity_logs.subject_id, + activity_logs.subject_type, + activity_logs.causer_id, + activity_logs.causer_type, + activity_logs.properties, + activity_logs.created_at, + activity_logs.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM activity_logs +), + + +-- FINAL +final__stg_exchange__activity_logs AS ( + + SELECT + id, + log_name, + description, + subject_id, + subject_type, + causer_id, + causer_type, + properties, + created_at, + updated_at, + _dbt_ran_at + + FROM activity_logs_listout +) + +SELECT * FROM final__stg_exchange__activity_logs \ No newline at end of file diff --git a/models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql b/models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql new file mode 100644 index 0000000..ca8ceea --- /dev/null +++ b/models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql @@ -0,0 +1,44 @@ +-- IMPORTS +WITH announcement_segments AS ( + SELECT * FROM {{ ref('base_exchange__announcement_segments') }} +), + +segments AS ( + SELECT * FROM {{ ref('base_exchange__segments') }} +), + +-- LOGIC +announcement_segments_join_segments AS ( + + SELECT + announcement_segments.announcement_id, + + announcement_segments.segment_id, + segments.name AS segment_name, + + announcement_segments.created_at, + announcement_segments.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM announcement_segments + + LEFT JOIN segments + ON (announcement_segments.segment_id = segments.id) +), + + +-- FINAL +final__stg_exchange__announcement_segments AS ( + + SELECT + announcement_id, + segment_id, + segment_name, + created_at, + updated_at, + _dbt_ran_at + + FROM announcement_segments_join_segments +) + +SELECT * FROM final__stg_exchange__announcement_segments \ No newline at end of file diff --git a/models/exchange/staging/announcements/stg_exchange__announcements.sql b/models/exchange/staging/announcements/stg_exchange__announcements.sql new file mode 100644 index 0000000..b2a3846 --- /dev/null +++ b/models/exchange/staging/announcements/stg_exchange__announcements.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH announcements AS ( + SELECT * FROM {{ ref('base_exchange__announcements') }} +), + + +-- LOGIC +announcements_listout AS ( + + SELECT + id, + title, + description, + started_at, + ended_at, + is_all_day, + duration, + is_recurring, + recurrence_pattern, + deleted_at, + created_at, + updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM announcements +), + + +-- FINAL +final__stg_exchange__announcements AS ( + + SELECT + id, + title, + description, + started_at, + ended_at, + is_all_day, + duration, + is_recurring, + recurrence_pattern, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM announcements_listout +) + +SELECT * FROM final__stg_exchange__announcements \ No newline at end of file diff --git a/models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql b/models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql new file mode 100644 index 0000000..0c50631 --- /dev/null +++ b/models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql @@ -0,0 +1,93 @@ +-- IMPORTS +WITH bank_logs AS ( + SELECT * FROM {{ ref('base_exchange__bank_logs') }} +), + +bank_types AS ( + SELECT * FROM {{ ref('seed_exchange__bank_types') }} + WHERE category = 'REPHRASE' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + +-- LOGIC +bank_logs_join_status_contries AS ( + + SELECT + bank_logs.id, + bank_logs.bank_id, + bank_logs.company_id, + bank_logs.reference, + bank_logs.bank_name, + bank_logs.holder_name, + bank_logs.account_no, + bank_logs.bank_branch, + bank_logs.swift_code, + bank_logs.cnaps_code, + + bank_logs.bank_type, + bank_types.name AS bank_type_name, + + bank_logs.is_default, + + bank_logs.status, + status.name AS status_name, + + bank_logs.country_id, + countries.name AS country_name, + + bank_logs.deleted_at, + bank_logs.created_at, + bank_logs.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM bank_logs + + LEFT JOIN bank_types + ON (bank_logs.bank_type = bank_types.id) + + LEFT JOIN status + ON (bank_logs.status = status.id) + + LEFT JOIN countries + ON (bank_logs.country_id = countries.id) +), + + +-- FINAL +final__stg_bank_logs__exchange AS ( + + SELECT + id, + bank_id, + company_id, + reference, + bank_name, + holder_name, + account_no, + bank_branch, + swift_code, + cnaps_code, + bank_type, + bank_type_name, + is_default, + status, + status_name, + country_id, + country_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM bank_logs_join_status_contries +) + +SELECT * FROM final__stg_bank_logs__exchange \ No newline at end of file diff --git a/models/exchange/staging/banks/stg_exchange__banks.sql b/models/exchange/staging/banks/stg_exchange__banks.sql new file mode 100644 index 0000000..243ae3c --- /dev/null +++ b/models/exchange/staging/banks/stg_exchange__banks.sql @@ -0,0 +1,91 @@ +-- IMPORTS +WITH banks AS ( + SELECT * FROM {{ ref('base_exchange__banks') }} +), + +bank_types AS ( + SELECT * FROM {{ ref('seed_exchange__bank_types') }} + WHERE category = 'REPHRASE' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + +-- LOGIC +banks_join_status_contries AS ( + + SELECT + banks.id, + banks.company_id, + banks.reference, + banks.bank_name, + banks.holder_name, + banks.account_no, + banks.bank_branch, + banks.swift_code, + banks.cnaps_code, + + banks.bank_type, + bank_types.name AS bank_type_name, + + banks.is_default, + + banks.status, + status.name AS status_name, + + banks.country_id, + countries.name AS country_name, + + banks.deleted_at, + banks.created_at, + banks.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM banks + + LEFT JOIN bank_types + ON (banks.bank_type = bank_types.id) + + LEFT JOIN status + ON (banks.status = status.id) + + LEFT JOIN countries + ON (banks.country_id = countries.id) +), + + +-- FINAL +final__stg_exchange__banks AS ( + + SELECT + id, + company_id, + reference, + bank_name, + holder_name, + account_no, + bank_branch, + swift_code, + cnaps_code, + bank_type, + bank_type_name, + is_default, + status, + status_name, + country_id, + country_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM banks_join_status_contries +) + +SELECT * FROM final__stg_exchange__banks \ No newline at end of file diff --git a/models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql b/models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql new file mode 100644 index 0000000..a65d7ec --- /dev/null +++ b/models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql @@ -0,0 +1,165 @@ +-- IMPORTS +WITH booking_logs AS ( + SELECT * FROM {{ ref('base_exchange__booking_logs') }} +), + +bookings AS ( + SELECT * FROM {{ ref('base_exchange__bookings') }} +), + +activity_logs AS ( + SELECT * FROM {{ ref('base_exchange__activity_logs') }} +), + +service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +bookings_get_hash_ids AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(id, created_at)) AS id, + id AS booking_id, + company_id, + marking_id, + service_type, + bank_id, + fix_value, + fix_currency_id, + quote_currency_id, + base_currency_id, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM bookings +), + + +get_booking_user_ids AS ( + + SELECT + subject_id AS booking_id, + causer_id AS user_id + FROM + activity_logs + WHERE + subject_type='App\\Models\\Booking' + AND + description='created' + +), + + +booking_logs_unions AS ( + SELECT * FROM bookings_get_hash_ids + UNION + SELECT * FROM booking_logs +), + + +booking_logs_unions_join_service_types_currencies_status_get_booking_user_ids AS ( + + SELECT + booking_logs_unions.id, + booking_logs_unions.booking_id, + booking_logs_unions.company_id, + + get_booking_user_ids.user_id, + + booking_logs_unions.marking_id, + + booking_logs_unions.service_type, + service_types.name AS service_type_name, + + booking_logs_unions.bank_id, + booking_logs_unions.fix_value, + + booking_logs_unions.fix_currency_id, + fix_currencies.name AS fix_currency_name, + fix_currencies.short_code AS fix_currency_short_code, + + booking_logs_unions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + + booking_logs_unions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + + booking_logs_unions.status, + status.name AS status_name, + + booking_logs_unions.deleted_at, + booking_logs_unions.created_at, + booking_logs_unions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM booking_logs_unions + + LEFT JOIN get_booking_user_ids + ON (booking_logs_unions.booking_id = get_booking_user_ids.booking_id) + + LEFT JOIN service_types + ON (booking_logs_unions.service_type = service_types.id) + + LEFT JOIN currencies AS fix_currencies + ON (booking_logs_unions.fix_currency_id = fix_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (booking_logs_unions.quote_currency_id = quote_currencies.id) + + LEFT JOIN currencies AS base_currencies + ON (booking_logs_unions.base_currency_id = base_currencies.id) + + LEFT JOIN status + ON (booking_logs_unions.status = status.id) +), + + +-- FINAL +final__stg_exchange__booking_logs AS ( + + SELECT + id, + booking_id, + company_id, + user_id, + marking_id, + service_type, + service_type_name, + bank_id, + fix_value, + fix_currency_id, + fix_currency_name, + fix_currency_short_code, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + base_currency_id, + base_currency_name, + base_currency_short_code, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM booking_logs_unions_join_service_types_currencies_status_get_booking_user_ids + +) + +SELECT * FROM final__stg_exchange__booking_logs \ No newline at end of file diff --git a/models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md b/models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md new file mode 100644 index 0000000..9ccfc2f --- /dev/null +++ b/models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md @@ -0,0 +1,15 @@ +{% docs generic_test_stg_bookings_status__exchange %} +Using seeds csv to transform + +| STATUS | STATUS_DISPLAYNAME | definition | +|----------------|----------------------------|--------------------------------------------------| +| 0 | PENDING_SUBMISSION | Order placed, not yet shipped | +| 1 | PENDING_VERIFICATION | Order has been shipped, not yet been delivered | +| 2 | APPROVED | Order has been received by customers | +| 3 | COMPLETED | Order has been received by customers | +| 4 | REJECTED | Order has been received by customers | +| 5 | SUSPENDED | Order has been received by customers | +| 6 | EXPIRED | Order has been received by customers | +| 7 | REFUNDED | Order has been received by customers | +| ELSE | | Refer to seeds for define the status | +{% enddocs %} \ No newline at end of file diff --git a/models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml b/models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml new file mode 100644 index 0000000..347d3fa --- /dev/null +++ b/models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml @@ -0,0 +1,100 @@ +# version: 2 + +# models: +# - name: stg_exchange__bookings +# columns: +# - name: ID +# description: PK ID +# tests: +# - unique +# - not_null + +# - name: COMPANY_ID +# description: FK, for source `src_companies` +# tests: +# - not_null +# - relationships: +# to: ref('stg_companies') +# field: COMPANY_ID + +# - name: HUMAN_READABLE_MARKING_ID +# description: Auto generate unique key +# tests: +# - unique +# - not_null + +# - name: SERVICE_ID +# description: FK, for source `src_service_types` +# tests: +# - not_null +# - relationships: +# to: ref('stg_exchange__service_types') +# field: SERVICE_TYPE_ID + +# - name: BANK_ID +# description: FK, for source `src_banks` +# tests: +# - not_null +# - relationships: +# to: ref('stg_banks') +# field: BANK_ID + +# - name: FIX_AMOUNT +# description: |- +# Amount base on 'FIX_CURRENCY_ID' +# Contain singular test +# Purpose of singular test:- It must only contain positive number & greater than 0 +# Location singular test: tests/exchange/staging/test_stg_bookings_fixAmount_positive.sql +# tests: +# - not_null + +# - name: FIX_CURRENCY_ID +# description: FK, for source `src_currencies` +# tests: +# - not_null +# - relationships: +# to: ref('stg_currencies') +# field: CURRENCY_ID + +# - name: QUOTE_CURRENCY_ID +# description: FK, for source `src_currencies`, currency that user want change into +# tests: +# - not_null +# - relationships: +# to: ref('stg_currencies') +# field: CURRENCY_ID + +# - name: BASE_CURRENCY_ID +# description: FK, for source `src_currencies`, currency that user use to change to other currency +# tests: +# - not_null +# - relationships: +# to: ref('stg_currencies') +# field: CURRENCY_ID + +# - name: STATUS +# description: "{{ doc('generic_test_stg_bookings_status__exchange') }}" +# tests: +# - not_null +# - accepted_values: +# values: [0, 1, 2, 3, 4, 5, 6, 7] + +# - name: DELETED_AT +# description: Database row deleted datetime, if it occur, means row has been deleted, can be NULL +# tests: + +# - name: CREATED_AT +# description: Database row created datetime +# tests: +# - not_null + +# - name: UPDATED_AT +# description: Database row updated datetime, can be same as 'CREATED_AT' +# tests: +# - not_null + +# - name: _SOURCE_LOADED_AT +# description: Python extract datetime + +# - name: _DBT_RUN_AT +# description: Dbt run query datetime \ No newline at end of file diff --git a/models/exchange/staging/bookings/stg_exchange__bookings.sql b/models/exchange/staging/bookings/stg_exchange__bookings.sql new file mode 100644 index 0000000..f4decd9 --- /dev/null +++ b/models/exchange/staging/bookings/stg_exchange__bookings.sql @@ -0,0 +1,128 @@ +-- IMPORTS +WITH bookings AS ( + SELECT * FROM {{ ref('base_exchange__bookings') }} +), + +activity_logs AS ( + SELECT * FROM {{ ref('base_exchange__activity_logs') }} +), + +service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +booking_get_user_ids AS ( + + SELECT + subject_id AS booking_id, + causer_id AS user_id + FROM + activity_logs + WHERE + subject_type='App\\Models\\Booking' + AND + description='created' + +), + +bookings_join_service_types_currencies_status_booking_get_user_ids AS ( + + SELECT + bookings.id, + bookings.company_id, + + booking_get_user_ids.user_id, + + bookings.marking_id, + + bookings.service_type, + service_types.name AS service_type_name, + + bookings.bank_id, + bookings.fix_value, + + bookings.fix_currency_id, + fix_currencies.name AS fix_currency_name, + fix_currencies.short_code AS fix_currency_short_code, + + bookings.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + + bookings.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + + bookings.status, + status.name AS status_name, + + bookings.deleted_at AS booking_deleted_datetime, + bookings.created_at AS booking_created_datetime, + bookings.updated_at AS booking_updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM bookings + + LEFT JOIN booking_get_user_ids + ON (bookings.id = booking_get_user_ids.booking_id) + + LEFT JOIN service_types + ON (bookings.service_type = service_types.id) + + LEFT JOIN currencies AS fix_currencies + ON (bookings.fix_currency_id = fix_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (bookings.quote_currency_id = quote_currencies.id) + + LEFT JOIN currencies AS base_currencies + ON (bookings.base_currency_id = base_currencies.id) + + LEFT JOIN status + ON (bookings.status = status.id) +), + +-- FINAL +final__stg_exchange__bookings AS ( + + SELECT + id, + company_id, + user_id, + marking_id, + service_type, + service_type_name, + bank_id, + fix_value, + fix_currency_id, + fix_currency_name, + fix_currency_short_code, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + base_currency_id, + base_currency_name, + base_currency_short_code, + status, + status_name, + booking_deleted_datetime, + booking_created_datetime, + booking_updated_datetime, + _dbt_ran_at + + FROM bookings_join_service_types_currencies_status_booking_get_user_ids + +) + +SELECT * FROM final__stg_exchange__bookings \ No newline at end of file diff --git a/models/exchange/staging/companies/stg_exchange__companies.sql b/models/exchange/staging/companies/stg_exchange__companies.sql new file mode 100644 index 0000000..4eaaafe --- /dev/null +++ b/models/exchange/staging/companies/stg_exchange__companies.sql @@ -0,0 +1,80 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ ref('base_exchange__companies') }} +), + +company_types AS ( + SELECT * FROM {{ ref('seed_exchange__company_types') }} + WHERE category = '2_AS_CORPORATE' +), + +business_types AS ( + SELECT * FROM {{ ref('seed_exchange__business_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +companies_rename_join_company_types_business_types_status AS ( + + SELECT + companies.id, + companies.name, + companies.marking_id, + companies.account_software_id, + + companies.company_type, + company_types.name AS company_type_name, + + companies.business_type, + business_types.name AS business_type_name, + + companies.status, + status.name AS status_name, + + companies.deleted_at, + companies.created_at, + companies.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM companies + + LEFT JOIN company_types + ON (companies.company_type = company_types.id) + + LEFT JOIN business_types + ON (companies.business_type = business_types.id) + + LEFT JOIN status + ON (companies.status = status.id) +), + +-- FINAL +final__stg_companies__exchange AS ( + + SELECT + id, + name, + marking_id, + account_software_id, + company_type, + company_type_name, + business_type, + business_type_name, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM companies_rename_join_company_types_business_types_status + +) + +SELECT * FROM final__stg_companies__exchange \ No newline at end of file diff --git a/models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql b/models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql new file mode 100644 index 0000000..96fd03f --- /dev/null +++ b/models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH segment_companies AS ( + SELECT * FROM {{ ref('base_exchange__segment_companies') }} +), + + +-- LOGIC +segment_companies_list AS ( + + SELECT + segment_companies.concat_id, + segment_companies.segment_id, + segment_companies.company_id, + segment_companies.deleted_at, + segment_companies.created_at, + segment_companies.updated_at, + segment_companies.dbt_valid_from, + segment_companies.dbt_valid_to, + current_timestamp() AS _dbt_ran_at + + FROM segment_companies +), + + +-- FINAL +final__stg_exchange__segment_companies AS ( + + SELECT + concat_id, + segment_id, + company_id, + deleted_at, + created_at, + updated_at, + dbt_valid_from, + dbt_valid_to, + _dbt_ran_at + + FROM segment_companies_list + +) + +SELECT * FROM final__stg_exchange__segment_companies \ No newline at end of file diff --git a/models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql b/models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql new file mode 100644 index 0000000..857cece --- /dev/null +++ b/models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH employees AS ( + SELECT * FROM {{ ref('base_exchange__employees') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +employees_join_status AS ( + + SELECT + employees.company_id, + employees.user_id, + + employees.status, + status.name AS status_name, + + current_timestamp() AS _dbt_ran_at + + FROM employees + + LEFT JOIN status + ON (employees.status = status.id) + +), + + +-- FINAL +final__stg_exchange__companies_and_users_relations AS ( + + SELECT + company_id, + user_id, + status, + status_name, + _dbt_ran_at + + FROM employees_join_status + +) + +SELECT * FROM final__stg_exchange__companies_and_users_relations \ No newline at end of file diff --git a/models/exchange/staging/company_addresses/stg_exchange__company_addresses.sql b/models/exchange/staging/company_addresses/stg_exchange__company_addresses.sql new file mode 100644 index 0000000..9fb353b --- /dev/null +++ b/models/exchange/staging/company_addresses/stg_exchange__company_addresses.sql @@ -0,0 +1,83 @@ +-- IMPORTS +WITH company_addresses AS ( + SELECT * FROM {{ ref('base_exchange__addresses') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + +states AS ( + SELECT * FROM {{ ref('base_exchange__states') }} +), + +districts AS ( + SELECT * FROM {{ ref('base_exchange__districts') }} +), + +-- LOGIC +company_addressess_join_companies_countries_states_districts AS ( + + SELECT + company_addresses.id, + company_addresses.company_id, + + company_addresses.country_id, + countries.name AS country_name, + countries.short_code AS country_short_code, + countries.phone_code AS country_phone_code, + + company_addresses.state_id, + states.name AS state_name, + + company_addresses.district_id, + districts.name AS district_name, + + company_addresses.postcode, + company_addresses.address_line_one, + company_addresses.address_line_two, + company_addresses.is_billing_address, + company_addresses.deleted_at, + company_addresses.created_at, + company_addresses.updated_At, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM company_addresses + + LEFT JOIN countries + ON (company_addresses.country_id = countries.id) + + LEFT JOIN states + ON (company_addresses.state_id = states.id) + + LEFT JOIN districts + ON (company_addresses.district_id = districts.id) +), + +-- FINAL +final__stg_exchange__company_addresses AS ( + + SELECT + id, + company_id, + country_id, + country_name, + country_short_code, + country_phone_code + state_id, + state_name, + district_id, + district_name, + postcode, + address_line_one, + address_line_two, + is_billing_address, + deleted_at, + created_at, + updated_At, + _dbt_ran_at + + FROM company_addressess_join_companies_countries_states_districts +) + +SELECT * FROM final__stg_exchange__company_addresses \ No newline at end of file diff --git a/models/exchange/staging/contacts/stg_exchange__contacts.sql b/models/exchange/staging/contacts/stg_exchange__contacts.sql new file mode 100644 index 0000000..5a679c6 --- /dev/null +++ b/models/exchange/staging/contacts/stg_exchange__contacts.sql @@ -0,0 +1,59 @@ +-- IMPORTS +WITH contacts AS ( + SELECT * FROM {{ ref('base_exchange__contacts') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + + +-- LOGIC +companies_join_countries_status AS ( + + SELECT + contacts.id, + contacts.company_id, + + contacts.country_id, + countries.name AS country_name, + + contacts.reference, + contacts.phone, + contacts.email, + contacts.wechat_id, + contacts.deleted_at, + contacts.created_at, + contacts.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM contacts + + LEFT JOIN countries + ON (contacts.country_id = countries.id) + +), + + +-- FINAL +final__stg_exchange__contacts AS ( + + SELECT + id, + company_id, + country_id, + country_name, + reference, + phone, + email, + wechat_id, + deleted_at AS contact_deleted_datetime, + created_at AS contact_created_datetime, + updated_at AS contact_updated_datetime, + _dbt_ran_at + + FROM companies_join_countries_status + +) + +SELECT * FROM final__stg_exchange__contacts \ No newline at end of file diff --git a/models/exchange/staging/countries/stg_exchange__countries.sql b/models/exchange/staging/countries/stg_exchange__countries.sql new file mode 100644 index 0000000..72318a5 --- /dev/null +++ b/models/exchange/staging/countries/stg_exchange__countries.sql @@ -0,0 +1,41 @@ +-- IMPORTS +WITH countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + +-- LOGIC +countries_rename AS ( + + SELECT + countries.id, + countries.name, + countries.short_code, + countries.phone_code, + countries.deleted_at, + countries.created_at, + countries.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM countries + +), + + +-- FINAL +final__stg_countries__exchange AS ( + + SELECT + id, + name, + short_code, + phone_code, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM countries_rename + +) + +SELECT * FROM final__stg_countries__exchange \ No newline at end of file diff --git a/models/exchange/staging/currencies/stg_exchange__currencies.sql b/models/exchange/staging/currencies/stg_exchange__currencies.sql new file mode 100644 index 0000000..8827d2e --- /dev/null +++ b/models/exchange/staging/currencies/stg_exchange__currencies.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + + + +-- LOGIC +currencies_join_countries AS ( + + SELECT + currencies.id, + + currencies.country_id, + countries.name AS country_name, + + currencies.name, + currencies.short_code, + currencies.symbol, + currencies.deleted_at, + currencies.created_at, + currencies.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM currencies + + LEFT JOIN countries + ON (currencies.country_id = countries.id) +), + + +-- FINAL +final__stg_exchange__currencies AS ( + + SELECT + id, + country_id, + country_name, + name, + short_code, + symbol, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM currencies_join_countries + +) + +SELECT * FROM final__stg_exchange__currencies \ No newline at end of file diff --git a/models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql b/models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql new file mode 100644 index 0000000..8872783 --- /dev/null +++ b/models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH currency_logs AS ( + SELECT * FROM {{ ref('base_exchange__currency_logs') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + + + +-- LOGIC +currency_logs_join_currencies AS ( + + SELECT + currency_logs.id, + + currency_logs.currency_id, + currencies.name AS currency_name, + currencies.short_code AS currency_short_code, + currencies.symbol AS currency_symbol, + + currency_logs.created_by, + currencies.deleted_at, + currencies.created_at, + currencies.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM currency_logs + + LEFT JOIN currencies + ON (currency_logs.currency_id = currencies.id) +), + + +-- FINAL +final__stg_exchange__currency_logs AS ( + + SELECT + id, + currency_id, + currency_name, + currency_short_code, + currency_symbol, + created_by, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM currency_logs_join_currencies + +) + +SELECT * FROM final__stg_exchange__currency_logs \ No newline at end of file diff --git a/models/exchange/staging/currency_rate_logs/stg_exchange__currency_rate_logs.sql b/models/exchange/staging/currency_rate_logs/stg_exchange__currency_rate_logs.sql new file mode 100644 index 0000000..f9cf5ed --- /dev/null +++ b/models/exchange/staging/currency_rate_logs/stg_exchange__currency_rate_logs.sql @@ -0,0 +1,172 @@ +-- IMPORTS +WITH currency_rate_logs AS ( + SELECT * FROM {{ ref('base_exchange__currency_rate_logs') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +currency_rates AS ( + SELECT * FROM {{ ref('base_exchange__currency_rates') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} + WHERE category = 'DEFAULT' +), + +service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + + +-- LOGIC +currency_rate_logs_join_currency_rates AS ( + + SELECT + currency_rate_logs.id, + currency_rate_logs.currency_rate_id, + + currency_rates.quote_currency_id, + currency_rate_logs.base_to_quote_currency_exchange_rate AS base_to_quote_currency_exchange_rate, + currency_rates.payment_method, + currency_rates.service_type, + + currency_rate_logs.created_by, + currency_rate_logs.created_at, + currency_rate_logs.updated_at + + FROM currency_rate_logs + + LEFT JOIN currency_rates + ON (currency_rate_logs.currency_rate_id = currency_rates.id) +), + + +currency_rate_logs_quote_currency_id_join_currencies AS ( + + SELECT + currency_rate_logs_join_currency_rates.id, + currency_rate_logs_join_currency_rates.currency_rate_id, + + currency_rate_logs_join_currency_rates.quote_currency_id, + currencies.name AS quote_currency_name, + currencies.short_code AS quote_currency_short_code, + currencies.symbol AS quote_currency_symbol, + + currency_rate_logs_join_currency_rates.base_to_quote_currency_exchange_rate, + + currency_rate_logs_join_currency_rates.payment_method, + payment_methods.name AS payment_method_name, + + currency_rate_logs_join_currency_rates.service_type, + service_types.name AS service_type_name, + + currency_rate_logs_join_currency_rates.created_by, + currency_rate_logs_join_currency_rates.created_at, + currency_rate_logs_join_currency_rates.updated_at + + FROM + currency_rate_logs_join_currency_rates + + LEFT JOIN currencies + ON (currency_rate_logs_join_currency_rates.quote_currency_id = currencies.id) + + LEFT JOIN payment_methods + ON (currency_rate_logs_join_currency_rates.payment_method = payment_methods.id) + + LEFT JOIN service_types + ON (currency_rate_logs_join_currency_rates.service_type = service_types.id) +), + +currency_rate_logs_add_dense_rank AS ( + + SELECT + currency_rate_logs_quote_currency_id_join_currencies.id, + + dense_rank() OVER + (PARTITION BY currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id + ORDER BY currency_rate_logs_quote_currency_id_join_currencies.created_at) AS dense_rank_index, + + currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id AS currency_rate_id, + currency_rate_logs_quote_currency_id_join_currencies.quote_currency_id, + currency_rate_logs_quote_currency_id_join_currencies.quote_currency_name, + currency_rate_logs_quote_currency_id_join_currencies.quote_currency_short_code, + currency_rate_logs_quote_currency_id_join_currencies.quote_currency_symbol, + currency_rate_logs_quote_currency_id_join_currencies.payment_method, + currency_rate_logs_quote_currency_id_join_currencies.payment_method_name, + currency_rate_logs_quote_currency_id_join_currencies.service_type, + currency_rate_logs_quote_currency_id_join_currencies.service_type_name, + currency_rate_logs_quote_currency_id_join_currencies.base_to_quote_currency_exchange_rate, + currency_rate_logs_quote_currency_id_join_currencies.created_by AS created_by, + currency_rate_logs_quote_currency_id_join_currencies.created_at AS created_at + + FROM currency_rate_logs_quote_currency_id_join_currencies + + ORDER BY + currency_rate_id, + created_at +), + + +currency_rate_logs_resturucure AS ( + SELECT + currency_rate_logs_add_dense_rank.id, + currency_rate_logs_add_dense_rank.currency_rate_id, + currency_rate_logs_add_dense_rank.quote_currency_id, + currency_rate_logs_add_dense_rank.quote_currency_name, + currency_rate_logs_add_dense_rank.quote_currency_short_code, + currency_rate_logs_add_dense_rank.quote_currency_symbol, + currency_rate_logs_add_dense_rank.payment_method, + currency_rate_logs_add_dense_rank.payment_method_name, + currency_rate_logs_add_dense_rank.service_type, + currency_rate_logs_add_dense_rank.service_type_name, + currency_rate_logs_add_dense_rank.base_to_quote_currency_exchange_rate, + currency_rate_logs_add_dense_rank.created_by, + + CASE + WHEN currency_rate_logs_add_dense_rank.dense_rank_index=1 THEN CAST('1990-01-01 00:00:00 +08:00' AS DATETIME) + ELSE currency_rate_logs_add_dense_rank.created_at + END AS created_date_from, + + COALESCE(currency_rate_logs_add_dense_rank_clone.created_at, + CAST('2999-12-31 23:59:59 +08:00' AS DATETIME)) AS created_date_to, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM + currency_rate_logs_add_dense_rank + + LEFT JOIN currency_rate_logs_add_dense_rank AS currency_rate_logs_add_dense_rank_clone + ON currency_rate_logs_add_dense_rank.dense_rank_index = currency_rate_logs_add_dense_rank_clone.dense_rank_index-1 + AND currency_rate_logs_add_dense_rank.currency_rate_id = currency_rate_logs_add_dense_rank_clone.currency_rate_id + + ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_at +), + + +-- FINAL +final__stg_exchange__currency_rate_logs AS ( + + SELECT + id, + currency_rate_id, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + payment_method, + payment_method_name, + service_type, + service_type_name, + base_to_quote_currency_exchange_rate, + created_by, + created_date_from, + created_date_to, + _dbt_ran_at + + FROM currency_rate_logs_resturucure + +) + +SELECT * FROM final__stg_exchange__currency_rate_logs \ No newline at end of file diff --git a/models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql b/models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql new file mode 100644 index 0000000..e4b0748 --- /dev/null +++ b/models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql @@ -0,0 +1,79 @@ +-- IMPORTS +WITH currency_rates AS ( + SELECT * FROM {{ ref('base_exchange__currency_rates') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} + WHERE category = 'DEFAULT' +), + +service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + +-- LOGIC +currency_rates_join_currencies_payment_methods_service_types AS ( + + SELECT + currency_rates.id, + + currency_rates.quote_currency_id, + currencies.name AS quote_currency_name, + currencies.short_code AS quote_currency_short_code, + currencies.symbol AS quote_currency_symbol, + + currency_rates.base_to_quote_currency_exchange_rate, + + currency_rates.payment_method, + payment_methods.name AS payment_method_name, + + currency_rates.service_type, + service_types.name AS service_type_name, + + currency_rates.deleted_at, + currency_rates.created_at, + currency_rates.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM currency_rates + + LEFT JOIN currencies + ON (currency_rates.quote_currency_id = currencies.id) + + LEFT JOIN payment_methods + ON (currency_rates.payment_method = payment_methods.id) + + LEFT JOIN service_types + ON (currency_rates.service_type = service_types.id) +), + + +-- FINAL +final__stg_exchange__current_currency_rates AS ( + + SELECT + id, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + payment_method, + payment_method_name, + service_type, + service_type_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM currency_rates_join_currencies_payment_methods_service_types + +) + +SELECT * FROM final__stg_exchange__current_currency_rates \ No newline at end of file diff --git a/models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql b/models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql new file mode 100644 index 0000000..11c8c99 --- /dev/null +++ b/models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql @@ -0,0 +1,61 @@ +-- IMPORTS +WITH wallets AS ( + SELECT * FROM {{ ref('base_exchange__wallets') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + + +-- LOGIC +wallets_join_currencies AS ( + + SELECT + wallets.id, + wallets.owner_type, + wallets.owner_id, + wallets.code, + + wallets.currency_id, + currencies.name AS currency_name, + currencies.short_code AS currency_short_code, + currencies.symbol AS currency_symbol, + + wallets.amount, + wallets.deleted_at, + wallets.created_at, + wallets.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM wallets + + LEFT JOIN currencies + ON (wallets.currency_id = currencies.id) + +), + + +-- FINAL +final__stg_exchange__current_wallets AS ( + + SELECT + id, + owner_type, + owner_id, + code, + currency_id, + currency_name, + currency_short_code, + currency_symbol, + amount, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM wallets_join_currencies + +) + +SELECT * FROM final__stg_exchange__current_wallets \ No newline at end of file diff --git a/models/exchange/staging/districts/stg_exchange__districts.sql b/models/exchange/staging/districts/stg_exchange__districts.sql new file mode 100644 index 0000000..bca54b5 --- /dev/null +++ b/models/exchange/staging/districts/stg_exchange__districts.sql @@ -0,0 +1,39 @@ +-- IMPORTS +WITH districts AS ( + SELECT * FROM {{ ref('base_exchange__districts') }} +), + + +-- LOGIC +districts_list AS ( + + SELECT + districts.id, + districts.country_id, + districts.state_id, + districts.name, + districts.postcode, + districts.status, + current_timestamp() AS _dbt_ran_at + + FROM districts +), + + +-- FINAL +final__stg_exchange__districts AS ( + + SELECT + id, + country_id, + state_id, + name, + postcode, + status, + _dbt_ran_at + + FROM districts_list + +) + +SELECT * FROM districts \ No newline at end of file diff --git a/models/exchange/staging/documents/stg_exchange__documents.sql b/models/exchange/staging/documents/stg_exchange__documents.sql new file mode 100644 index 0000000..b209b22 --- /dev/null +++ b/models/exchange/staging/documents/stg_exchange__documents.sql @@ -0,0 +1,66 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.id, + documents.owner_type, + documents.owner_id, + documents.document_type, + documents.reference, + + documents.status, + status.name AS status_name, + + documents.issued_at, + documents.expired_at, + documents.approved_by, + documents.approved_at, + documents.deleted_at, + documents.created_at, + documents.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM documents + + LEFT JOIN status + ON (documents.status = status.id) + +), + + +-- FINAL +final__stg_exchange__documents AS ( + + SELECT + id, + owner_type, + owner_id, + document_type, + reference, + status, + status_name, + issued_at, + expired_at, + approved_by, + approved_at, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM documents_join_status + +) + +SELECT * FROM final__stg_exchange__documents \ No newline at end of file diff --git a/models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql b/models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql new file mode 100644 index 0000000..1b57653 --- /dev/null +++ b/models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH failed_jobs AS ( + SELECT * FROM {{ ref('base_exchange__failed_jobs') }} +), + + +-- LOGIC +failed_jobs_rename AS ( + + SELECT + failed_jobs.id, + failed_jobs.connection, + failed_jobs.queue, + failed_jobs.payload, + failed_jobs.exception, + failed_jobs.failed_at, + current_timestamp() AS _dbt_ran_at + + FROM failed_jobs + +), + + +-- FINAL +final__stg_exchange__failed_jobs AS ( + + SELECT + id, + connection, + queue, + payload, + exception, + failed_at, + _dbt_ran_at + + FROM failed_jobs_rename + +) + +SELECT * FROM final__stg_exchange__failed_jobs \ No newline at end of file diff --git a/models/exchange/staging/files/stg_exchange__files.sql b/models/exchange/staging/files/stg_exchange__files.sql new file mode 100644 index 0000000..3cf5a46 --- /dev/null +++ b/models/exchange/staging/files/stg_exchange__files.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH files AS ( + SELECT * FROM {{ ref('base_exchange__files') }} +), + + +-- LOGIC +files_list AS ( + + SELECT + files.id, + files.document_id, + files.file, + files.file_type, + files.deleted_at, + files.created_at, + files.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM files + +), + + +-- FINAL +final__stg_exchange__files AS ( + + SELECT + id, + document_id, + file, + file_type, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM files_list + +) + +SELECT * FROM final__stg_exchange__files \ No newline at end of file diff --git a/models/exchange/staging/group_costs/stg_exchange__group_costs.sql b/models/exchange/staging/group_costs/stg_exchange__group_costs.sql new file mode 100644 index 0000000..e9dc262 --- /dev/null +++ b/models/exchange/staging/group_costs/stg_exchange__group_costs.sql @@ -0,0 +1,93 @@ +-- IMPORTS +WITH groups AS ( + SELECT * FROM {{ ref('base_exchange__groups') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +groups_join_currencies_status AS ( + + SELECT + groups.id, + + groups.marking_id, + groups.issued_by, + groups.received_by, + groups.base_amount, + groups.quote_amount, + + groups.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + groups.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + groups.base_to_quote_currency_exchange_rate, + groups.base_tax, + groups.base_service_charge, + + groups.status, + status.name AS status_name, + + groups.created_at, + groups.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM groups + + LEFT JOIN currencies AS base_currencies + ON (groups.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (groups.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (groups.status = status.id) +), + + +-- FINAL +final__stg_exchange__groups AS ( + + SELECT + id, + marking_id, + issued_by, + received_by, + base_amount, + quote_amount, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + status, + status_name, + created_at, + updated_at, + _dbt_ran_at + + FROM groups_join_currencies_status + +) + +SELECT * FROM final__stg_exchange__groups \ No newline at end of file diff --git a/models/exchange/staging/jobs/stg_exchange__jobs.sql b/models/exchange/staging/jobs/stg_exchange__jobs.sql new file mode 100644 index 0000000..3ef745d --- /dev/null +++ b/models/exchange/staging/jobs/stg_exchange__jobs.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH jobs AS ( + SELECT * FROM {{ ref('base_exchange__jobs') }} +), + + +-- LOGIC +jobs_list AS ( + + SELECT + jobs.id, + jobs.queue, + jobs.payload, + jobs.attempts, + reserved_at, + available_at, + created_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM jobs + +), + + +-- FINAL +final__stg_exchange__jobs AS ( + + SELECT + id, + queue, + payload, + attempts, + reserved_at, + available_at, + created_at, + _dbt_ran_at + + FROM jobs_list + +) + +SELECT * FROM final__stg_exchange__jobs \ No newline at end of file diff --git a/models/exchange/staging/migrations/stg_exchange__migrations.sql b/models/exchange/staging/migrations/stg_exchange__migrations.sql new file mode 100644 index 0000000..94aaee1 --- /dev/null +++ b/models/exchange/staging/migrations/stg_exchange__migrations.sql @@ -0,0 +1,34 @@ +-- IMPORTS +WITH migrations AS ( + SELECT * FROM {{ ref('base_exchange__migrations') }} +), + + +-- LOGIC +migrations_lsit AS ( + + SELECT + migrations.id, + migrations.migration, + migrations.batch, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM migrations + +), + + +-- FINAL +final__stg_exchange__migrations AS ( + + SELECT + id, + migration, + batch, + _dbt_ran_at + + FROM migrations_lsit + +) + +SELECT * FROM final__stg_exchange__migrations \ No newline at end of file diff --git a/models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql b/models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql new file mode 100644 index 0000000..7f03429 --- /dev/null +++ b/models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH model_has_roles AS ( + SELECT * FROM {{ ref('base_exchange__model_has_roles') }} +), + +roles AS ( + SELECT * FROM {{ ref('base_exchange__roles') }} +), + +-- LOGIC +migrations_join_roles AS ( + + SELECT + model_has_roles.role_id, + roles.name AS role_name, + + model_has_roles.model_type, + model_has_roles.model_id, + current_timestamp() AS _dbt_ran_at + + FROM model_has_roles + + LEFT JOIN roles + ON (model_has_roles.role_id = roles.id) + +), + + +-- FINAL +final__stg_exchange__model_has_roles AS ( + + SELECT + role_id, + role_name, + model_type, + model_id, + _dbt_ran_at + + FROM migrations_join_roles + +) + +SELECT * FROM final__stg_exchange__model_has_roles \ No newline at end of file diff --git a/models/exchange/staging/notifications/stg_exchange__notifications.sql b/models/exchange/staging/notifications/stg_exchange__notifications.sql new file mode 100644 index 0000000..20009d6 --- /dev/null +++ b/models/exchange/staging/notifications/stg_exchange__notifications.sql @@ -0,0 +1,66 @@ +-- ASk edmond to talk about this + +-- IMPORTS +WITH notifications AS ( + SELECT * FROM {{ ref('base_exchange__notifications') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +notifications_join_status AS ( + + SELECT + notifications.id, + notifications.title, + notifications.description, + notifications.subject_type, + notifications.subject_id, + notifications.target_type, + notifications.target_id, + notifications.causer_type, + notifications.causer_id, + + notifications.status, + status.name AS status_name, + + notifications.deleted_at, + notifications.created_at, + notifications.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM notifications + + LEFT JOIN status + ON (notifications.status = status.id) +), + + +-- FINAL +final__stg_exchange__notifications AS ( + + SELECT + id, + title, + description, + subject_type, + subject_id, + target_type, + target_id, + causer_type, + causer_id, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM notifications_join_status + +) + +SELECT * FROM final__stg_exchange__notifications \ No newline at end of file diff --git a/models/exchange/staging/password_resets/stg_exchange__password_resets.sql b/models/exchange/staging/password_resets/stg_exchange__password_resets.sql new file mode 100644 index 0000000..65997b6 --- /dev/null +++ b/models/exchange/staging/password_resets/stg_exchange__password_resets.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH password_resets AS ( + SELECT * FROM {{ ref('base_exchange__password_resets') }} +), + + +-- LOGIC +password_resets_list AS ( + + SELECT + password_resets.id, + password_resets.user_id, + password_resets.token, + password_resets.is_expired, + password_resets.is_complete, + password_resets.created_at, + password_resets.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM password_resets + +), + + +-- FINAL +final__stg_exchange__password_resets AS ( + + SELECT + id, + user_id, + token, + is_expired, + is_complete, + created_at, + updated_at, + _dbt_ran_at + + FROM password_resets_list + +) + +SELECT * FROM final__stg_exchange__password_resets \ No newline at end of file diff --git a/models/exchange/staging/permissions/stg_exchange__permissions.sql b/models/exchange/staging/permissions/stg_exchange__permissions.sql new file mode 100644 index 0000000..5a01d85 --- /dev/null +++ b/models/exchange/staging/permissions/stg_exchange__permissions.sql @@ -0,0 +1,38 @@ +-- IMPORTS +WITH permissions AS ( + SELECT * FROM {{ ref('base_exchange__permissions') }} +), + + +-- LOGIC +permissions_list AS ( + + SELECT + permissions.id, + permissions.name, + permissions.guard_name, + permissions.created_at, + permissions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM permissions + +), + + +-- FINAL +final__stg_exchange__permissions AS ( + + SELECT + id, + name, + guard_name, + created_at, + updated_at, + _dbt_ran_at + + FROM permissions_list + +) + +SELECT * FROM final__stg_exchange__permissions \ No newline at end of file diff --git a/models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql b/models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql new file mode 100644 index 0000000..49975ff --- /dev/null +++ b/models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql @@ -0,0 +1,32 @@ +-- IMPORTS +WITH role_has_permissions AS ( + SELECT * FROM {{ ref('base_exchange__role_has_permissions') }} +), + + +-- LOGIC +role_has_permissions_list AS ( + + SELECT + role_has_permissions.permission_id, + role_has_permissions.role_id, + current_timestamp() AS _dbt_ran_at + + FROM role_has_permissions + +), + + +-- FINAL +final__stg_exchange__role_has_permissions AS ( + + SELECT + permission_id, + role_id, + _dbt_ran_at + + FROM role_has_permissions_list + +) + +SELECT * FROM final__stg_exchange__role_has_permissions \ No newline at end of file diff --git a/models/exchange/staging/roles/stg_exchange__roles.sql b/models/exchange/staging/roles/stg_exchange__roles.sql new file mode 100644 index 0000000..bd4d609 --- /dev/null +++ b/models/exchange/staging/roles/stg_exchange__roles.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH roles AS ( + SELECT * FROM {{ ref('base_exchange__roles') }} +), + + +-- LOGIC +roles_list AS ( + + SELECT + roles.id, + roles.name, + roles.guard_name, + roles.type, + roles.created_at, + roles.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM roles + +), + + +-- FINAL +final__stg_exchange__roles AS ( + + SELECT + id, + name, + guard_name, + type, + created_at, + updated_at, + _dbt_ran_at + + FROM roles_list + +) + +SELECT * FROM final__stg_exchange__roles \ No newline at end of file diff --git a/models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql b/models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql new file mode 100644 index 0000000..5393fd5 --- /dev/null +++ b/models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH segment_constants AS ( + SELECT * FROM {{ ref('base_exchange__segment_constants') }} +), + +segments AS ( + SELECT * FROM {{ ref('base_exchange__segments') }} +), + +-- LOGIC +segment_constants_join_segments AS ( + + SELECT + segment_constants.id, + + segment_constants.segment_id, + segments.name AS segment_name, + segments.type AS segment_type, + + segment_constants.name, + segment_constants.marking_id, + segment_constants.detail, + segment_constants.deleted_at, + segment_constants.created_at, + segment_constants.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM segment_constants + + LEFT JOIN segments + ON (segment_constants.segment_id = segments.id) +), + + +-- FINAL +final__stg_exchange__segment_constants AS ( + + SELECT + id, + segment_id, + segment_name, + segment_type, + name, + marking_id, + detail, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM segment_constants_join_segments + +) + +SELECT * FROM final__stg_exchange__segment_constants \ No newline at end of file diff --git a/models/exchange/staging/segments/stg_exchange__segments.sql b/models/exchange/staging/segments/stg_exchange__segments.sql new file mode 100644 index 0000000..a3ad8d9 --- /dev/null +++ b/models/exchange/staging/segments/stg_exchange__segments.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH segments AS ( + SELECT * FROM {{ ref('base_exchange__segments') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +segments_join_status AS ( + + SELECT + segments.id, + UPPER(segments.name) AS name, + segments.type, + + segments.status, + status.name AS status_name, + + segments.deleted_at, + segments.created_at, + segments.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM segments + + LEFT JOIN status + ON (segments.status = status.id) +), + + +-- FINAL +final__stg_exchange__segments AS ( + + SELECT + id, + name, + type, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM segments_join_status + +) + +SELECT * FROM final__stg_exchange__segments \ No newline at end of file diff --git a/models/exchange/staging/service_types/stg_exchange__service_types.sql b/models/exchange/staging/service_types/stg_exchange__service_types.sql new file mode 100644 index 0000000..12e82f8 --- /dev/null +++ b/models/exchange/staging/service_types/stg_exchange__service_types.sql @@ -0,0 +1,51 @@ +-- IMPORTS +WITH service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +service_types_join_status AS ( + + SELECT + service_types.id, + service_types.name, + + service_types.status, + status.name AS status_name, + + service_types.deleted_at, + service_types.created_at, + service_types.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM service_types + + LEFT JOIN status + ON (service_types.status = status.id) +), + + +-- FINAL +final__stg_exchange__service_types AS ( + + SELECT + id, + name, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM service_types_join_status + +) + +SELECT * FROM final__stg_exchange__service_types \ No newline at end of file diff --git a/models/exchange/staging/states/stg_exchange__states.sql b/models/exchange/staging/states/stg_exchange__states.sql new file mode 100644 index 0000000..5b041f9 --- /dev/null +++ b/models/exchange/staging/states/stg_exchange__states.sql @@ -0,0 +1,66 @@ +-- IMPORTS +WITH states AS ( + SELECT * FROM {{ ref('base_exchange__states') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +states_join_countries_status AS ( + + SELECT + states.id, + + states.country_id, + countries.name AS country_name, + countries.short_code AS country_short_code, + + states.name, + + states.status, + status.name AS status_name, + + states.deleted_at, + states.created_at, + states.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM states + + LEFT JOIN countries + ON (states.country_id = countries.id) + + LEFT JOIN status + ON (states.status = status.id) +), + + +-- FINAL +final__stg_exchange___states AS ( + + SELECT + id, + country_id, + country_name, + country_short_code, + name, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM states_join_countries_status + +) + +SELECT * FROM final__stg_exchange___states \ No newline at end of file diff --git a/models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql b/models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql new file mode 100644 index 0000000..0f95a3a --- /dev/null +++ b/models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql @@ -0,0 +1,139 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transactions.id, + transactions.owner_type, + transactions.owner_id AS transaction_order_id, + + transactions.transaction_type, + transaction_types.name AS transaction_type_name, + + transactions.issued_by AS supplier_company_id, + transactions.received_by AS buyer_company_id, + + transactions.recipient_bank_account_id AS bank_id, + + transactions.payment_method, + payment_methods.name AS payment_method_name, + + transactions.payment_reference, + transactions.bill_no, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + transactions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + transactions.expired_at, + + transactions.status, + status.name AS status_name, + + transactions.deleted_at, + transactions.created_at, + transactions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN currencies AS base_currencies + ON (transactions.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (transactions.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + + WHERE + owner_type = 'App\\Models\\Transaction' + AND + transaction_type = '3' + +), + + +-- FINAL +final__stg_exchange__transaction_costs AS ( + + SELECT + id, + owner_type, + transaction_order_id, + transaction_type, + transaction_type_name, + supplier_company_id, + buyer_company_id, + bank_id, + payment_method, + payment_method_name, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transaction_costs \ No newline at end of file diff --git a/models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql b/models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql new file mode 100644 index 0000000..fef1193 --- /dev/null +++ b/models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql @@ -0,0 +1,34 @@ +-- IMPORTS +WITH group_transactions AS ( + SELECT * FROM {{ ref('base_exchange__group_transactions') }} +), + + +-- LOGIC +group_transactions_list AS ( + + SELECT + group_transactions.id, + group_transactions.group_id, + group_transactions.transaction_id, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM group_transactions + +), + + +-- FINAL +final__stg_exchange__group_transactions AS ( + + SELECT + id, + group_id, + transaction_id, + _dbt_ran_at + + FROM group_transactions_list + +) + +SELECT * FROM final__stg_exchange__group_transactions \ No newline at end of file diff --git a/models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql b/models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql new file mode 100644 index 0000000..2db4783 --- /dev/null +++ b/models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql @@ -0,0 +1,183 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_logs AS ( + SELECT * FROM {{ ref('base_exchange__transaction_logs') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_add_hash_ids AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(id, created_at)) AS id, + transactions.id AS transaction_cost_id, + owner_type, + owner_id, + transaction_type, + issued_by, + received_by, + recipient_bank_account_id, + payment_method, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions +), + + +transaction_logs_unions AS ( + SELECT * FROM transactions_add_hash_ids + UNION + SELECT * FROM transaction_logs +), + + +transaction_logs_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transaction_logs_unions.id, + transaction_logs_unions.transaction_cost_id, + transaction_logs_unions.owner_type, + transaction_logs_unions.owner_id AS transaction_order_id, + + transaction_logs_unions.transaction_type, + transaction_types.name AS transaction_type_name, + + transaction_logs_unions.issued_by AS supplier_company_id, + transaction_logs_unions.received_by AS buyer_company_id, + + transaction_logs_unions.recipient_bank_account_id AS bank_id, + + transaction_logs_unions.payment_method, + payment_methods.name AS payment_method_name, + + transaction_logs_unions.payment_reference, + transaction_logs_unions.bill_no, + + transaction_logs_unions.base_value, + transaction_logs_unions.quote_value, + + transaction_logs_unions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + transaction_logs_unions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + transaction_logs_unions.base_to_quote_currency_exchange_rate, + transaction_logs_unions.base_tax, + transaction_logs_unions.base_service_charge, + transaction_logs_unions.expired_at, + + transaction_logs_unions.status, + status.name AS status_name, + + transaction_logs_unions.deleted_at, + transaction_logs_unions.created_at, + transaction_logs_unions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transaction_logs_unions + + LEFT JOIN transaction_types + ON (transaction_logs_unions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transaction_logs_unions.payment_method = payment_methods.id) + + LEFT JOIN currencies AS base_currencies + ON (transaction_logs_unions.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (transaction_logs_unions.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (transaction_logs_unions.status = status.id) + + + WHERE + owner_type = 'App\\Models\\Transaction' + AND + transaction_type = '3' + +), + + +-- FINAL +final__stg_exchange__transaction_log_costs AS ( + + SELECT + id, + transaction_cost_id, + owner_type, + transaction_order_id, + transaction_type, + transaction_type_name, + supplier_company_id, + buyer_company_id, + bank_id, + payment_method, + payment_method_name, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transaction_logs_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transaction_log_costs \ No newline at end of file diff --git a/models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql b/models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql new file mode 100644 index 0000000..c9d9e89 --- /dev/null +++ b/models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql @@ -0,0 +1,183 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_logs AS ( + SELECT * FROM {{ ref('base_exchange__transaction_logs') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_add_hash_ids AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(id, created_at)) AS id, + transactions.id AS transaction_order_id, + owner_type, + owner_id, + transaction_type, + issued_by, + received_by, + recipient_bank_account_id, + payment_method, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions +), + + +transaction_logs_unions AS ( + SELECT * FROM transactions_add_hash_ids + UNION + SELECT * FROM transaction_logs +), + + +transaction_logs_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transaction_logs_unions.id, + transaction_logs_unions.transaction_order_id, + transaction_logs_unions.owner_type, + transaction_logs_unions.owner_id AS booking_id, + + transaction_logs_unions.transaction_type, + transaction_types.name AS transaction_type_name, + + transaction_logs_unions.issued_by AS seller_company_id, + transaction_logs_unions.received_by AS company_id, + + transaction_logs_unions.recipient_bank_account_id AS bank_id, + + transaction_logs_unions.payment_method, + payment_methods.name AS payment_method_name, + + transaction_logs_unions.payment_reference, + transaction_logs_unions.bill_no, + + transaction_logs_unions.base_value, + transaction_logs_unions.quote_value, + + transaction_logs_unions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + transaction_logs_unions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + transaction_logs_unions.base_to_quote_currency_exchange_rate, + transaction_logs_unions.base_tax, + transaction_logs_unions.base_service_charge, + transaction_logs_unions.expired_at, + + transaction_logs_unions.status, + status.name AS status_name, + + transaction_logs_unions.deleted_at, + transaction_logs_unions.created_at, + transaction_logs_unions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transaction_logs_unions + + LEFT JOIN transaction_types + ON (transaction_logs_unions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transaction_logs_unions.payment_method = payment_methods.id) + + LEFT JOIN currencies AS base_currencies + ON (transaction_logs_unions.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (transaction_logs_unions.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (transaction_logs_unions.status = status.id) + + + WHERE + transaction_logs_unions.owner_type = 'App\\Models\\Booking' + AND + transaction_logs_unions.transaction_type = '1' + +), + + +-- FINAL +final__stg_exchange__transaction_log_orders AS ( + + SELECT + id, + transaction_order_id, + owner_type, + booking_id, + transaction_type, + transaction_type_name, + seller_company_id, + company_id, + bank_id, + payment_method, + payment_method_name, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transaction_logs_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transaction_log_orders \ No newline at end of file diff --git a/models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql b/models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql new file mode 100644 index 0000000..4ac337b --- /dev/null +++ b/models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH transaction_details AS ( + SELECT * FROM {{ ref('base_exchange__transaction_details') }} +), + + +-- LOGIC +transaction_details_list AS ( + + SELECT + transaction_details.id, + transaction_details.transaction_id, + transaction_details.product_code, + transaction_details.product_name, + transaction_details.quantity, + transaction_details.price, + transaction_details.amount, + transaction_details.deleted_at, + transaction_details.created_at, + transaction_details.updated_at, + CURRENT_TIMESTAMP() AS _dbt_ran_at + + FROM transaction_details + +), + + +-- FINAL +final__stg_exchange__transaction_details AS ( + + SELECT + id, + transaction_id, + product_code, + product_name, + quantity, + price, + amount, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transaction_details_list + +) + +SELECT * FROM final__stg_exchange__transaction_details \ No newline at end of file diff --git a/models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql b/models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql new file mode 100644 index 0000000..87a116c --- /dev/null +++ b/models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql @@ -0,0 +1,139 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transactions.id, + transactions.owner_type, + transactions.owner_id AS booking_id, + + transactions.transaction_type, + transaction_types.name AS transaction_type_name, + + transactions.issued_by AS seller_company_id, + transactions.received_by AS company_id, + + transactions.recipient_bank_account_id AS bank_id, + + transactions.payment_method, + payment_methods.name AS payment_method_name, + + transactions.payment_reference, + transactions.bill_no, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + transactions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + transactions.expired_at, + + transactions.status, + status.name AS status_name, + + transactions.deleted_at, + transactions.created_at, + transactions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN currencies AS base_currencies + ON (transactions.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (transactions.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + + WHERE + transactions.owner_type = 'App\\Models\\Booking' + AND + transactions.transaction_type = '1' + +), + + +-- FINAL +final__stg_exchange__transaction_orders AS ( + + SELECT + id, + owner_type, + booking_id, + transaction_type, + transaction_type_name, + seller_company_id, + company_id, + bank_id, + payment_method, + payment_method_name, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transaction_orders \ No newline at end of file diff --git a/models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql b/models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql new file mode 100644 index 0000000..de2b23e --- /dev/null +++ b/models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql @@ -0,0 +1,137 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transactions.id, + transactions.owner_type, + transactions.owner_id AS booking_id, + + transactions.transaction_type, + transaction_types.name AS transaction_type_name, + + transactions.issued_by AS seller_company_id, + transactions.received_by AS company_id, + + transactions.recipient_bank_account_id, + + transactions.payment_method, + payment_methods.name AS payment_method_name, + + transactions.payment_reference, + transactions.bill_no, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + transactions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + transactions.expired_at, + + transactions.status, + status.name AS status_name, + + transactions.deleted_at, + transactions.created_at, + transactions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN currencies AS base_currencies + ON (transactions.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (transactions.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + + WHERE + owner_type = 'App\\Models\\Wallet' + +), + + +-- FINAL +final__stg_exchange__transactions AS ( + + SELECT + id, + owner_type, + booking_id, + transaction_type, + transaction_type_name, + seller_company_id, + company_id, + recipient_bank_account_id, + payment_method, + payment_method_name, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transactions \ No newline at end of file diff --git a/models/exchange/staging/transactions/stg_exchange__transactions.sql b/models/exchange/staging/transactions/stg_exchange__transactions.sql new file mode 100644 index 0000000..6494389 --- /dev/null +++ b/models/exchange/staging/transactions/stg_exchange__transactions.sql @@ -0,0 +1,132 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +currencies AS ( + SELECT * FROM {{ ref('base_exchange__currencies') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transactions.id, + transactions.owner_type, + transactions.owner_id, + + transactions.transaction_type, + transaction_types.name AS transaction_type_name, + + transactions.issued_by, + transactions.received_by, + + transactions.recipient_bank_account_id, + + transactions.payment_method, + payment_methods.name AS payment_method_name, + + transactions.payment_reference, + transactions.bill_no, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + base_currencies.name AS base_currency_name, + base_currencies.short_code AS base_currency_short_code, + base_currencies.symbol AS base_currency_symbol, + + transactions.quote_currency_id, + quote_currencies.name AS quote_currency_name, + quote_currencies.short_code AS quote_currency_short_code, + quote_currencies.symbol AS quote_currency_symbol, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + transactions.expired_at, + + transactions.status, + status.name AS status_name, + + transactions.deleted_at, + transactions.created_at, + transactions.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN currencies AS base_currencies + ON (transactions.base_currency_id = base_currencies.id) + + LEFT JOIN currencies AS quote_currencies + ON (transactions.quote_currency_id = quote_currencies.id) + + LEFT JOIN status + ON (transactions.status = status.id) +), + + +-- FINAL +final__stg_exchange__transactions AS ( + + SELECT + id, + owner_type, + owner_id, + transaction_type, + transaction_type_name, + issued_by, + received_by, + recipient_bank_account_id, + payment_method, + payment_method_name, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + base_currency_name, + base_currency_short_code, + base_currency_symbol, + quote_currency_id, + quote_currency_name, + quote_currency_short_code, + quote_currency_symbol, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_at, + status, + status_name, + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM transactions_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transactions \ No newline at end of file diff --git a/models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql b/models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql new file mode 100644 index 0000000..8d319ef --- /dev/null +++ b/models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH user_email_verifications AS ( + SELECT * FROM {{ ref('base_exchange__user_email_verifications') }} +), + +users AS ( + SELECT * FROM {{ ref('base_exchange__users') }} +), + +-- LOGIC +user_email_verifications_join_users AS ( + + SELECT + user_email_verifications.id, + + users.id AS user_id, + + user_email_verifications.email, + user_email_verifications.token, + user_email_verifications.is_completed, + user_email_verifications.is_activated, + user_email_verifications.is_sent, + user_email_verifications.created_at, + user_email_verifications.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM user_email_verifications + + LEFT JOIN users + ON (user_email_verifications.email = users.email) +), + + +-- FINAL +final__base_exchange__user_email_verifications AS ( + + SELECT + id, + user_id, + email, + token, + is_completed, + is_activated, + is_sent, + created_at, + updated_at, + _dbt_ran_at + + FROM user_email_verifications_join_users + +) + +SELECT * FROM final__base_exchange__user_email_verifications \ No newline at end of file diff --git a/models/exchange/staging/users/stg_exchange__users.sql b/models/exchange/staging/users/stg_exchange__users.sql new file mode 100644 index 0000000..15c6fb8 --- /dev/null +++ b/models/exchange/staging/users/stg_exchange__users.sql @@ -0,0 +1,80 @@ +-- IMPORTS +WITH users AS ( + SELECT * FROM {{ ref('base_exchange__users') }} +), + +employees AS ( + SELECT * FROM {{ ref('base_exchange__employees') }} +), + +user_types AS ( + SELECT * FROM {{ ref('seed_exchange__user_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +users_join_user_types_status AS ( + + SELECT + users.id, + users.name, + users.email, + + users.user_type, + user_types.name AS user_type_name, + + users.status, + status.name AS status_name, + + employees.company_id AS company_id, + + users.remember_token, + users.active_at, + users.deleted_at, + users.created_at, + users.updated_at, + current_timestamp() AS _dbt_ran_at + + FROM users + + LEFT JOIN employees + ON (users.id = employees.user_id) + + LEFT JOIN user_types + ON (users.user_type = user_types.id) + + LEFT JOIN status + ON (users.status = status.id) +), + + +-- FINAL +final__stg_exchange__users AS ( + + SELECT + id, + name, + email, + user_type, + user_type_name, + status, + status_name, + company_id, + remember_token, + active_at + deleted_at, + created_at, + updated_at, + _dbt_ran_at + + FROM users_join_user_types_status + +) + +SELECT * FROM final__stg_exchange__users \ No newline at end of file diff --git a/models/shipping/base/activity_logs/base_shipping__activity_logs.sql b/models/shipping/base/activity_logs/base_shipping__activity_logs.sql new file mode 100644 index 0000000..8f30a63 --- /dev/null +++ b/models/shipping/base/activity_logs/base_shipping__activity_logs.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH activity_logs AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'activity_log') }} +), + + +-- LOGIC +activity_logs_rename AS ( + + SELECT + activity_logs.id, + activity_logs.log_name, + activity_logs.description, + activity_logs.subject_id, + activity_logs.subject_type, + activity_logs.causer_id, + activity_logs.causer_type, + activity_logs.properties, + activity_logs.created_at AS created_datetime, + activity_logs.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM activity_logs +), + + +-- FINAL +final__base_exchange__activity_logs AS ( + + SELECT + id, + log_name, + description, + subject_id, + subject_type, + causer_id, + causer_type, + properties, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM activity_logs_rename +) + +SELECT * FROM final__base_exchange__activity_logs \ No newline at end of file diff --git a/models/shipping/base/addresses/base_shipping__addresses.sql b/models/shipping/base/addresses/base_shipping__addresses.sql new file mode 100644 index 0000000..fc756a3 --- /dev/null +++ b/models/shipping/base/addresses/base_shipping__addresses.sql @@ -0,0 +1,56 @@ +-- IMPORTS +WITH addresses AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'addresses') }} +), + +-- LOGIC +addresses_rename AS ( + + SELECT + addresses.id, + addresses.owner_type, + addresses.owner_id, + addresses.reference, + addresses.street_one AS address_line_one, + addresses.street_two AS address_line_two, + addresses.district_id, + addresses.state_id, + addresses.country_id, + addresses.postcode, + addresses.type, + addresses.default AS is_default, + addresses.status, + addresses.deleted_at AS deleted_datetime, + addresses.created_at AS created_datetime, + addresses.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM addresses +), + +-- FINAL +final__base_shipping__addresses AS ( + + SELECT + id, + owner_type, + owner_id, + reference, + address_line_one, + address_line_two, + district_id, + state_id, + country_id, + postcode, + type, + is_default, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM addresses_rename +) + +SELECT * FROM final__base_shipping__addresses \ No newline at end of file diff --git a/models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql b/models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql new file mode 100644 index 0000000..251794f --- /dev/null +++ b/models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql @@ -0,0 +1,34 @@ +-- IMPORTS +WITH announcement_segments AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'announcement_segment') }} +), + + +-- LOGIC +announcement_segments_rename AS ( + + SELECT + announcement_segments.announcement_id, + announcement_segments.segment_id, + announcement_segments.created_at AS created_datetime, + announcement_segments.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM announcement_segments +), + + +-- FINAL +final__base_shipping__announcement_segments AS ( + + SELECT + announcement_id, + segment_id, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM announcement_segments_rename +) + +SELECT * FROM final__base_shipping__announcement_segments \ No newline at end of file diff --git a/models/shipping/base/announcements/base_shipping__announcements.sql b/models/shipping/base/announcements/base_shipping__announcements.sql new file mode 100644 index 0000000..02e6dc4 --- /dev/null +++ b/models/shipping/base/announcements/base_shipping__announcements.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH announcements AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'announcements') }} +), + + +-- LOGIC +announcements_rename AS ( + + SELECT + id, + title, + description, + starting_on AS started_datetime, + ending_on AS ended_datetime, + is_all_day, + duration, + is_recurring, + recurrence_pattern, + deleted_at AS deleted_datetime, + created_at AS created_datetime, + updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM announcements +), + + +-- FINAL +final__base_shipping__announcements AS ( + + SELECT + id, + title, + description, + started_datetime, + ended_datetime, + is_all_day, + duration, + is_recurring, + recurrence_pattern, + deleted_datetime, + created_datetime, + updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM announcements_rename +) + +SELECT * FROM final__base_shipping__announcements \ No newline at end of file diff --git a/models/shipping/base/companies/base_shipping__companies.sql b/models/shipping/base/companies/base_shipping__companies.sql new file mode 100644 index 0000000..47ac082 --- /dev/null +++ b/models/shipping/base/companies/base_shipping__companies.sql @@ -0,0 +1,45 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'companies') }} +), + + +-- LOGIC +companies_rename AS ( + + SELECT + companies.id, + companies.name, + companies.reference, --Link with company_module + companies.debtor AS account_software_id, + companies.type AS company_type, + companies.status, + companies.deleted_at AS deleted_datetime, + companies.created_at AS created_datetime, + companies.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM companies +), + + +-- FINAL +final__base_shipping__companies AS ( + + SELECT + id, + name, + reference, + account_software_id, + company_type, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM companies_rename + +) + +SELECT * FROM final__base_shipping__companies \ No newline at end of file diff --git a/models/shipping/base/company_connections/base_shipping__company_connections.sql b/models/shipping/base/company_connections/base_shipping__company_connections.sql new file mode 100644 index 0000000..7f14b2c --- /dev/null +++ b/models/shipping/base/company_connections/base_shipping__company_connections.sql @@ -0,0 +1,45 @@ +-- IMPORTS +WITH company_connections AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'company_connections') }} +), + + +-- LOGIC +company_connections_rename AS ( + + SELECT + company_connections.id, + company_connections.inviter_id, + company_connections.invitee_id, + company_connections.inviter_reference, + company_connections.invitee_reference, + company_connections.status, + company_connections.deleted_at AS deleted_datetime, + company_connections.created_at AS created_datetime, + company_connections.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM company_connections +), + + +-- FINAL +final__base_shipping__company_connections AS ( + + SELECT + id, + inviter_id, + invitee_id, + inviter_reference, + invitee_reference, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM company_connections_rename + +) + +SELECT * FROM final__base_shipping__company_connections \ No newline at end of file diff --git a/models/shipping/base/company_employees/base_shipping__company_employees.sql b/models/shipping/base/company_employees/base_shipping__company_employees.sql new file mode 100644 index 0000000..ffeb4b9 --- /dev/null +++ b/models/shipping/base/company_employees/base_shipping__company_employees.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH company_employees AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'company_employees') }} +), + + +-- LOGIC +company_employees_rename AS ( + + SELECT + company_employees.company_module_id, + company_employees.user_id, + company_employees.role_id, + company_employees.status, + company_employees.deleted_at AS deleted_datetime, + company_employees.created_at AS created_datetime, + company_employees.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM company_employees +), + + +-- FINAL +final__base_shipping__company_employees AS ( + + SELECT + company_module_id, + user_id, + role_id, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM company_employees_rename +) + +SELECT * FROM final__base_shipping__company_employees \ No newline at end of file diff --git a/models/shipping/base/company_modules/base_shipping__company_modules.sql b/models/shipping/base/company_modules/base_shipping__company_modules.sql new file mode 100644 index 0000000..c7f1d27 --- /dev/null +++ b/models/shipping/base/company_modules/base_shipping__company_modules.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH company_modules AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'company_modules') }} +), + + +-- LOGIC +company_modules_rename AS ( + + SELECT + company_modules.id, + company_modules.company_id, + company_modules.name, + company_modules.reference AS marking_id, + company_modules.type, + company_modules.status, + company_modules.unity_hash_id, + company_modules.unity_signature, + company_modules.deleted_at AS deleted_datetime, + company_modules.created_at AS created_datetime, + company_modules.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM company_modules +), + + +-- FINAL +final__base_shipping__company_modules AS ( + + SELECT + id, + company_id, + name, + marking_id, + type, + status, + unity_hash_id, + unity_signature, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM company_modules_rename +) + +SELECT * FROM final__base_shipping__company_modules \ No newline at end of file diff --git a/models/shipping/base/connection_segments/base_shipping__connection_segments.sql b/models/shipping/base/connection_segments/base_shipping__connection_segments.sql new file mode 100644 index 0000000..7f6bd84 --- /dev/null +++ b/models/shipping/base/connection_segments/base_shipping__connection_segments.sql @@ -0,0 +1,38 @@ +-- IMPORTS +WITH connection_segments AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'connection_segments') }} +), + + +-- LOGIC +connection_segments_rename AS ( + + SELECT + connection_segments.id, + connection_segments.company_connection_id, + connection_segments.segment_id, + connection_segments.deleted_at AS deleted_datetime, + connection_segments.created_at AS created_datetime, + connection_segments.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM connection_segments +), + + +-- FINAL +final__base_shipping__connection_segments AS ( + + SELECT + id, + company_connection_id, + segment_id, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM connection_segments_rename +) + +SELECT * FROM final__base_shipping__connection_segments \ No newline at end of file diff --git a/models/shipping/base/contacts/base_shipping__contacts.sql b/models/shipping/base/contacts/base_shipping__contacts.sql new file mode 100644 index 0000000..bd18ac0 --- /dev/null +++ b/models/shipping/base/contacts/base_shipping__contacts.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH contacts AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'contacts') }} +), + + +-- LOGIC +contacts_rename AS ( + + SELECT + contacts.id, + contacts.owner_type, + contacts.owner_id, + contacts.reference, + contacts.phone, + contacts.email, + contacts.wechat_id, + contacts.default AS is_default, + contacts.deleted_at AS deleted_datetime, + contacts.created_at AS created_datetime, + contacts.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM contacts + +), + + +-- FINAL +final__base_shipping__contacts AS ( + + SELECT + id, + owner_type, + owner_id, + reference, + phone, + email, + wechat_id, + is_default, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM contacts_rename + +) + +SELECT * FROM final__base_shipping__contacts \ No newline at end of file diff --git a/models/shipping/base/containers/base_shipping__containers.sql b/models/shipping/base/containers/base_shipping__containers.sql new file mode 100644 index 0000000..402d9ee --- /dev/null +++ b/models/shipping/base/containers/base_shipping__containers.sql @@ -0,0 +1,52 @@ +-- IMPORTS +WITH containers AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'containers') }} +), + + +-- LOGIC +containers_rename AS ( + + SELECT + containers.id, + containers.owner_type, + containers.owner_id, + containers.reference, + containers.container_number, + containers.container_type, + containers.seal_reference, + containers.loading_date, + containers.status, + containers.deleted_at AS deleted_datetime, + containers.created_at AS created_datetime, + containers.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM containers + +), + + +-- FINAL +final__base_shipping__containers AS ( + + SELECT + id, + owner_type, + owner_id, + reference, + container_number, + container_type, + seal_reference, + loading_date, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM containers_rename + +) + +SELECT * FROM final__base_shipping__containers \ No newline at end of file diff --git a/models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql b/models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql new file mode 100644 index 0000000..c1d6ba8 --- /dev/null +++ b/models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql @@ -0,0 +1,40 @@ +-- IMPORTS +WITH container_packing_lists AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'container_packing_lists') }} +), + + +-- LOGIC +container_packing_lists_rename AS ( + + SELECT + container_packing_lists.id, + container_packing_lists.container_id, + container_packing_lists.packing_list_id, + container_packing_lists.deleted_at AS deleted_datetime, + container_packing_lists.created_at AS created_datetime, + container_packing_lists.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM container_packing_lists + +), + + +-- FINAL +final__base_shipping__container_packing_lists AS ( + + SELECT + id, + container_id, + packing_list_id, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM container_packing_lists_rename + +) + +SELECT * FROM final__base_shipping__container_packing_lists \ No newline at end of file diff --git a/models/shipping/base/documents/base_shipping__documents.sql b/models/shipping/base/documents/base_shipping__documents.sql new file mode 100644 index 0000000..f419fcf --- /dev/null +++ b/models/shipping/base/documents/base_shipping__documents.sql @@ -0,0 +1,54 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'documents') }} +), + + +-- LOGIC +documents_rename AS ( + + SELECT + documents.id, + documents.owner_type, + documents.owner_id, + documents.document_type, + documents.reference, + documents.status, + documents.issued_date, + documents.expired_date, + documents.approver, + documents.approval_date, + documents.deleted_at AS deleted_datetime, + documents.created_at AS created_datetime, + documents.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM documents + +), + + +-- FINAL +final__base_shipping__documents AS ( + + SELECT + id, + owner_type, + owner_id, + document_type, + reference, + status, + issued_date, + expired_date, + approver, + approval_date, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM documents_rename + +) + +SELECT * FROM final__base_shipping__documents \ No newline at end of file diff --git a/models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql b/models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql new file mode 100644 index 0000000..bc9e2fa --- /dev/null +++ b/models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH failed_jobs AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'failed_jobs') }} +), + + +-- LOGIC +failed_jobs_rename AS ( + + SELECT + failed_jobs.id, + failed_jobs.uuid, + failed_jobs.connection AS is_connection, + failed_jobs.queue, + failed_jobs.payload, + failed_jobs.exception AS is_exception, + failed_jobs.failed_at AS failed_datetime, + current_timestamp() AS _dbt_ran_at + + FROM failed_jobs + +), + + +-- FINAL +final__base_shipping__failed_jobs AS ( + + SELECT + id, + uuid, + is_connection, + queue, + payload, + is_exception, + failed_datetime, + _dbt_ran_at + + FROM failed_jobs_rename + +) + +SELECT * FROM final__base_shipping__failed_jobs \ No newline at end of file diff --git a/models/shipping/base/files/base_shipping__files.sql b/models/shipping/base/files/base_shipping__files.sql new file mode 100644 index 0000000..15013ef --- /dev/null +++ b/models/shipping/base/files/base_shipping__files.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH files AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'files') }} +), + + +-- LOGIC +files_rename AS ( + + SELECT + files.id, + files.document_id, + files.file AS is_file, + files.file_type, + files.deleted_at AS deleted_datetime, + files.created_at AS created_datetime, + files.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM files + +), + + +-- FINAL +final__base_shipping__files AS ( + + SELECT + id, + document_id, + is_file, + file_type, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM files_rename + +) + +SELECT * FROM final__base_shipping__files \ No newline at end of file diff --git a/models/shipping/base/order_roles/base_shipping__order_roles.sql b/models/shipping/base/order_roles/base_shipping__order_roles.sql new file mode 100644 index 0000000..67a827c --- /dev/null +++ b/models/shipping/base/order_roles/base_shipping__order_roles.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH order_roles AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'order_roles') }} +), + + +-- LOGIC +order_roles_rename AS ( + + SELECT + order_roles.id, + order_roles.order_id, + order_roles.company_module_id, + order_roles.role_id, + order_roles.entity_hash_id, + order_roles.entity_signature, + order_roles.deleted_at AS deleted_datetime, + order_roles.created_at AS created_datetime, + order_roles.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM order_roles + +), + + +-- FINAL +final__base_shipping__order_roles AS ( + + SELECT + id, + order_id, + company_module_id, + role_id, + entity_hash_id, + entity_signature, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM order_roles_rename + +) + +SELECT * FROM final__base_shipping__order_roles \ No newline at end of file diff --git a/models/shipping/base/orders/base_shipping__orders.sql b/models/shipping/base/orders/base_shipping__orders.sql new file mode 100644 index 0000000..ca0f407 --- /dev/null +++ b/models/shipping/base/orders/base_shipping__orders.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH orders AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'orders') }} +), + + +-- LOGIC +orders_rename AS ( + + SELECT + orders.id, + orders.company_module_id, + orders.reference, + orders.reference_contract, + orders.type, + orders.status, + orders.deleted_at AS deleted_datetime, + orders.created_at AS created_datetime, + orders.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM orders + +), + + +-- FINAL +final__base_shipping__orders AS ( + + SELECT + id, + company_module_id, + reference, + reference_contract, + type, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM orders_rename + +) + +SELECT * FROM final__base_shipping__orders \ No newline at end of file diff --git a/models/shipping/base/packages/base_shipping__packages.sql b/models/shipping/base/packages/base_shipping__packages.sql new file mode 100644 index 0000000..0bcf81e --- /dev/null +++ b/models/shipping/base/packages/base_shipping__packages.sql @@ -0,0 +1,56 @@ +-- IMPORTS +WITH packages AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'packages') }} +), + + +-- LOGIC +packages_rename AS ( + + SELECT + packages.id, + packages.packing_list_id, + packages.type, + packages.reference, + packages.description, + packages.width, + packages.height, + packages.length, + packages.weight, + packages.quantity, + packages.status, + packages.deleted_at AS deleted_datetime, + packages.created_at AS created_datetime, + packages.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM packages + +), + + +-- FINAL +final__base_shipping__packages AS ( + + SELECT + id, + packing_list_id, + type, + reference, + description, + width, + height, + length, + weight, + quantity, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM packages_rename + +) + +SELECT * FROM final__base_shipping__packages \ No newline at end of file diff --git a/models/shipping/base/packing_lists/base_shipping__packing_lists.sql b/models/shipping/base/packing_lists/base_shipping__packing_lists.sql new file mode 100644 index 0000000..32962e7 --- /dev/null +++ b/models/shipping/base/packing_lists/base_shipping__packing_lists.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH packing_lists AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'packing_lists') }} +), + + +-- LOGIC +packing_lists_rename AS ( + + SELECT + packing_lists.id, + packing_lists.owner_type, + packing_lists.owner_id, + packing_lists.claimant_id, + packing_lists.reference_contract, + packing_lists.reference, + packing_lists.type, + packing_lists.status, + packing_lists.deleted_at AS deleted_datetime, + packing_lists.created_at AS created_datetime, + packing_lists.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM packing_lists + +), + + +-- FINAL +final__base_shipping__packing_lists AS ( + + SELECT + id, + owner_type, + owner_id, + claimant_id, + reference_contract, + reference, + type, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM packing_lists_rename + +) + +SELECT * FROM final__base_shipping__packing_lists \ No newline at end of file diff --git a/models/shipping/base/password_resets/base_shipping__password_resets.sql b/models/shipping/base/password_resets/base_shipping__password_resets.sql new file mode 100644 index 0000000..e9193f2 --- /dev/null +++ b/models/shipping/base/password_resets/base_shipping__password_resets.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH password_resets AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'password_resets') }} +), + + +-- LOGIC +password_resets_rename AS ( + + SELECT + password_resets.id, + password_resets.user_id, + password_resets.token, + password_resets.is_expired, + password_resets.is_complete, + password_resets.created_at AS created_datetime, + password_resets.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM password_resets + +), + + +-- FINAL +final__base_shipping__password_resets AS ( + + SELECT + id, + user_id, + token, + is_expired, + is_complete, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM password_resets_rename + +) + +SELECT * FROM final__base_shipping__password_resets \ No newline at end of file diff --git a/models/shipping/base/remarks/base_shipping__remarks.sql b/models/shipping/base/remarks/base_shipping__remarks.sql new file mode 100644 index 0000000..bc70514 --- /dev/null +++ b/models/shipping/base/remarks/base_shipping__remarks.sql @@ -0,0 +1,44 @@ +-- IMPORTS +WITH remarks AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'remarks') }} +), + + +-- LOGIC +remarks_rename AS ( + + SELECT + remarks.id, + remarks.owner_type, + remarks.owner_id, + remarks.commenter_id, + remarks.content, + remarks.deleted_at AS deleted_datetime, + remarks.created_at AS created_datetime, + remarks.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM remarks + +), + + +-- FINAL +final__base_shipping__remarks AS ( + + SELECT + id, + owner_type, + owner_id, + commenter_id, + content, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM remarks_rename + +) + +SELECT * FROM final__base_shipping__remarks \ No newline at end of file diff --git a/models/shipping/base/schedules/base_shipping__schedules.sql b/models/shipping/base/schedules/base_shipping__schedules.sql new file mode 100644 index 0000000..a680672 --- /dev/null +++ b/models/shipping/base/schedules/base_shipping__schedules.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH schedules AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'schedules') }} +), + + +-- LOGIC +schedules_rename AS ( + + SELECT + schedules.id, + schedules.owner_type, + schedules.owner_id, + schedules.etd, + schedules.eta, + schedules.status, + schedules.deleted_at AS deleted_datetime, + schedules.created_at AS created_datetime, + schedules.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM schedules + +), + + +-- FINAL +final__base_shipping__schedules AS ( + + SELECT + id, + owner_type, + owner_id, + etd, + eta, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM schedules_rename + +) + +SELECT * FROM final__base_shipping__schedules \ No newline at end of file diff --git a/models/shipping/base/segment_constants/base_shipping__segment_constants.sql b/models/shipping/base/segment_constants/base_shipping__segment_constants.sql new file mode 100644 index 0000000..aef6f88 --- /dev/null +++ b/models/shipping/base/segment_constants/base_shipping__segment_constants.sql @@ -0,0 +1,42 @@ +-- IMPORTS +WITH segment_constants AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'segment_constants') }} +), + + +-- LOGIC +segment_constants_rename AS ( + + SELECT + segment_constants.id, + segment_constants.segment_id, + segment_constants.reference, + segment_constants.value, + segment_constants.deleted_at AS deleted_datetime, + segment_constants.created_at AS created_datetime, + segment_constants.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM segment_constants + +), + + +-- FINAL +final__base_shipping__segment_constants AS ( + + SELECT + id, + segment_id, + reference, + value, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM segment_constants_rename + +) + +SELECT * FROM final__base_shipping__segment_constants \ No newline at end of file diff --git a/models/shipping/base/transaction_details/base_shipping__transaction_details.sql b/models/shipping/base/transaction_details/base_shipping__transaction_details.sql new file mode 100644 index 0000000..ec563ed --- /dev/null +++ b/models/shipping/base/transaction_details/base_shipping__transaction_details.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH transaction_details AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transaction_details') }} +), + + +-- LOGIC +transaction_details_rename AS ( + + SELECT + transaction_details.id, + transaction_details.transaction_id, + transaction_details.reference, + transaction_details.name, + transaction_details.quantity, + transaction_details.price, + transaction_details.amount, + transaction_details.deleted_at AS deleted_datetime, + transaction_details.created_at AS created_datetime, + transaction_details.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM transaction_details + +), + + +-- FINAL +final__base_shipping__transaction_details AS ( + + SELECT + id, + transaction_id, + reference, + name, + quantity, + price, + amount, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM transaction_details_rename + +) + +SELECT * FROM final__base_shipping__transaction_details \ No newline at end of file diff --git a/models/shipping/base/transactions/base_shipping__transactions.sql b/models/shipping/base/transactions/base_shipping__transactions.sql new file mode 100644 index 0000000..ccc012d --- /dev/null +++ b/models/shipping/base/transactions/base_shipping__transactions.sql @@ -0,0 +1,72 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transactions') }} +), + + +-- LOGIC +transactions_rename AS ( + + SELECT + transactions.id, + transactions.owner_type, + transactions.owner_id, + transactions.type AS transaction_type, + transactions.issuer AS issued_by, + transactions.receiver AS received_by, + transactions.recipient_bank_account_id, + transactions.payment_method, + transactions.payment_reference, + transactions.bill_no, + transactions.amount AS base_value, + transactions.original_amount AS quote_value, + transactions.currency_id AS base_currency_id, + transactions.original_currency_id AS quote_currency_id, + transactions.currency_rate AS base_to_quote_currency_exchange_rate, + transactions.tax AS base_tax, + transactions.service_charge AS base_service_charge, + transactions.expires_on AS expired_datetime, + transactions.status, + transactions.deleted_at AS deleted_datetime, + transactions.created_at AS created_datetime, + transactions.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM transactions + +), + + +-- FINAL +final__base_shipping__transactions AS ( + + SELECT + id, + owner_type, + owner_id, + transaction_type, + issued_by, + received_by, + recipient_bank_account_id, + payment_method, + payment_reference, + bill_no, + base_value, + quote_value, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + expired_datetime, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM transactions_rename + +) + +SELECT * FROM final__base_shipping__transactions \ No newline at end of file diff --git a/models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql b/models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql new file mode 100644 index 0000000..c03da36 --- /dev/null +++ b/models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql @@ -0,0 +1,52 @@ +-- IMPORTS +WITH transport_arrangements AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'transport_arrangements') }} +), + + +-- LOGIC +transport_arrangements_rename AS ( + + SELECT + transport_arrangements.id, + transport_arrangements.owner_type, + transport_arrangements.owner_id, + transport_arrangements.type, + transport_arrangements.courier, + transport_arrangements.tracking_number, + transport_arrangements.dispatch_date AS dispatched_datetime, + transport_arrangements.drop_date AS droped_datetime, + transport_arrangements.status, + transport_arrangements.deleted_at AS deleted_datetime, + transport_arrangements.created_at AS created_datetime, + transport_arrangements.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM transport_arrangements + +), + + +-- FINAL +final__base_shipping__transport_arrangements AS ( + + SELECT + id, + owner_type, + owner_id, + type, + courier, + tracking_number, + dispatched_datetime, + droped_datetime, + status, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM transport_arrangements_rename + +) + +SELECT * FROM final__base_shipping__transport_arrangements \ No newline at end of file diff --git a/models/shipping/base/users/base_shipping__users.sql b/models/shipping/base/users/base_shipping__users.sql new file mode 100644 index 0000000..fe47323 --- /dev/null +++ b/models/shipping/base/users/base_shipping__users.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH users AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'users') }} +), + + +-- LOGIC +users_rename AS ( + + SELECT + users.id, + users.name, + users.email, + users.type AS user_type, + users.status, + users.remember_token, + users.active_at AS activated_datetime, + users.deleted_at AS deleted_datetime, + users.created_at AS created_datetime, + users.updated_at AS updated_datetime, + current_timestamp() AS _dbt_ran_at + + FROM users + +), + + +-- FINAL +final__base_shipping__users AS ( + + SELECT + id, + name, + email, + user_type, + status, + remember_token, + activated_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_at + + FROM users_rename + +) + +SELECT * FROM final__base_shipping__users \ No newline at end of file diff --git a/models/shipping/source/src_shipping__activity_log.yml b/models/shipping/source/src_shipping__activity_log.yml new file mode 100644 index 0000000..205a7d0 --- /dev/null +++ b/models/shipping/source/src_shipping__activity_log.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: activity_log + description: Raw `activity_log` table from Snowflake + columns: + - name: id + description: Primary key for 'activity_log' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__addresses.yml b/models/shipping/source/src_shipping__addresses.yml new file mode 100644 index 0000000..2318835 --- /dev/null +++ b/models/shipping/source/src_shipping__addresses.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: addresses + description: Raw `addresses` table from Snowflake + columns: + - name: id + description: Primary key for 'addresses' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__announcement_segment.yml b/models/shipping/source/src_shipping__announcement_segment.yml new file mode 100644 index 0000000..94e24b8 --- /dev/null +++ b/models/shipping/source/src_shipping__announcement_segment.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: announcement_segment + description: Raw `announcement_segment` table from Snowflake + columns: + - name: announcement_id + description: Primary key for 'announcement_segment' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__announcements.yml b/models/shipping/source/src_shipping__announcements.yml new file mode 100644 index 0000000..23a8449 --- /dev/null +++ b/models/shipping/source/src_shipping__announcements.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: announcements + description: Raw `announcements` table from Snowflake + columns: + - name: id + description: Primary key for 'announcements' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__companies.yml b/models/shipping/source/src_shipping__companies.yml new file mode 100644 index 0000000..c88b538 --- /dev/null +++ b/models/shipping/source/src_shipping__companies.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: companies + description: Raw `companies` table from Snowflake + columns: + - name: id + description: Primary key for 'companies' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__company_connections.yml b/models/shipping/source/src_shipping__company_connections.yml new file mode 100644 index 0000000..79b37dd --- /dev/null +++ b/models/shipping/source/src_shipping__company_connections.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: company_connections + description: Raw `company_connections` table from Snowflake + columns: + - name: id + description: Primary key for 'company_connections' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__company_employees.yml b/models/shipping/source/src_shipping__company_employees.yml new file mode 100644 index 0000000..514d5c5 --- /dev/null +++ b/models/shipping/source/src_shipping__company_employees.yml @@ -0,0 +1,23 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: company_employees + description: Raw `company_employees` table from Snowflake + columns: + - name: company_module_id + description: Primary key for 'company_employees' + tests: + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__company_modules.yml b/models/shipping/source/src_shipping__company_modules.yml new file mode 100644 index 0000000..b03b0f2 --- /dev/null +++ b/models/shipping/source/src_shipping__company_modules.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: company_modules + description: Raw `company_modules` table from Snowflake + columns: + - name: id + description: Primary key for 'company_modules' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__connection_segments.yml b/models/shipping/source/src_shipping__connection_segments.yml new file mode 100644 index 0000000..bbf35e7 --- /dev/null +++ b/models/shipping/source/src_shipping__connection_segments.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: connection_segments + description: Raw `connection_segments` table from Snowflake + columns: + - name: id + description: Primary key for 'connection_segments' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__contacts.yml b/models/shipping/source/src_shipping__contacts.yml new file mode 100644 index 0000000..e53ba6e --- /dev/null +++ b/models/shipping/source/src_shipping__contacts.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: contacts + description: Raw `contacts` table from Snowflake + columns: + - name: id + description: Primary key for 'contacts' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__container_packing_lists.yml b/models/shipping/source/src_shipping__container_packing_lists.yml new file mode 100644 index 0000000..19000c8 --- /dev/null +++ b/models/shipping/source/src_shipping__container_packing_lists.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: container_packing_lists + description: Raw `container_packing_lists` table from Snowflake + columns: + - name: id + description: Primary key for 'container_packing_lists' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__containers.yml b/models/shipping/source/src_shipping__containers.yml new file mode 100644 index 0000000..4a2ae0f --- /dev/null +++ b/models/shipping/source/src_shipping__containers.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: containers + description: Raw `containers` table from Snowflake + columns: + - name: id + description: Primary key for 'containers' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__documents.yml b/models/shipping/source/src_shipping__documents.yml new file mode 100644 index 0000000..5d685b3 --- /dev/null +++ b/models/shipping/source/src_shipping__documents.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: documents + description: Raw `documents` table from Snowflake + columns: + - name: id + description: Primary key for 'documents' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__failed_jobs.yml b/models/shipping/source/src_shipping__failed_jobs.yml new file mode 100644 index 0000000..731dadb --- /dev/null +++ b/models/shipping/source/src_shipping__failed_jobs.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: failed_jobs + description: Raw `failed_jobs` table from Snowflake + columns: + - name: id + description: Primary key for 'failed_jobs' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__files.yml b/models/shipping/source/src_shipping__files.yml new file mode 100644 index 0000000..db26432 --- /dev/null +++ b/models/shipping/source/src_shipping__files.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: files + description: Raw `files` table from Snowflake + columns: + - name: id + description: Primary key for 'files' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__order_roles.yml b/models/shipping/source/src_shipping__order_roles.yml new file mode 100644 index 0000000..de69e2b --- /dev/null +++ b/models/shipping/source/src_shipping__order_roles.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: order_roles + description: Raw `order_roles` table from Snowflake + columns: + - name: id + description: Primary key for 'order_roles' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__orders.yml b/models/shipping/source/src_shipping__orders.yml new file mode 100644 index 0000000..3e3e5c1 --- /dev/null +++ b/models/shipping/source/src_shipping__orders.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: orders + description: Raw `orders` table from Snowflake + columns: + - name: id + description: Primary key for 'orders' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__packages.yml b/models/shipping/source/src_shipping__packages.yml new file mode 100644 index 0000000..930afb3 --- /dev/null +++ b/models/shipping/source/src_shipping__packages.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: packages + description: Raw `packages` table from Snowflake + columns: + - name: id + description: Primary key for 'packages' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__packing_lists.yml b/models/shipping/source/src_shipping__packing_lists.yml new file mode 100644 index 0000000..bc247dd --- /dev/null +++ b/models/shipping/source/src_shipping__packing_lists.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: packing_lists + description: Raw `packing_lists` table from Snowflake + columns: + - name: id + description: Primary key for 'packing_lists' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__password_resets.yml b/models/shipping/source/src_shipping__password_resets.yml new file mode 100644 index 0000000..5a805d7 --- /dev/null +++ b/models/shipping/source/src_shipping__password_resets.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: password_resets + description: Raw `password_resets` table from Snowflake + columns: + - name: id + description: Primary key for 'password_resets' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__remarks.yml b/models/shipping/source/src_shipping__remarks.yml new file mode 100644 index 0000000..65fa3bd --- /dev/null +++ b/models/shipping/source/src_shipping__remarks.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: remarks + description: Raw `remarks` table from Snowflake + columns: + - name: id + description: Primary key for 'remarks' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__schedules.yml b/models/shipping/source/src_shipping__schedules.yml new file mode 100644 index 0000000..94300e4 --- /dev/null +++ b/models/shipping/source/src_shipping__schedules.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: schedules + description: Raw `schedules` table from Snowflake + columns: + - name: id + description: Primary key for 'schedules' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__segment_constants.yml b/models/shipping/source/src_shipping__segment_constants.yml new file mode 100644 index 0000000..4bde167 --- /dev/null +++ b/models/shipping/source/src_shipping__segment_constants.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: segment_constants + description: Raw `segment_constants` table from Snowflake + columns: + - name: id + description: Primary key for 'segment_constants' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__transaction_details.yml b/models/shipping/source/src_shipping__transaction_details.yml new file mode 100644 index 0000000..a8583a9 --- /dev/null +++ b/models/shipping/source/src_shipping__transaction_details.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: transaction_details + description: Raw `transaction_details` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_details' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__transactions.yml b/models/shipping/source/src_shipping__transactions.yml new file mode 100644 index 0000000..20952eb --- /dev/null +++ b/models/shipping/source/src_shipping__transactions.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: transactions + description: Raw `transactions` table from Snowflake + columns: + - name: id + description: Primary key for 'transactions' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__transport_arrangements.yml b/models/shipping/source/src_shipping__transport_arrangements.yml new file mode 100644 index 0000000..4994542 --- /dev/null +++ b/models/shipping/source/src_shipping__transport_arrangements.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: transport_arrangements + description: Raw `transport_arrangements` table from Snowflake + columns: + - name: id + description: Primary key for 'transport_arrangements' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__users.yml b/models/shipping/source/src_shipping__users.yml new file mode 100644 index 0000000..b13fbb4 --- /dev/null +++ b/models/shipping/source/src_shipping__users.yml @@ -0,0 +1,24 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: Source data from snowflake 'DEV_CIEF_RAW_DB' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + + tables: + - name: users + description: Raw `users` table from Snowflake + columns: + - name: id + description: Primary key for 'users' + tests: + - unique + - not_null + loaded_at_field: _source_loaded_at + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..15f3370 --- /dev/null +++ b/packages.yml @@ -0,0 +1,20 @@ +# Can find packages in https://hub.getdbt.com/ , github, or local create own packages +# Example +# packages: +# - name: fivertan/github +# version: 0.1.1 +# - git: https://github.com/etc.... +# revision: master + + + # - package: calogica/dbt_date + # version: 0.6.0 + # description: |- + # Warnning this package includes dbt-utils so there"s no need to also import dbt-utils in your local project. (In fact, you may get an error if you do.) + # Link https://github.com/calogica/dbt-date/#get_date_dimensionstart_date-end_date +packages: + - package: calogica/dbt_expectations + version: 0.8.2 + + - package: dbt-labs/dbt_utils + version: 1.0.0 \ No newline at end of file diff --git a/seeds/.gitkeep b/seeds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/seeds/exchange/seed_exchange__bank_types.csv b/seeds/exchange/seed_exchange__bank_types.csv new file mode 100644 index 0000000..4c19f40 --- /dev/null +++ b/seeds/exchange/seed_exchange__bank_types.csv @@ -0,0 +1,7 @@ +ID,CATEGORY,NAME +1,DEFAULT,PERSONAL +2,DEFAULT,EXTERNAL +3,DEFAULT,ALIPAY +1,REPHRASE,PERSONAL +2,REPHRASE,EXTERNAL +3,REPHRASE,EXTERNAL_ALIPAY \ No newline at end of file diff --git a/seeds/exchange/seed_exchange__business_types.csv b/seeds/exchange/seed_exchange__business_types.csv new file mode 100644 index 0000000..7e7cecc --- /dev/null +++ b/seeds/exchange/seed_exchange__business_types.csv @@ -0,0 +1,5 @@ +ID,CATEGORY,NAME +1,DEFAULT,FREIGHT_FORWARDER +2,DEFAULT,IMPORTER +3,DEFAULT,CURRENCY_VENDOR +4,DEFAULT,TRANSFER_AGENT diff --git a/seeds/exchange/seed_exchange__company_types.csv b/seeds/exchange/seed_exchange__company_types.csv new file mode 100644 index 0000000..f2362c4 --- /dev/null +++ b/seeds/exchange/seed_exchange__company_types.csv @@ -0,0 +1,6 @@ +ID,CATEGORY,NAME +0,DEFAULT,PERSONAL +1,DEFAULT,CORPORATE +0,2_AS_CORPORATE,PERSONAL +1,2_AS_CORPORATE,CORPORATE +2,2_AS_CORPORATE,CORPORATE \ No newline at end of file diff --git a/seeds/exchange/seed_exchange__holiday_and_non_working_day_lists.csv b/seeds/exchange/seed_exchange__holiday_and_non_working_day_lists.csv new file mode 100644 index 0000000..f0acea2 --- /dev/null +++ b/seeds/exchange/seed_exchange__holiday_and_non_working_day_lists.csv @@ -0,0 +1,15 @@ +date,date_desc,is_holiday,holiday_desc,holiday_race,holiday_country,holiday_state,is_non_working_day,non_working_day_desc,non_working_day_state +23/12/2022,,0,,,,,0,, +24/12/2022,,0,,,,,1,Week End, +25/12/2022,,1,Christmas - 1,Christian,[Malaysia],,1,Public Holiday, +26/12/2022,,1,Christmas - 2,Christian,[Malaysia],,1,Public Holiday, +27/12/2022,,1,Christmas - 3 (Replacement),Christian,[Malaysia],,1,Public Holiday, +28/12/2022,,0,,,,,0,, +29/12/2022,,0,,,,,0,, +30/12/2022,,0,,,,,0,, +31/12/2022,,0,,,,,1,Week End, +1/1/2023,,1,New Year - 1,Worldwide,[Malaysia; China],,1,Public Holiday, +2/1/2023,,0,New Year - 2 (Replacement),Worldwide,[Malaysia; China],,1,Public Holiday, +3/1/2023,,0,,,,,0,, +4/1/2023,,0,,,,,0,, +5/1/2023,,0,,,,,0,, diff --git a/seeds/exchange/seed_exchange__payment_methods.csv b/seeds/exchange/seed_exchange__payment_methods.csv new file mode 100644 index 0000000..7726ae0 --- /dev/null +++ b/seeds/exchange/seed_exchange__payment_methods.csv @@ -0,0 +1,6 @@ +ID,CATEGORY,NAME +1,DEFAULT,CASH +2,DEFAULT,CHEQUE +3,DEFAULT,BA +4,DEFAULT,WALLET +5,DEFAULT,PAYMENT_GATEWAY \ No newline at end of file diff --git a/seeds/exchange/seed_exchange__status.csv b/seeds/exchange/seed_exchange__status.csv new file mode 100644 index 0000000..5fbd987 --- /dev/null +++ b/seeds/exchange/seed_exchange__status.csv @@ -0,0 +1,9 @@ +ID,CATEGORY,NAME +0,DEFAULT,PENDING_SUBMISSION +1,DEFAULT,PENDING_VERIFICATION +2,DEFAULT,APPROVED +3,DEFAULT,COMPLETED +4,DEFAULT,REJECTED +5,DEFAULT,SUSPENDED +6,DEFAULT,EXPIRED +7,DEFAULT,REFUNDED \ No newline at end of file diff --git a/seeds/exchange/seed_exchange__transaction_types.csv b/seeds/exchange/seed_exchange__transaction_types.csv new file mode 100644 index 0000000..3048db0 --- /dev/null +++ b/seeds/exchange/seed_exchange__transaction_types.csv @@ -0,0 +1,15 @@ +ID,CATEGORY,NAME +0,DEFAULT,PAYMENT_ATTEMPT +1,DEFAULT,PAYMENT +2,DEFAULT,INVOICE +3,DEFAULT,BILL +4,DEFAULT,PROFORMA +5,DEFAULT,TOP_UP +6,DEFAULT,REFUND +7,DEFAULT,PURCHASE_ORDER +8,DEFAULT,SUPPLIER_DELIVER +9,DEFAULT,CREDIT_NOTE +10,DEFAULT,WITHDRAW +11,DEFAULT,DEBIT_NOTE +12,DEFAULT,TRANSFER_FEE +13,DEFAULT,CASH_BACK \ No newline at end of file diff --git a/seeds/exchange/seed_exchange__user_types.csv b/seeds/exchange/seed_exchange__user_types.csv new file mode 100644 index 0000000..a251c2f --- /dev/null +++ b/seeds/exchange/seed_exchange__user_types.csv @@ -0,0 +1,9 @@ +ID,CATEGORY,NAME +0,DEFAULT,SHADOW_ADMIN +1,DEFAULT,SUPER_ADMIN +2,DEFAULT,ADMIN +3,DEFAULT,USER +4,DEFAULT,CURRENCY_SUPPLIER +5,DEFAULT,MONEY_MULE +6,DEFAULT,ORIGIN_ACCOUNT_ADMIN +7,DEFAULT,DESTINATION_ACCOUNT_ADMIN \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__address_types.csv b/seeds/shipping/seed_shipping__address_types.csv new file mode 100644 index 0000000..9616a2f --- /dev/null +++ b/seeds/shipping/seed_shipping__address_types.csv @@ -0,0 +1,4 @@ +ID,CATEGORY,NAME +1,DEFAULT,BILLING +2,DEFAULT,DELIVERY +3,DEFAULT,PICK_UP \ No newline at end of file diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/snapshots/crisp/snap_crisp__get_people_subscription_status.sql b/snapshots/crisp/snap_crisp__get_people_subscription_status.sql new file mode 100644 index 0000000..deaa136 --- /dev/null +++ b/snapshots/crisp/snap_crisp__get_people_subscription_status.sql @@ -0,0 +1,13 @@ +{% snapshot snap_crisp__get_people_subscription_status %} + + {{ + config( + strategy='check', + unique_key='people_id', + check_cols=['email'], + ) + }} + + select * from {{ source('src_crisp_api', 'get_people_subscription_status') }} + +{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/exchange/snap_exchange__segment_companies.sql b/snapshots/exchange/snap_exchange__segment_companies.sql new file mode 100644 index 0000000..8207332 --- /dev/null +++ b/snapshots/exchange/snap_exchange__segment_companies.sql @@ -0,0 +1,16 @@ +{% snapshot snap_exchange__segment_companies %} + + {{ + config( + strategy='check', + unique_key="segment_id||'-'||company_id", + check_cols=['segment_id'], + ) + }} + + select + CONCAT(segment_id,'-',company_id) as concat_id, + * + from {{ source('src_exchange_mysql', 'segment_companies') }} + +{% endsnapshot %} \ No newline at end of file diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29