From 2d2501cbfedd7c434e0688c080fa30ea35fda7c8 Mon Sep 17 00:00:00 2001 From: Yam ZhengLim Date: Tue, 9 May 2023 09:53:28 +0000 Subject: [PATCH] DBT structure change --- analyses/exchange/blog_ger.sql | 3 - analyses/exchange/company_conversionrate.sql | 299 --------- analyses/shipping/y2022_companies_ltv.sql | 84 --- dbt_project.yml | 118 ++-- ...sers_and_companies_and_company_marking.sql | 68 -- .../marts/warehouse/dim_crisp__companies.sql | 86 --- .../marts/warehouse/dim_crisp__users.sql | 48 -- .../fct_crisp__user_email_subscriptions.sql | 15 - ...rc_crisp__get_message_in_conversations.yml | 26 - .../source/src_crisp__get_people_data.yml | 24 - ..._crisp__get_people_subscription_status.yml | 24 - .../source/src_crisp__list_conversations.yml | 24 - .../src_crisp__list_people_profiles.yml | 24 - .../src_crisp__list_website_operator.yml | 24 - .../staging/stg_crisp__conversations.sql | 22 - models/crisp/staging/stg_crisp__messages.sql | 32 - models/crisp/staging/stg_crisp__operators.sql | 20 - .../crisp/staging/stg_crisp__people_data.sql | 23 - .../stg_crisp__people_email_subscriptions.sql | 20 - .../addresses/base_exchange__addresses.sql | 48 -- .../base_exchange__announcement_segments.sql | 34 - .../base_exchange__announcements.sql | 50 -- .../bank_logs/base_exchange__bank_logs.sql | 61 -- .../base/banks/base_exchange__banks.sql | 58 -- .../base_exchange__booking_logs.sql | 55 -- .../base/bookings/base_exchange__bookings.sql | 53 -- .../companies/base_exchange__companies.sql | 47 -- .../base/contacts/base_exchange__contacts.sql | 48 -- .../countries/base_exchange__countries.sql | 41 -- .../base_exchange__currency_logs.sql | 39 -- .../base_exchange__currency_rates.sql | 43 -- .../districts/base_exchange__districts.sql | 44 -- .../documents/base_exchange__documents.sql | 53 -- .../employees/base_exchange__employees.sql | 32 - .../base_exchange__failed_jobs.sql | 40 -- .../base/files/base_exchange__files.sql | 42 -- .../base_exchange__group_transactions.sql | 34 - .../base/groups/base_exchange__groups.sql | 55 -- .../base/jobs/base_exchange__jobs.sql | 41 -- .../migrations/base_exchange__migrations.sql | 33 - .../base_exchange__model_has_roles.sql | 34 - .../base_exchange__notifications.sql | 53 -- .../base_exchange__password_resets.sql | 42 -- .../base_exchange__permissions.sql | 38 -- .../base_exchange__role_has_permissions.sql | 32 - .../base/roles/base_exchange__roles.sql | 40 -- .../base_exchange__segment_companies.sql | 43 -- .../base_exchange__segment_constants.sql | 43 -- .../base/segments/base_exchange__segments.sql | 40 -- .../base_exchange__service_types.sql | 39 -- .../base/states/base_exchange__states.sql | 41 -- .../base_exchange__transaction_details.sql | 48 -- .../base_exchange__transaction_logs.sql | 73 --- .../base_exchange__transactions.sql | 71 -- .../base/users/base_exchange__users.sql | 47 -- .../base_exchange__wallet_logs.sql | 47 -- .../base/wallets/base_exchange__wallets.sql | 45 -- ...hange__booking_logs_get_estimate_rates.sql | 112 ---- .../int_exchange__companies_level_segment.sql | 81 --- ...hange__dedupe_user_email_verifications.sql | 45 -- .../int_exchange__latest_company_contacts.sql | 38 -- ...transaction_costs_remove_duplicate_row.sql | 72 -- ...e__transaction_log_orders_get_user_ids.sql | 28 - ...nsaction_log_orders_remove_undo_status.sql | 26 - ...hange__transaction_orders_get_user_ids.sql | 30 - .../rep_exchange__daily_orders.sql | 234 ------- .../test_exchange__daily_orders.yml | 211 ------ .../warehouse/dim_exchange__companies.sql | 178 ----- .../marts/warehouse/dim_exchange__dates.sql | 202 ------ .../marts/warehouse/dim_exchange__users.sql | 45 -- .../warehouse/fct_exchange__booking_logs.sql | 81 --- .../warehouse/fct_exchange__bookings.sql | 60 -- .../fct_exchange__email_verifications.sql | 48 -- .../fct_exchange__transaction_costs.sql | 60 -- .../fct_exchange__transaction_log_costs.sql | 61 -- .../fct_exchange__transaction_log_orders.sql | 116 ---- .../fct_exchange__transaction_orders.sql | 92 --- .../test_exchange__transaction_orders.yml | 0 .../marts/warehouse/test_exchange__users.yml | 0 .../source/src_activity_log__exchange.yml | 24 - .../source/src_addresses__exchange.yml | 24 - .../src_announcement_segment__exchange.yml | 24 - .../source/src_announcements__exchange.yml | 24 - .../source/src_bank_logs__exchange.yml | 24 - .../exchange/source/src_banks__exchange.yml | 24 - .../source/src_booking_logs__exchange.yml | 24 - .../source/src_bookings__exchange.yml | 24 - .../source/src_companies__exchange.yml | 24 - .../source/src_contacts__exchange.yml | 24 - .../source/src_countries__exchange.yml | 24 - .../source/src_currencies__exchange.yml | 24 - .../source/src_currency_logs__exchange.yml | 24 - .../source/src_currency_rate_logs.yml | 24 - .../source/src_currency_rates__exchange.yml | 24 - .../source/src_districts__exchange.yml | 24 - .../source/src_documents__exchange.yml | 24 - .../source/src_employees__exchange.yml | 22 - .../source/src_failed_jobs__exchange.yml | 24 - .../exchange/source/src_files__exchange.yml | 24 - .../src_group_transactions__exchange.yml | 24 - .../exchange/source/src_groups__exchange.yml | 24 - models/exchange/source/src_jobs__exchange.yml | 24 - .../source/src_migrations__exchange.yml | 24 - .../source/src_model_has_roles__exchange.yml | 22 - .../source/src_notifications__exchange.yml | 24 - .../source/src_password_resets__exchange.yml | 24 - .../source/src_permissions__exchange.yml | 24 - .../src_role_has_permissions__exchange.yml | 22 - .../exchange/source/src_roles__exchange.yml | 24 - .../src_segment_companies__exchange.yml | 22 - .../src_segment_constants__exchange.yml | 24 - .../source/src_segments__exchange.yml | 24 - .../source/src_service_types__exchange.yml | 24 - .../exchange/source/src_states__exchange.yml | 24 - .../src_transaction_detail__exchange.yml | 24 - .../source/src_transaction_logs__exchange.yml | 24 - .../source/src_transactions__exchange.yml | 24 - ...src_user_email_verifications__exchange.yml | 24 - .../exchange/source/src_users__exchange.yml | 24 - .../source/src_wallet_logs__exchange.yml | 24 - .../exchange/source/src_wallets__exchange.yml | 24 - .../stg_exchange__activity_logs.sql | 46 -- .../stg_exchange__announcement_segments.sql | 44 -- .../stg_exchange__announcements.sql | 50 -- .../bank_logs/stg_exchange__bank_logs.sql | 93 --- .../staging/banks/stg_exchange__banks.sql | 91 --- .../stg_exchange__booking_logs.sql | 165 ----- ...doc_generic_test_stg_bookings__exchange.md | 15 - .../generic_test_stg_bookings__exchange.yml | 100 --- .../bookings/stg_exchange__bookings.sql | 128 ---- .../companies/stg_exchange__companies.sql | 80 --- ...ange__companies_and_segments_relations.sql | 43 -- ...xchange__companies_and_users_relations.sql | 46 -- .../contacts/stg_exchange__contacts.sql | 59 -- .../countries/stg_exchange__countries.sql | 41 -- .../stg_exchange__currency_logs.sql | 55 -- .../stg_exchange__current_currency_rates.sql | 79 --- .../stg_exchange__current_wallets.sql | 61 -- .../districts/stg_exchange__districts.sql | 39 -- .../documents/stg_exchange__documents.sql | 66 -- .../failed_jobs/stg_exchange__failed_jobs.sql | 40 -- .../staging/files/stg_exchange__files.sql | 42 -- .../group_costs/stg_exchange__group_costs.sql | 93 --- .../staging/jobs/stg_exchange__jobs.sql | 42 -- .../migrations/stg_exchange__migrations.sql | 34 - .../stg_exchange__model_has_roles.sql | 43 -- .../stg_exchange__notifications.sql | 66 -- .../stg_exchange__password_resets.sql | 42 -- .../permissions/stg_exchange__permissions.sql | 38 -- .../stg_exchange__role_has_permissions.sql | 32 - .../staging/roles/stg_exchange__roles.sql | 40 -- .../stg_exchange__segment_constants.sql | 55 -- .../segments/stg_exchange__segments.sql | 53 -- .../stg_exchange__service_types.sql | 51 -- .../staging/states/stg_exchange__states.sql | 66 -- .../stg_exchange__transaction_costs.sql | 139 ---- ...action_costs_and_group_costs_relations.sql | 34 - .../stg_exchange__transaction_log_costs.sql | 183 ------ .../stg_exchange__transaction_log_orders.sql | 183 ------ ...tg_exchange__transaction_order_details.sql | 48 -- .../stg_exchange__transaction_orders.sql | 139 ---- .../stg_exchange__transaction_wallets.sql | 137 ---- .../stg_exchange__transactions.sql | 132 ---- ...stg_exchange__user_email_verifications.sql | 53 -- .../staging/users/stg_exchange__users.sql | 80 --- ...t_crisp__messages_generate_dialog_ids.sql} | 61 +- ..._bookings_get_estimate_booking_values.sql} | 65 +- .../int_exchange__companies_join_segments.sql | 95 +++ ...hange__transaction_orders_get_user_ids.sql | 61 ++ .../reporting/_reporting_models.yml} | 0 .../havent_do_metric_comparisons.sql} | 0 ...p_exchange__company_conversion_funnels.sql | 157 +++-- ...exchange__company_engagement_behaviors.sql | 216 ++++++ .../reporting/rep_exchange__company_lists.sql | 78 +++ .../reporting/rep_exchange__daily_orders.sql | 257 ++++++++ models/marts/reporting/rep_exchange__rfm.sql | 290 ++++++++ models/marts/warehouse/_warehouse_models.yml | 47 ++ models/marts/warehouse/dim__dates.sql | 201 ++++++ models/marts/warehouse/dim_crisp__users.sql | 5 + .../warehouse/dim_exchange__companies.sql | 197 ++++++ .../warehouse/dim_exchange__currencies.sql | 43 ++ .../marts/warehouse/fct_crisp__dialogs.sql | 125 ++-- .../warehouse/fct_exchange__bookings.sql | 107 +++ ...t_exchange__company_identity_documents.sql | 20 +- .../fct_exchange__transaction_costs.sql | 134 ++++ .../fct_exchange__transaction_orders.sql | 158 +++++ ...fct_exchange__user_email_verifications.sql | 57 ++ .../addresses/base_shipping__addresses.sql | 56 -- .../base_shipping__announcement_segments.sql | 34 - .../base_shipping__announcements.sql | 50 -- .../companies/base_shipping__companies.sql | 45 -- .../base_shipping__company_connections.sql | 45 -- .../base_shipping__company_employees.sql | 40 -- .../base_shipping__company_modules.sql | 48 -- .../base_shipping__connection_segments.sql | 38 -- .../base/contacts/base_shipping__contacts.sql | 50 -- .../containers/base_shipping__containers.sql | 52 -- ...base_shipping__container_packing_lists.sql | 40 -- .../documents/base_shipping__documents.sql | 54 -- .../base_shipping__failed_jobs.sql | 42 -- .../base/files/base_shipping__files.sql | 42 -- .../base_shipping__order_roles.sql | 46 -- .../base/packages/base_shipping__packages.sql | 56 -- .../base_shipping__packing_lists.sql | 50 -- .../base_shipping__password_resets.sql | 42 -- .../base/remarks/base_shipping__remarks.sql | 44 -- .../schedules/base_shipping__schedules.sql | 46 -- .../base_shipping__segment_constants.sql | 42 -- .../base_shipping__transaction_details.sql | 48 -- .../base_shipping__transactions.sql | 72 -- .../base_shipping__transport_arrangements.sql | 52 -- .../base/users/base_shipping__users.sql | 48 -- .../source/src_shipping__activity_log.yml | 24 - .../source/src_shipping__addresses.yml | 24 - .../src_shipping__announcement_segment.yml | 24 - .../source/src_shipping__announcements.yml | 24 - .../source/src_shipping__companies.yml | 24 - .../src_shipping__company_connections.yml | 24 - .../src_shipping__company_employees.yml | 23 - .../source/src_shipping__company_modules.yml | 24 - .../src_shipping__connection_segments.yml | 24 - .../source/src_shipping__contacts.yml | 24 - .../src_shipping__container_packing_lists.yml | 24 - .../source/src_shipping__containers.yml | 24 - .../source/src_shipping__documents.yml | 24 - .../source/src_shipping__failed_jobs.yml | 24 - .../shipping/source/src_shipping__files.yml | 24 - .../source/src_shipping__order_roles.yml | 24 - .../shipping/source/src_shipping__orders.yml | 24 - .../source/src_shipping__packages.yml | 24 - .../source/src_shipping__packing_lists.yml | 24 - .../source/src_shipping__password_resets.yml | 24 - .../shipping/source/src_shipping__remarks.yml | 24 - .../source/src_shipping__schedules.yml | 24 - .../src_shipping__segment_constants.yml | 24 - .../src_shipping__transaction_details.yml | 24 - .../source/src_shipping__transactions.yml | 24 - .../src_shipping__transport_arrangements.yml | 24 - .../shipping/source/src_shipping__users.yml | 24 - models/staging/crisp/_crisp__docs.md | 45 ++ models/staging/crisp/_crisp__sources.yml | 85 +++ .../crisp/base/base_crisp__user_data.sql | 48 ++ .../base_crisp__user_email_subscriptions.sql | 41 ++ .../crisp/base/base_crisp__users.sql} | 89 ++- .../crisp/stg_crisp__conversations.sql | 46 ++ models/staging/crisp/stg_crisp__messages.sql | 65 ++ models/staging/crisp/stg_crisp__operators.sql | 37 ++ models/staging/crisp/stg_crisp__users.sql | 149 +++++ models/staging/exchange/_exchange__docs.md | 21 + models/staging/exchange/_exchange__models.yml | 26 + .../staging/exchange/_exchange__sources.yml | 618 ++++++++++++++++++ .../base}/base_exchange__activity_logs.sql | 30 +- .../exchange/base/base_exchange__banks.sql | 64 ++ .../base/base_exchange__booking_arch_logs.sql | 63 ++ .../exchange/base/base_exchange__bookings.sql | 61 ++ .../base/base_exchange__companies.sql | 54 ++ .../base_exchange__companies_brg_segments.sql | 43 ++ .../base/base_exchange__company_addresses.sql | 81 +++ .../base/base_exchange__company_contacts.sql | 68 ++ .../base/base_exchange__countries.sql | 50 ++ .../base}/base_exchange__currencies.sql | 20 +- .../base_exchange__currency_rate_logs.sql | 32 +- .../base/base_exchange__currency_rates.sql | 51 ++ .../base/base_exchange__districts.sql | 53 ++ .../base/base_exchange__documents.sql | 62 ++ .../base/base_exchange__group_costs.sql | 62 ++ .../exchange/base/base_exchange__segments.sql | 46 ++ .../base/base_exchange__service_types.sql | 48 ++ .../exchange/base/base_exchange__states.sql | 50 ++ .../base_exchange__transaction_arch_logs.sql | 81 +++ .../base/base_exchange__transactions.sql | 79 +++ ...ase_exchange__user_email_verifications.sql | 30 +- .../exchange/base/base_exchange__users.sql | 51 ++ .../exchange/stg_exchange__activity_logs.sql} | 32 +- .../staging/exchange/stg_exchange__banks.sql | 81 +++ .../stg_exchange__booking_documents.sql | 50 ++ .../exchange/stg_exchange__booking_logs.sql | 124 ++++ .../exchange/stg_exchange__bookings.sql | 104 +++ .../exchange/stg_exchange__companies.sql | 81 +++ .../stg_exchange__companies_brg_segments.sql | 43 ++ .../stg_exchange__companies_brg_users.sql | 39 ++ .../stg_exchange__company_addresses.sql | 45 +- .../stg_exchange__company_contacts.sql | 67 ++ .../stg_exchange__company_documents.sql | 66 ++ .../stg_exchange__company_wallets.sql | 51 ++ .../exchange}/stg_exchange__currencies.sql | 35 +- .../stg_exchange__currency_rate_logs.sql | 82 +-- .../stg_exchange__group_cost_documents.sql | 49 ++ .../exchange/stg_exchange__group_costs.sql | 76 +++ .../exchange/stg_exchange__segments.sql | 55 ++ .../stg_exchange__transaction_cost_logs.sql | 140 ++++ .../stg_exchange__transaction_costs.sql | 136 ++++ ...nge__transaction_costs_brg_group_costs.sql | 43 ++ .../stg_exchange__transaction_documents.sql | 65 ++ .../stg_exchange__transaction_order_logs.sql | 138 ++++ ...g_exchange__transaction_order_products.sql | 55 ++ .../stg_exchange__transaction_orders.sql | 116 ++++ .../stg_exchange__transaction_wallet_logs.sql | 133 ++++ .../stg_exchange__transaction_wallets.sql | 110 ++++ ...stg_exchange__user_email_verifications.sql | 56 ++ .../staging/exchange/stg_exchange__users.sql | 69 ++ models/staging/shipping/_shipping__docs.md | 21 + .../staging/shipping/_shipping__sources.yml | 293 +++++++++ .../base/base_shipping__bookings.sql} | 25 +- .../base/base_shipping__parent_companies.sql | 48 ++ .../base/base_shipping__sub_companies.sql | 48 ++ ...pping__sub_company_marking_connections.sql | 46 ++ .../shipping/stg_shipping__bookings.sql | 65 ++ ...stg_shipping__parent_and_sub_companies.sql | 123 ++++ .../exchange/seed_exchange__segment_types.csv | 4 + .../seed_shipping__packing_list_types.csv | 8 + .../seed_shipping__parent_company_types.csv | 5 + .../shipping/seed_shipping__service_types.csv | 4 + seeds/shipping/seed_shipping__status.csv | 9 + .../seed_shipping__sub_company_types.csv | 10 + .../snap_exchange__segment_companies.sql | 5 +- tests/generic/test_is_positive.sql | 26 + 317 files changed, 7448 insertions(+), 10782 deletions(-) delete mode 100644 analyses/exchange/blog_ger.sql delete mode 100644 analyses/exchange/company_conversionrate.sql delete mode 100644 analyses/shipping/y2022_companies_ltv.sql delete mode 100644 models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql delete mode 100644 models/crisp/marts/warehouse/dim_crisp__companies.sql delete mode 100644 models/crisp/marts/warehouse/dim_crisp__users.sql delete mode 100644 models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql delete mode 100644 models/crisp/source/src_crisp__get_message_in_conversations.yml delete mode 100644 models/crisp/source/src_crisp__get_people_data.yml delete mode 100644 models/crisp/source/src_crisp__get_people_subscription_status.yml delete mode 100644 models/crisp/source/src_crisp__list_conversations.yml delete mode 100644 models/crisp/source/src_crisp__list_people_profiles.yml delete mode 100644 models/crisp/source/src_crisp__list_website_operator.yml delete mode 100644 models/crisp/staging/stg_crisp__conversations.sql delete mode 100644 models/crisp/staging/stg_crisp__messages.sql delete mode 100644 models/crisp/staging/stg_crisp__operators.sql delete mode 100644 models/crisp/staging/stg_crisp__people_data.sql delete mode 100644 models/crisp/staging/stg_crisp__people_email_subscriptions.sql delete mode 100644 models/exchange/base/addresses/base_exchange__addresses.sql delete mode 100644 models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql delete mode 100644 models/exchange/base/announcements/base_exchange__announcements.sql delete mode 100644 models/exchange/base/bank_logs/base_exchange__bank_logs.sql delete mode 100644 models/exchange/base/banks/base_exchange__banks.sql delete mode 100644 models/exchange/base/booking_logs/base_exchange__booking_logs.sql delete mode 100644 models/exchange/base/bookings/base_exchange__bookings.sql delete mode 100644 models/exchange/base/companies/base_exchange__companies.sql delete mode 100644 models/exchange/base/contacts/base_exchange__contacts.sql delete mode 100644 models/exchange/base/countries/base_exchange__countries.sql delete mode 100644 models/exchange/base/currency_logs/base_exchange__currency_logs.sql delete mode 100644 models/exchange/base/currency_rates/base_exchange__currency_rates.sql delete mode 100644 models/exchange/base/districts/base_exchange__districts.sql delete mode 100644 models/exchange/base/documents/base_exchange__documents.sql delete mode 100644 models/exchange/base/employees/base_exchange__employees.sql delete mode 100644 models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql delete mode 100644 models/exchange/base/files/base_exchange__files.sql delete mode 100644 models/exchange/base/group_transactions/base_exchange__group_transactions.sql delete mode 100644 models/exchange/base/groups/base_exchange__groups.sql delete mode 100644 models/exchange/base/jobs/base_exchange__jobs.sql delete mode 100644 models/exchange/base/migrations/base_exchange__migrations.sql delete mode 100644 models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql delete mode 100644 models/exchange/base/notifications/base_exchange__notifications.sql delete mode 100644 models/exchange/base/password_resets/base_exchange__password_resets.sql delete mode 100644 models/exchange/base/permissions/base_exchange__permissions.sql delete mode 100644 models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql delete mode 100644 models/exchange/base/roles/base_exchange__roles.sql delete mode 100644 models/exchange/base/segment_companies/base_exchange__segment_companies.sql delete mode 100644 models/exchange/base/segment_constants/base_exchange__segment_constants.sql delete mode 100644 models/exchange/base/segments/base_exchange__segments.sql delete mode 100644 models/exchange/base/service_types/base_exchange__service_types.sql delete mode 100644 models/exchange/base/states/base_exchange__states.sql delete mode 100644 models/exchange/base/transaction_details/base_exchange__transaction_details.sql delete mode 100644 models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql delete mode 100644 models/exchange/base/transactions/base_exchange__transactions.sql delete mode 100644 models/exchange/base/users/base_exchange__users.sql delete mode 100644 models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql delete mode 100644 models/exchange/base/wallets/base_exchange__wallets.sql delete mode 100644 models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql delete mode 100644 models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql delete mode 100644 models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql delete mode 100644 models/exchange/intermediate/int_exchange__latest_company_contacts.sql delete mode 100644 models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql delete mode 100644 models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql delete mode 100644 models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql delete mode 100644 models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql delete mode 100644 models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql delete mode 100644 models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml delete mode 100644 models/exchange/marts/warehouse/dim_exchange__companies.sql delete mode 100644 models/exchange/marts/warehouse/dim_exchange__dates.sql delete mode 100644 models/exchange/marts/warehouse/dim_exchange__users.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__booking_logs.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__bookings.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__email_verifications.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql delete mode 100644 models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql delete mode 100644 models/exchange/marts/warehouse/test_exchange__transaction_orders.yml delete mode 100644 models/exchange/marts/warehouse/test_exchange__users.yml delete mode 100644 models/exchange/source/src_activity_log__exchange.yml delete mode 100644 models/exchange/source/src_addresses__exchange.yml delete mode 100644 models/exchange/source/src_announcement_segment__exchange.yml delete mode 100644 models/exchange/source/src_announcements__exchange.yml delete mode 100644 models/exchange/source/src_bank_logs__exchange.yml delete mode 100644 models/exchange/source/src_banks__exchange.yml delete mode 100644 models/exchange/source/src_booking_logs__exchange.yml delete mode 100644 models/exchange/source/src_bookings__exchange.yml delete mode 100644 models/exchange/source/src_companies__exchange.yml delete mode 100644 models/exchange/source/src_contacts__exchange.yml delete mode 100644 models/exchange/source/src_countries__exchange.yml delete mode 100644 models/exchange/source/src_currencies__exchange.yml delete mode 100644 models/exchange/source/src_currency_logs__exchange.yml delete mode 100644 models/exchange/source/src_currency_rate_logs.yml delete mode 100644 models/exchange/source/src_currency_rates__exchange.yml delete mode 100644 models/exchange/source/src_districts__exchange.yml delete mode 100644 models/exchange/source/src_documents__exchange.yml delete mode 100644 models/exchange/source/src_employees__exchange.yml delete mode 100644 models/exchange/source/src_failed_jobs__exchange.yml delete mode 100644 models/exchange/source/src_files__exchange.yml delete mode 100644 models/exchange/source/src_group_transactions__exchange.yml delete mode 100644 models/exchange/source/src_groups__exchange.yml delete mode 100644 models/exchange/source/src_jobs__exchange.yml delete mode 100644 models/exchange/source/src_migrations__exchange.yml delete mode 100644 models/exchange/source/src_model_has_roles__exchange.yml delete mode 100644 models/exchange/source/src_notifications__exchange.yml delete mode 100644 models/exchange/source/src_password_resets__exchange.yml delete mode 100644 models/exchange/source/src_permissions__exchange.yml delete mode 100644 models/exchange/source/src_role_has_permissions__exchange.yml delete mode 100644 models/exchange/source/src_roles__exchange.yml delete mode 100644 models/exchange/source/src_segment_companies__exchange.yml delete mode 100644 models/exchange/source/src_segment_constants__exchange.yml delete mode 100644 models/exchange/source/src_segments__exchange.yml delete mode 100644 models/exchange/source/src_service_types__exchange.yml delete mode 100644 models/exchange/source/src_states__exchange.yml delete mode 100644 models/exchange/source/src_transaction_detail__exchange.yml delete mode 100644 models/exchange/source/src_transaction_logs__exchange.yml delete mode 100644 models/exchange/source/src_transactions__exchange.yml delete mode 100644 models/exchange/source/src_user_email_verifications__exchange.yml delete mode 100644 models/exchange/source/src_users__exchange.yml delete mode 100644 models/exchange/source/src_wallet_logs__exchange.yml delete mode 100644 models/exchange/source/src_wallets__exchange.yml delete mode 100644 models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql delete mode 100644 models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql delete mode 100644 models/exchange/staging/announcements/stg_exchange__announcements.sql delete mode 100644 models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql delete mode 100644 models/exchange/staging/banks/stg_exchange__banks.sql delete mode 100644 models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql delete mode 100644 models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md delete mode 100644 models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml delete mode 100644 models/exchange/staging/bookings/stg_exchange__bookings.sql delete mode 100644 models/exchange/staging/companies/stg_exchange__companies.sql delete mode 100644 models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql delete mode 100644 models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql delete mode 100644 models/exchange/staging/contacts/stg_exchange__contacts.sql delete mode 100644 models/exchange/staging/countries/stg_exchange__countries.sql delete mode 100644 models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql delete mode 100644 models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql delete mode 100644 models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql delete mode 100644 models/exchange/staging/districts/stg_exchange__districts.sql delete mode 100644 models/exchange/staging/documents/stg_exchange__documents.sql delete mode 100644 models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql delete mode 100644 models/exchange/staging/files/stg_exchange__files.sql delete mode 100644 models/exchange/staging/group_costs/stg_exchange__group_costs.sql delete mode 100644 models/exchange/staging/jobs/stg_exchange__jobs.sql delete mode 100644 models/exchange/staging/migrations/stg_exchange__migrations.sql delete mode 100644 models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql delete mode 100644 models/exchange/staging/notifications/stg_exchange__notifications.sql delete mode 100644 models/exchange/staging/password_resets/stg_exchange__password_resets.sql delete mode 100644 models/exchange/staging/permissions/stg_exchange__permissions.sql delete mode 100644 models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql delete mode 100644 models/exchange/staging/roles/stg_exchange__roles.sql delete mode 100644 models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql delete mode 100644 models/exchange/staging/segments/stg_exchange__segments.sql delete mode 100644 models/exchange/staging/service_types/stg_exchange__service_types.sql delete mode 100644 models/exchange/staging/states/stg_exchange__states.sql delete mode 100644 models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql delete mode 100644 models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql delete mode 100644 models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql delete mode 100644 models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql delete mode 100644 models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql delete mode 100644 models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql delete mode 100644 models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql delete mode 100644 models/exchange/staging/transactions/stg_exchange__transactions.sql delete mode 100644 models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql delete mode 100644 models/exchange/staging/users/stg_exchange__users.sql rename models/{crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql => intermediate/int_crisp__messages_generate_dialog_ids.sql} (58%) rename models/{exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql => intermediate/int_exchange__bookings_get_estimate_booking_values.sql} (58%) create mode 100644 models/intermediate/int_exchange__companies_join_segments.sql create mode 100644 models/intermediate/int_exchange__transaction_orders_get_user_ids.sql rename models/{exchange/marts/warehouse/test_exchange__bookings.yml => marts/reporting/_reporting_models.yml} (100%) rename models/{exchange/marts/warehouse/test_exchange__companies.yml => marts/reporting/havent_do_metric_comparisons.sql} (100%) rename analyses/exchange/company_conversion_v2.sql => models/marts/reporting/rep_exchange__company_conversion_funnels.sql (55%) create mode 100644 models/marts/reporting/rep_exchange__company_engagement_behaviors.sql create mode 100644 models/marts/reporting/rep_exchange__company_lists.sql create mode 100644 models/marts/reporting/rep_exchange__daily_orders.sql create mode 100644 models/marts/reporting/rep_exchange__rfm.sql create mode 100644 models/marts/warehouse/_warehouse_models.yml create mode 100644 models/marts/warehouse/dim__dates.sql create mode 100644 models/marts/warehouse/dim_crisp__users.sql create mode 100644 models/marts/warehouse/dim_exchange__companies.sql create mode 100644 models/marts/warehouse/dim_exchange__currencies.sql rename models/{crisp => }/marts/warehouse/fct_crisp__dialogs.sql (57%) create mode 100644 models/marts/warehouse/fct_exchange__bookings.sql rename models/{exchange => }/marts/warehouse/fct_exchange__company_identity_documents.sql (56%) create mode 100644 models/marts/warehouse/fct_exchange__transaction_costs.sql create mode 100644 models/marts/warehouse/fct_exchange__transaction_orders.sql create mode 100644 models/marts/warehouse/fct_exchange__user_email_verifications.sql delete mode 100644 models/shipping/base/addresses/base_shipping__addresses.sql delete mode 100644 models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql delete mode 100644 models/shipping/base/announcements/base_shipping__announcements.sql delete mode 100644 models/shipping/base/companies/base_shipping__companies.sql delete mode 100644 models/shipping/base/company_connections/base_shipping__company_connections.sql delete mode 100644 models/shipping/base/company_employees/base_shipping__company_employees.sql delete mode 100644 models/shipping/base/company_modules/base_shipping__company_modules.sql delete mode 100644 models/shipping/base/connection_segments/base_shipping__connection_segments.sql delete mode 100644 models/shipping/base/contacts/base_shipping__contacts.sql delete mode 100644 models/shipping/base/containers/base_shipping__containers.sql delete mode 100644 models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql delete mode 100644 models/shipping/base/documents/base_shipping__documents.sql delete mode 100644 models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql delete mode 100644 models/shipping/base/files/base_shipping__files.sql delete mode 100644 models/shipping/base/order_roles/base_shipping__order_roles.sql delete mode 100644 models/shipping/base/packages/base_shipping__packages.sql delete mode 100644 models/shipping/base/packing_lists/base_shipping__packing_lists.sql delete mode 100644 models/shipping/base/password_resets/base_shipping__password_resets.sql delete mode 100644 models/shipping/base/remarks/base_shipping__remarks.sql delete mode 100644 models/shipping/base/schedules/base_shipping__schedules.sql delete mode 100644 models/shipping/base/segment_constants/base_shipping__segment_constants.sql delete mode 100644 models/shipping/base/transaction_details/base_shipping__transaction_details.sql delete mode 100644 models/shipping/base/transactions/base_shipping__transactions.sql delete mode 100644 models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql delete mode 100644 models/shipping/base/users/base_shipping__users.sql delete mode 100644 models/shipping/source/src_shipping__activity_log.yml delete mode 100644 models/shipping/source/src_shipping__addresses.yml delete mode 100644 models/shipping/source/src_shipping__announcement_segment.yml delete mode 100644 models/shipping/source/src_shipping__announcements.yml delete mode 100644 models/shipping/source/src_shipping__companies.yml delete mode 100644 models/shipping/source/src_shipping__company_connections.yml delete mode 100644 models/shipping/source/src_shipping__company_employees.yml delete mode 100644 models/shipping/source/src_shipping__company_modules.yml delete mode 100644 models/shipping/source/src_shipping__connection_segments.yml delete mode 100644 models/shipping/source/src_shipping__contacts.yml delete mode 100644 models/shipping/source/src_shipping__container_packing_lists.yml delete mode 100644 models/shipping/source/src_shipping__containers.yml delete mode 100644 models/shipping/source/src_shipping__documents.yml delete mode 100644 models/shipping/source/src_shipping__failed_jobs.yml delete mode 100644 models/shipping/source/src_shipping__files.yml delete mode 100644 models/shipping/source/src_shipping__order_roles.yml delete mode 100644 models/shipping/source/src_shipping__orders.yml delete mode 100644 models/shipping/source/src_shipping__packages.yml delete mode 100644 models/shipping/source/src_shipping__packing_lists.yml delete mode 100644 models/shipping/source/src_shipping__password_resets.yml delete mode 100644 models/shipping/source/src_shipping__remarks.yml delete mode 100644 models/shipping/source/src_shipping__schedules.yml delete mode 100644 models/shipping/source/src_shipping__segment_constants.yml delete mode 100644 models/shipping/source/src_shipping__transaction_details.yml delete mode 100644 models/shipping/source/src_shipping__transactions.yml delete mode 100644 models/shipping/source/src_shipping__transport_arrangements.yml delete mode 100644 models/shipping/source/src_shipping__users.yml create mode 100644 models/staging/crisp/_crisp__docs.md create mode 100644 models/staging/crisp/_crisp__sources.yml create mode 100644 models/staging/crisp/base/base_crisp__user_data.sql create mode 100644 models/staging/crisp/base/base_crisp__user_email_subscriptions.sql rename models/{crisp/staging/stg_crisp__peoples.sql => staging/crisp/base/base_crisp__users.sql} (56%) create mode 100644 models/staging/crisp/stg_crisp__conversations.sql create mode 100644 models/staging/crisp/stg_crisp__messages.sql create mode 100644 models/staging/crisp/stg_crisp__operators.sql create mode 100644 models/staging/crisp/stg_crisp__users.sql create mode 100644 models/staging/exchange/_exchange__docs.md create mode 100644 models/staging/exchange/_exchange__models.yml create mode 100644 models/staging/exchange/_exchange__sources.yml rename models/{exchange/base/activity_logs => staging/exchange/base}/base_exchange__activity_logs.sql (59%) create mode 100644 models/staging/exchange/base/base_exchange__banks.sql create mode 100644 models/staging/exchange/base/base_exchange__booking_arch_logs.sql create mode 100644 models/staging/exchange/base/base_exchange__bookings.sql create mode 100644 models/staging/exchange/base/base_exchange__companies.sql create mode 100644 models/staging/exchange/base/base_exchange__companies_brg_segments.sql create mode 100644 models/staging/exchange/base/base_exchange__company_addresses.sql create mode 100644 models/staging/exchange/base/base_exchange__company_contacts.sql create mode 100644 models/staging/exchange/base/base_exchange__countries.sql rename models/{exchange/base/currencies => staging/exchange/base}/base_exchange__currencies.sql (53%) rename models/{exchange/base/currency_rate_logs => staging/exchange/base}/base_exchange__currency_rate_logs.sql (59%) create mode 100644 models/staging/exchange/base/base_exchange__currency_rates.sql create mode 100644 models/staging/exchange/base/base_exchange__districts.sql create mode 100644 models/staging/exchange/base/base_exchange__documents.sql create mode 100644 models/staging/exchange/base/base_exchange__group_costs.sql create mode 100644 models/staging/exchange/base/base_exchange__segments.sql create mode 100644 models/staging/exchange/base/base_exchange__service_types.sql create mode 100644 models/staging/exchange/base/base_exchange__states.sql create mode 100644 models/staging/exchange/base/base_exchange__transaction_arch_logs.sql create mode 100644 models/staging/exchange/base/base_exchange__transactions.sql rename models/{exchange/base/user_email_verifications => staging/exchange/base}/base_exchange__user_email_verifications.sql (55%) create mode 100644 models/staging/exchange/base/base_exchange__users.sql rename models/{shipping/base/activity_logs/base_shipping__activity_logs.sql => staging/exchange/stg_exchange__activity_logs.sql} (50%) create mode 100644 models/staging/exchange/stg_exchange__banks.sql create mode 100644 models/staging/exchange/stg_exchange__booking_documents.sql create mode 100644 models/staging/exchange/stg_exchange__booking_logs.sql create mode 100644 models/staging/exchange/stg_exchange__bookings.sql create mode 100644 models/staging/exchange/stg_exchange__companies.sql create mode 100644 models/staging/exchange/stg_exchange__companies_brg_segments.sql create mode 100644 models/staging/exchange/stg_exchange__companies_brg_users.sql rename models/{exchange/staging/company_addresses => staging/exchange}/stg_exchange__company_addresses.sql (67%) create mode 100644 models/staging/exchange/stg_exchange__company_contacts.sql create mode 100644 models/staging/exchange/stg_exchange__company_documents.sql create mode 100644 models/staging/exchange/stg_exchange__company_wallets.sql rename models/{exchange/staging/currencies => staging/exchange}/stg_exchange__currencies.sql (51%) rename models/{exchange/staging/currency_rate_logs => staging/exchange}/stg_exchange__currency_rate_logs.sql (71%) create mode 100644 models/staging/exchange/stg_exchange__group_cost_documents.sql create mode 100644 models/staging/exchange/stg_exchange__group_costs.sql create mode 100644 models/staging/exchange/stg_exchange__segments.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_cost_logs.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_costs.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_costs_brg_group_costs.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_documents.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_order_logs.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_order_products.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_orders.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_wallet_logs.sql create mode 100644 models/staging/exchange/stg_exchange__transaction_wallets.sql create mode 100644 models/staging/exchange/stg_exchange__user_email_verifications.sql create mode 100644 models/staging/exchange/stg_exchange__users.sql create mode 100644 models/staging/shipping/_shipping__docs.md create mode 100644 models/staging/shipping/_shipping__sources.yml rename models/{shipping/base/orders/base_shipping__orders.sql => staging/shipping/base/base_shipping__bookings.sql} (67%) create mode 100644 models/staging/shipping/base/base_shipping__parent_companies.sql create mode 100644 models/staging/shipping/base/base_shipping__sub_companies.sql create mode 100644 models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql create mode 100644 models/staging/shipping/stg_shipping__bookings.sql create mode 100644 models/staging/shipping/stg_shipping__parent_and_sub_companies.sql create mode 100644 seeds/exchange/seed_exchange__segment_types.csv create mode 100644 seeds/shipping/seed_shipping__packing_list_types.csv create mode 100644 seeds/shipping/seed_shipping__parent_company_types.csv create mode 100644 seeds/shipping/seed_shipping__service_types.csv create mode 100644 seeds/shipping/seed_shipping__status.csv create mode 100644 seeds/shipping/seed_shipping__sub_company_types.csv create mode 100644 tests/generic/test_is_positive.sql diff --git a/analyses/exchange/blog_ger.sql b/analyses/exchange/blog_ger.sql deleted file mode 100644 index a1e029f..0000000 --- a/analyses/exchange/blog_ger.sql +++ /dev/null @@ -1,3 +0,0 @@ - -SELECT distinct QUOTE_CURRENCY_ID ,QUOTE_CURRENCY_NAME - FROM {{ ref('fct_exchange__bookings') }} diff --git a/analyses/exchange/company_conversionrate.sql b/analyses/exchange/company_conversionrate.sql deleted file mode 100644 index 754845d..0000000 --- a/analyses/exchange/company_conversionrate.sql +++ /dev/null @@ -1,299 +0,0 @@ --- HOW TO USE IT? --- MUST: company_migration_segment filter need to be choose - --- IMPORT -WITH companies AS ( - SELECT * FROM {{ ref('dim_exchange__companies') }} -), - -transaction_orders AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} -), - -company_identity_documents AS ( - SELECT * FROM {{ ref('fct_exchange__company_identity_documents') }} -), - -email_verifications AS ( - SELECT * FROM {{ ref('fct_exchange__email_verifications') }} -), - - --- LOGIC -remove_supplier_company AS ( - SELECT - * - FROM companies - - WHERE business_type = 2 -), - - - -company_register_events AS ( - SELECT - remove_supplier_company.id AS company_id, - 'Register' AS register_event, - - MIN(remove_supplier_company.created_at) AS event_datetime - - FROM remove_supplier_company - - GROUP BY - remove_supplier_company.id -), - -company_email_verification_send_events AS ( - SELECT - company_id, - 'Email Verification Send' AS email_verification_send_event, - verification_email_created_datetime AS event_datetime, - row_number() OVER - (PARTITION BY company_id - ORDER BY verification_email_created_datetime) AS first_row_number - - FROM email_verifications - - QUALIFY - first_row_number = 1 -), - -company_email_verification_complete_events AS ( - SELECT - company_id, - 'Email Verification Complete' AS email_verification_complete_event, - verification_email_updated_datetime AS event_datetime, - row_number() OVER - (PARTITION BY company_id - ORDER BY verification_email_updated_datetime) AS first_row_number - - FROM email_verifications - - WHERE - is_completed = '1' - - QUALIFY - first_row_number = 1 -), - -company_upload_identity_events AS ( - SELECT - company_id, - 'Upload Identity' AS upload_identity_event, - - identity_created_datetime AS event_datetime, - row_number() OVER - (PARTITION BY company_id - ORDER BY identity_created_datetime) AS first_row_number - - FROM company_identity_documents - - QUALIFY - first_row_number = 1 -), - -company_approve_identity_events AS ( - SELECT - company_id, - 'Approve Identity' AS approve_identity_event, - - identity_approved_datetine AS event_datetime, - row_number() OVER - (PARTITION BY company_id - ORDER BY identity_created_datetime) AS first_row_number - - FROM company_identity_documents - - WHERE - identity_status in ('2','3') - - QUALIFY - first_row_number = 1 -), - -company_first_time_order_events AS ( - SELECT - first_time_transaction_orders.company_id, - companies.id, - 'First Time Order' AS first_time_order_event, - first_time_transaction_orders.created_at AS event_datetime - - FROM - (SELECT - *, - row_number() OVER - (PARTITION BY transaction_orders.company_id - ORDER BY transaction_orders.created_at) AS row_rank_index - - FROM transaction_orders - - QUALIFY - row_rank_index = 1 - ) AS first_time_transaction_orders - - INNER JOIN companies - ON(first_time_transaction_orders.company_id = companies.id) -), - - - -company_funnel_events AS ( - SELECT - company_id, - register_event, - NULL AS email_verification_send_event, - NULL AS email_verification_complete_event, - NULL AS upload_identity_event, - NULL AS approve_identity_event, - NULL AS first_time_order_event, - event_datetime - - FROM company_register_events - WHERE register_event IS NOT NULL - - UNION - - SELECT - company_id, - NULL AS register_event, - email_verification_send_event, - NULL AS email_verification_complete_event, - NULL AS upload_identity_event, - NULL AS approve_identity_event, - NULL AS first_time_order_event, - event_datetime - - FROM company_email_verification_send_events - WHERE email_verification_send_event IS NOT NULL - - UNION - - SELECT - company_id, - NULL AS register_event, - NULL AS email_verification_send_event, - email_verification_complete_event, - NULL AS upload_identity_event, - NULL AS approve_identity_event, - NULL AS first_time_order_event, - event_datetime - - FROM company_email_verification_complete_events - WHERE email_verification_complete_event IS NOT NULL - - UNION - - SELECT - company_id, - NULL AS register_event, - NULL AS email_verification_send_event, - NULL AS email_verification_complete_event, - upload_identity_event, - NULL AS approve_identity_event, - NULL AS first_time_order_event, - event_datetime - - FROM company_upload_identity_events - WHERE upload_identity_event IS NOT NULL - - UNION - - SELECT - company_id, - NULL AS register_event, - NULL AS email_verification_send_event, - NULL AS email_verification_complete_event, - NULL AS upload_identity_event, - approve_identity_event, - NULL AS first_time_order_event, - event_datetime - - FROM company_approve_identity_events - WHERE approve_identity_event IS NOT NULL - - UNION - - SELECT - company_id, - NULL AS register_event, - NULL AS email_verification_send_event, - NULL AS email_verification_complete_event, - NULL AS upload_identity_event, - NULL AS approve_identity_event, - first_time_order_event, - event_datetime - - FROM company_first_time_order_events - WHERE first_time_order_event IS NOT NULL - -), - ---FINAL -sem_exchange__company_funnel_events AS ( - SELECT - 'all_company' AS company_migration_segment, - company_funnel_events.event_datetime, - company_funnel_events.company_id, - remove_supplier_company.is_migrated_company, - - count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS register_company, - - count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company, - - count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company, - - count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company, - - count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event, - - count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company - - FROM company_funnel_events - - INNER JOIN remove_supplier_company - ON (company_funnel_events.company_id = remove_supplier_company.id) - - UNION - - SELECT - 'exclude_migrated_company' AS company_migration_segment, - company_funnel_events.event_datetime, - company_funnel_events.company_id, - remove_supplier_company.is_migrated_company, - - count(iff(company_funnel_events.register_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS register_company, - - count(iff(company_funnel_events.email_verification_send_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_send_company, - - count(iff(company_funnel_events.email_verification_complete_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS email_verification_complete_company, - - count(iff(company_funnel_events.upload_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS upload_identity_company, - - count(iff(company_funnel_events.approve_identity_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS approve_identity_event, - - count(iff(company_funnel_events.first_time_order_event IS NOT NULL, company_funnel_events.company_id, NULL)) - OVER (ORDER BY company_funnel_events.event_datetime) AS first_time_order_company - - FROM company_funnel_events - - INNER JOIN remove_supplier_company - ON (company_funnel_events.company_id = remove_supplier_company.id) - - WHERE remove_supplier_company.is_migrated_company = '0' - - -) - - -SELECT count(*) FROM sem_exchange__company_funnel_events \ No newline at end of file diff --git a/analyses/shipping/y2022_companies_ltv.sql b/analyses/shipping/y2022_companies_ltv.sql deleted file mode 100644 index 82f03d4..0000000 --- a/analyses/shipping/y2022_companies_ltv.sql +++ /dev/null @@ -1,84 +0,0 @@ -WITH container_packing_lists AS ( - SELECT * FROM {{ ref('base_shipping__container_packing_lists') }} -), - -containers AS ( - SELECT * FROM {{ ref('base_shipping__containers') }} -), - -packing_lists AS ( - SELECT * FROM {{ ref('base_shipping__packing_lists') }} -), - -packages AS ( - SELECT * FROM {{ ref('base_shipping__packages') }} -), - -company_modules AS ( - SELECT * FROM {{ ref('base_shipping__company_modules') }} -), - -company_connections AS ( - SELECT * FROM {{ ref('base_shipping__company_connections') }} -), - -orders AS ( - SELECT * FROM {{ ref('base_shipping__orders') }} -) --- LOGIC -company_module_join_companies AS ( - SELECT - company_modules.id AS company_id, - company_modules.name AS company_name, - company_connections.invitee_reference AS marking - - FROM - company_modules - LEFT JOIN company_connections - ON (company_modules.id = company_connections.invitee_id) -), - -final AS ( - SELECT - container_packing_lists.*, - containers.*, - packing_lists.*, - packages.*, - packages.created_at AS packages_created_at, - ((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm - - FROM container_packing_lists - - JOIN containers - ON (container_packing_lists.container_id = containers.id) - JOIN packing_lists - ON (container_packing_lists.packing_list_id = packing_lists.id) - JOIN packages - ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null) - - WHERE packing_lists.deleted_at is null AND packages_created_at >= '2022-01-01') --- ), - --- final_group_by_company_module AS ( --- SELECT --- final.owner_id, --- SUM(final.cbm) AS life_time_cbm - --- FROM final - --- GROUP BY --- owner_id --- ) - - - -SELECT * FROM final - --- SELECT --- containers.*, packing_lists.id as packing_list_id, SUM((packages.width/100) * (packages.height/100) * (packages.length/100) * packages.quantity) as cbm --- FROM --- container_packing_lists as packed_containers --- JOIN containers ON packed_containers.container_id = containers.id --- JOIN packing_lists ON packed_containers.packing_list_id = packing_lists.id --- JOIN packages ON (packages.packing_list_id = packing_lists.id and packages.deleted_at is null) --- WHERE packing_lists.deleted_at is null GROUP BY packing_lists.id; \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 2a411e4..724a3a4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -32,91 +32,56 @@ clean-targets: # directories to be removed by `dbt clean` # using the `{{ config(...) }}` macro. models: cief_dbt_cloud: - # Applies to all files under models/example/ - exchange: - marts: - reporting: - description: data modeling use in Preset (Superset) + staging: + +tags: + - staging + + autocount: +materialized: view - +schema: exchange__reporting - - warehouse: - description: data modeling contain all fct & dim - +materialized: table - +schema: exchange__warehouse - - intermediate: - description: data modeling use for pipeline transformation, contain all int + +schema: autocount__staging + + crisp: +materialized: view - +schema: exchange__intermediate + +schema: crisp__staging - staging: - description: light_transformation & will be occur in staging, contain all stg - +materialized: view - +schema: exchange__staging - - base: - description: light_transformation such as rename and remove test or wrong input data - +materialized: view - +schema: exchange__base - - - shipping: - marts: - reporting: - description: data modeling use in Preset (Superset) + exchange: +materialized: view - +schema: shipping__reporting - - warehouse: - description: data modeling contain all fct & dim + +schema: exchange__staging + + base: + +materialized: view + +schema: exchange__base + + shipping: + staging: +materialized: view - +schema: shipping__warehouse + +schema: shipping__staging - intermediate: - description: data modeling use for pipeline transformation, contain all int - +materialized: view - +schema: shipping__intermediate + base: + +materialized: view + +schema: shipping__base + + intermediate: + +materialized: ephemeral + +schema: intermediate + +tags: + - intermediate - staging: - description: light_transformation & will be occur in staging, contain all stg - +materialized: view - +schema: shipping__staging - - base: - description: light_transformation such as rename and remove test or wrong input data - +materialized: view - +schema: shipping__base - - - crisp: - marts: - reporting: - description: data modeling use in Preset (Superset) - +materialized: view - +schema: crisp__reporting - - warehouse: - description: data modeling contain all fct & dim - +materialized: view - +schema: crisp__warehouse - - intermediate: - description: data modeling use for pipeline transformation, contain all int - +materialized: view - +schema: crisp__intermediate - - staging: - description: light_transformation & will be occur in staging, contain all stg - +materialized: view - +schema: crisp__staging - - base: - description: light_transformation such as rename and remove test or wrong input data - +materialized: view - +schema: crisp__base + marts: + +tags: + - marts + warehouse: + +materialized: table + +schema: warehouse + +tags: + - warehouse + reporting: + +materialized: table + +schema: reporting + +tags: + - reporting seeds: cief_dbt_cloud: exchange: @@ -126,6 +91,7 @@ seeds: crisp: +schema: crisp__seeds + snapshots: cief_dbt_cloud: exchange: diff --git a/models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql b/models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql deleted file mode 100644 index 31d35d3..0000000 --- a/models/crisp/intermediate/int_crisp__users_and_companies_and_company_marking.sql +++ /dev/null @@ -1,68 +0,0 @@ ---IMPORT -WITH peoples AS ( - SELECT * FROM {{ ref('stg_crisp__peoples') }} -), - -people_data AS ( - SELECT * FROM {{ ref('stg_crisp__people_data') }} -), - ---LOGIC -people_get_marking_id AS ( - SELECT - peoples.id, - peoples.name, - peoples.phone, - peoples.email, - peoples.website, - peoples.score, - peoples.segment_list, - peoples.role, - peoples.seniority, - peoples.title, - peoples.gender, - peoples.timezone, - peoples.geo_country, - peoples.geo_city, - peoples.geo_region_code, - peoples.address, - peoples.locale_list, - peoples.geo_latitude, - peoples.geo_longitude, - peoples.last_activated_datetime, - peoples.people_created_datetime, - peoples.people_updated_datetime, - peoples.company_name, - peoples.company_legal_name, - peoples.company_description, - peoples.company_phone, - peoples.company_email_list, - peoples.company_website, - peoples.company_domain, - peoples.company_tag_list, - peoples.company_number_of_employees, - peoples.company_market_cap, - peoples.company_country, - peoples.company_city, - peoples.company_region, - peoples.company_latitude, - peoples.company_logitude, - peoples.company_timezone, - people_data.company_name AS marking_company_name, - people_data.whatsapp_number, - people_data.whatsapp_business_number, - people_data.facebook_page_name, - people_data.facebook_page_id, - people_data.exchange_marking_id, - people_data.izyim_marking_id, - people_data.lite_marking_id - - FROM peoples - - LEFT JOIN people_data - ON (peoples.id = people_data.people_id) -) - - ---FINAL -select * from people_get_marking_id \ No newline at end of file diff --git a/models/crisp/marts/warehouse/dim_crisp__companies.sql b/models/crisp/marts/warehouse/dim_crisp__companies.sql deleted file mode 100644 index 98751bc..0000000 --- a/models/crisp/marts/warehouse/dim_crisp__companies.sql +++ /dev/null @@ -1,86 +0,0 @@ ---IMPORT -WITH users_and_companies AS ( - SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }} -), - ---LOGIC -split_company_data AS ( - SELECT - exchange_marking_id, - izyim_marking_id, - lite_marking_id, - marking_company_name, - whatsapp_number, - whatsapp_business_number, - facebook_page_name, - facebook_page_id, - company_name, - company_legal_name, - company_description, - company_phone, - company_email_list, - company_website, - company_domain, - company_tag_list, - company_number_of_employees, - company_market_cap, - company_country, - company_city, - company_region, - company_latitude, - company_logitude, - company_timezone - - FROM users_and_companies -), - -companies_remove_duplicate AS ( - SELECT - DISTINCT * - - FROM split_company_data -), - ---FINAL -final_dim_crisp__companies AS ( - SELECT - exchange_marking_id, - izyim_marking_id, - lite_marking_id, - IFF(COUNT(DISTINCT(marking_company_name)) > 1, LISTAGG(marking_company_name, ','), MAX(marking_company_name) ) AS marking_company_name, - IFF(COUNT(DISTINCT(whatsapp_number)) > 1, LISTAGG(whatsapp_number, ','), MAX(whatsapp_number) ) AS whatsapp_number, - IFF(COUNT(DISTINCT(whatsapp_business_number)) > 1, LISTAGG(whatsapp_business_number, ','), MAX(whatsapp_business_number) ) AS whatsapp_business_number, - IFF(COUNT(DISTINCT(facebook_page_name)) > 1, LISTAGG(facebook_page_name, ','), MAX(facebook_page_name) ) AS facebook_page_name, - IFF(COUNT(DISTINCT(facebook_page_id)) > 1, LISTAGG(facebook_page_id, ','), MAX(facebook_page_id) ) AS facebook_page_id, - IFF(COUNT(DISTINCT(company_name)) > 1, LISTAGG(company_name, ','), MAX(company_name) ) AS company_name, - IFF(COUNT(DISTINCT(company_legal_name)) > 1, LISTAGG(company_legal_name, ','), MAX(company_legal_name) ) AS company_legal_name, - IFF(COUNT(DISTINCT(company_description)) > 1, LISTAGG(company_description, ','), MAX(company_description) ) AS company_description, - IFF(COUNT(DISTINCT(company_phone)) > 1, LISTAGG(company_phone, ','), MAX(company_phone) ) AS company_phone, - IFF(COUNT(DISTINCT(company_email_list)) > 1, LISTAGG(company_email_list, ','), MAX(company_email_list) ) AS company_email_list, - IFF(COUNT(DISTINCT(company_website)) > 1, LISTAGG(company_website, ','), MAX(company_website) ) AS company_website, - IFF(COUNT(DISTINCT(company_domain)) > 1, LISTAGG(company_domain, ','), MAX(company_domain) ) AS company_domain, - IFF(COUNT(DISTINCT(company_tag_list)) > 1, LISTAGG(company_tag_list, ','), MAX(company_tag_list) ) AS company_tag_list, - IFF(COUNT(DISTINCT(company_number_of_employees)) > 1, LISTAGG(company_number_of_employees, ','), MAX(company_number_of_employees) ) AS company_number_of_employees, - IFF(COUNT(DISTINCT(company_market_cap)) > 1, LISTAGG(company_market_cap, ','), MAX(company_market_cap) ) AS company_market_cap, - IFF(COUNT(DISTINCT(company_country)) > 1, LISTAGG(company_country, ','), MAX(company_country) ) AS company_country, - IFF(COUNT(DISTINCT(company_city)) > 1, LISTAGG(company_city, ','), MAX(company_city) ) AS company_city, - IFF(COUNT(DISTINCT(company_region)) > 1, LISTAGG(company_region, ','), MAX(company_region) ) AS company_region, - IFF(COUNT(DISTINCT(company_latitude)) > 1, LISTAGG(company_latitude, ','), MAX(company_latitude) ) AS company_latitude, - IFF(COUNT(DISTINCT(company_logitude)) > 1, LISTAGG(company_logitude, ','), MAX(company_logitude) ) AS company_logitude, - IFF(COUNT(DISTINCT(company_timezone)) > 1, LISTAGG(company_timezone, ','), MAX(company_timezone) ) AS company_timezone - - FROM companies_remove_duplicate - - WHERE - (IFF(exchange_marking_id IS NULL, 1, 0) + - IFF(izyim_marking_id IS NULL, 1, 0) + - IFF(lite_marking_id IS NULL, 1, 0)) < 3 - - GROUP BY - exchange_marking_id, - izyim_marking_id, - lite_marking_id - -) --- SELECT EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID, count(*) FROM final_dim_crisp__companies GROUP BY EXCHANGE_MARKING_ID, IZYIM_MARKING_ID, LITE_MARKING_ID ORDER BY count(*) desc -SELECT * FROM final_dim_crisp__companies \ No newline at end of file diff --git a/models/crisp/marts/warehouse/dim_crisp__users.sql b/models/crisp/marts/warehouse/dim_crisp__users.sql deleted file mode 100644 index 75878e2..0000000 --- a/models/crisp/marts/warehouse/dim_crisp__users.sql +++ /dev/null @@ -1,48 +0,0 @@ ---IMPORT -WITH users_and_companies AS ( - SELECT * FROM {{ ref('int_crisp__users_and_companies_and_company_marking') }} -), - ---LOGIC -split_user_data AS ( - SELECT - id, - name, - phone, - email, - website, - score, - segment_list, - role, - seniority, - title, - gender, - timezone, - geo_country, - geo_city, - geo_region_code, - address, - locale_list, - geo_latitude, - geo_longitude, - last_activated_datetime, - people_created_datetime, - people_updated_datetime, - exchange_marking_id AS company_exchange_marking_id, - izyim_marking_id AS company_izyim_marking_id, - lite_marking_id AS company_lite_marking_id - - FROM users_and_companies -), - ---FINAL -final_dim_crisp__users AS ( - SELECT - * - FROM split_user_data -) -SELECT * FROM final_dim_crisp__users --- SELECT company_lite_marking_id, count(distinct(company_exchange_marking_id)) AS t1, count(distinct(company_izyim_marking_id)) AS t2 --- FROM final_dim_crisp__users GROUP BY company_lite_marking_id HAVING t1 >1 or t2 > 1 --- SELECT company_IZYIM_MARKING_ID, company_exchange_marking_id FROM final_dim_crisp__users --- WHERE company_IZYIM_MARKING_ID IN ('1324FEN','3791YBE', '1291NSC', '2698LXA', '2431HEB', '2634PLT', '8299LIE', '1882KAI', '1889LBH', '913DLT', '1650GTB', '5903CSB', '6631GEB', '1656HBP', '1905JYW', '8265LSB', '7260UUT', '5989HSY') ORDER BY company_IZYIM_MARKING_ID \ No newline at end of file diff --git a/models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql b/models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql deleted file mode 100644 index 9685016..0000000 --- a/models/crisp/marts/warehouse/fct_crisp__user_email_subscriptions.sql +++ /dev/null @@ -1,15 +0,0 @@ ---IMPORT -WITH snap_user_email_subscriptions AS ( - SELECT * FROM {{ ref('stg_crisp__people_email_subscriptions') }} -), - - ---FINAL -final_fct_crisp__user_email_subscriptions AS ( - SELECT - * - - FROM snap_user_email_subscriptions -) - -SELECT * FROM snap_user_email_subscriptions \ No newline at end of file diff --git a/models/crisp/source/src_crisp__get_message_in_conversations.yml b/models/crisp/source/src_crisp__get_message_in_conversations.yml deleted file mode 100644 index 7a5f001..0000000 --- a/models/crisp/source/src_crisp__get_message_in_conversations.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 - -sources: - - name: src_crisp_api - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: CRISP_API - - tables: - - name: get_message_in_conversations - description: Raw `get_people_subscription_status` table from Snowflake - columns: - - name: fingerprint - description: Primary key for 'get_message_in_conversations' - tests: - - unique - - not_null - - name: delivered - description: the platform which after the message is output from origin, and delivered to the user/operator. When an operator send a message but the user is not online anymore, we send it via email only after 2min during these 2min, delivered is empty - loaded_at_field: api_called_at - freshness: - warn_after: {count: 1, period: day} - error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__get_people_data.yml b/models/crisp/source/src_crisp__get_people_data.yml deleted file mode 100644 index b691274..0000000 --- a/models/crisp/source/src_crisp__get_people_data.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_crisp_api - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: CRISP_API - - tables: - - name: get_people_data - description: Raw `get_people_data` table from Snowflake - columns: - - name: people_id - description: Primary key for 'get_people_data' - tests: - - unique - - not_null - loaded_at_field: api_called_at - freshness: - warn_after: {count: 1, period: day} - error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__get_people_subscription_status.yml b/models/crisp/source/src_crisp__get_people_subscription_status.yml deleted file mode 100644 index 0ea03ca..0000000 --- a/models/crisp/source/src_crisp__get_people_subscription_status.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_crisp_api - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: CRISP_API - - tables: - - name: get_people_subscription_status - description: Raw `get_people_subscription_status` table from Snowflake - columns: - - name: people_id - description: Primary key for 'get_people_subscription_status' - tests: - - unique - - not_null - loaded_at_field: api_called_at - freshness: - warn_after: {count: 1, period: day} - error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__list_conversations.yml b/models/crisp/source/src_crisp__list_conversations.yml deleted file mode 100644 index 3af0c3c..0000000 --- a/models/crisp/source/src_crisp__list_conversations.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_crisp_api - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: CRISP_API - - tables: - - name: list_conversations - description: Raw `get_people_subscription_status` table from Snowflake - columns: - - name: SESSION_ID - description: Primary key for 'list_conversations' - tests: - - unique - - not_null - loaded_at_field: api_called_at - freshness: - warn_after: {count: 1, period: day} - error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__list_people_profiles.yml b/models/crisp/source/src_crisp__list_people_profiles.yml deleted file mode 100644 index a7e7f69..0000000 --- a/models/crisp/source/src_crisp__list_people_profiles.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_crisp_api - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: CRISP_API - - tables: - - name: list_people_profiles - description: Raw `list_people_profiles` table from Snowflake - columns: - - name: people_id - description: Primary key for 'list_people_profiles' - tests: - - unique - - not_null - loaded_at_field: api_called_at - freshness: - warn_after: {count: 1, period: day} - error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/source/src_crisp__list_website_operator.yml b/models/crisp/source/src_crisp__list_website_operator.yml deleted file mode 100644 index b185736..0000000 --- a/models/crisp/source/src_crisp__list_website_operator.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_crisp_api - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: CRISP_API - - tables: - - name: list_website_operator - description: Raw `list_website_operator` table from Snowflake - columns: - - name: details__user_id - description: Primary key for 'list_website_operator' - tests: - - unique - - not_null - loaded_at_field: api_called_at - freshness: - warn_after: {count: 1, period: day} - error_after: {count: 2, period: hour} \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__conversations.sql b/models/crisp/staging/stg_crisp__conversations.sql deleted file mode 100644 index 39b9c6c..0000000 --- a/models/crisp/staging/stg_crisp__conversations.sql +++ /dev/null @@ -1,22 +0,0 @@ --- IMPORT -WITH list_conversations AS ( - SELECT * FROM {{ source('src_crisp_api', 'list_conversations') }} -), - ---LOGIC - ---FINAL -final_stg_crisp__conversations AS ( - SELECT - session_id AS id, - people_id AS user_id, - state, - status, - IFF(is_blocked = TRUE, 1, 0) AS is_blocked, - assigned__user_id AS operator_id, - convert_timezone('Asia/Kuala_Lumpur', created_at) AS conversation_created_datetime, - convert_timezone('Asia/Kuala_Lumpur', updated_at) AS conversation_updated_datetime - - FROM list_conversations -) -SELECT * FROM final_stg_crisp__conversations \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__messages.sql b/models/crisp/staging/stg_crisp__messages.sql deleted file mode 100644 index 6fa4992..0000000 --- a/models/crisp/staging/stg_crisp__messages.sql +++ /dev/null @@ -1,32 +0,0 @@ --- IMPORT -WITH get_message_in_conversations AS ( - SELECT * FROM {{ source('src_crisp_api', 'get_message_in_conversations') }} -), - --- LOGIC - --- FINAL -final_stg_crisp__messages AS ( -SELECT - fingerprint AS id, - session_id AS conversation_id, - user__user_id AS user_id, - user__nickname AS user_name, - from_user AS message_from_side, - origin AS message_generate_platform, - delivered AS message_deliverd_platform, - read AS message_read_platform, - type AS message_format, - content__namespace AS event_type, - content__text AS event_message, - content AS message, - edited AS is_edited_message, - content__type AS attachment_type, - content__name AS attachment_name, - content__url AS attachment_url, - convert_timezone('Asia/Kuala_Lumpur', timestamp) AS message_created_datetime - -FROM get_message_in_conversations ORDER BY session_id, message_created_datetime -) - -SELECT * FROM final_stg_crisp__messages \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__operators.sql b/models/crisp/staging/stg_crisp__operators.sql deleted file mode 100644 index cf9fae8..0000000 --- a/models/crisp/staging/stg_crisp__operators.sql +++ /dev/null @@ -1,20 +0,0 @@ --- IMPORT -WITH list_website_operator AS ( - SELECT * FROM {{ source('src_crisp_api', 'list_website_operator') }} -), - ---LOGIC - ---FINAL -final_stg_crisp__operators AS ( - SELECT - details__user_id AS id, - details__title AS title, - details__first_name AS first_name, - details__last_name AS last, - details__email AS email - - FROM list_website_operator -) - -SELECT * FROM final_stg_crisp__operators \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__people_data.sql b/models/crisp/staging/stg_crisp__people_data.sql deleted file mode 100644 index 8454886..0000000 --- a/models/crisp/staging/stg_crisp__people_data.sql +++ /dev/null @@ -1,23 +0,0 @@ --- IMPORT -WITH get_people_data AS ( - SELECT * FROM {{ source('src_crisp_api', 'get_people_data') }} -), - ---LOGIC - ---FINAL -final_stg_crisp__get_people_data AS ( - SELECT - people_id, - data__company_name AS company_name, - data__wa_number AS whatsapp_number, - data__whatsapp_business_number AS whatsapp_business_number, - data__fb_page AS facebook_page_name, - data__fb_page_id AS facebook_page_id, - data__exchange_marking AS exchange_marking_id, - data__izyim_marking AS izyim_marking_id, - data__lite_marking AS lite_marking_id - - FROM get_people_data -) -SELECT * FROM final_stg_crisp__get_people_data \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__people_email_subscriptions.sql b/models/crisp/staging/stg_crisp__people_email_subscriptions.sql deleted file mode 100644 index 9f8135f..0000000 --- a/models/crisp/staging/stg_crisp__people_email_subscriptions.sql +++ /dev/null @@ -1,20 +0,0 @@ --- IMPORT -WITH get_people_subscription_status AS ( - SELECT * FROM {{ ref('snap_crisp__get_people_subscription_status') }} -), - ---LOGIC - ---FINAL -final_stg_crisp__people_email_subscriptions AS ( - SELECT - dbt_scd_id, - people_id, - IFF(email = TRUE, 1, 0) AS is_subscribed, - DBT_VALID_FROM AS dbt_valid_from_datetime, - DBT_VALID_TO AS dbt_valid_to_datetime - - FROM get_people_subscription_status -) - -SELECT * FROM final_stg_crisp__people_email_subscriptions \ No newline at end of file diff --git a/models/exchange/base/addresses/base_exchange__addresses.sql b/models/exchange/base/addresses/base_exchange__addresses.sql deleted file mode 100644 index 19c0b1a..0000000 --- a/models/exchange/base/addresses/base_exchange__addresses.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH addresses AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'addresses') }} -), - --- LOGIC -addresses_rename AS ( - - SELECT - addresses.id, - addresses.company_id, - addresses.country_id, - addresses.state_id, - addresses.district_id, - addresses.postcode, - addresses.street_one AS address_line_one, - addresses.street_two AS address_line_two, - addresses.billing AS is_billing_address, - addresses.deleted_at, - addresses.created_at, - addresses.updated_At, - current_timestamp() as _dbt_ran_at - - FROM addresses -), - --- FINAL -final__base_exchange__addresses AS ( - - SELECT - id, - company_id, - country_id, - state_id, - district_id, - postcode, - address_line_one, - address_line_two, - is_billing_address, - deleted_at, - created_at, - updated_At, - _dbt_ran_at - - FROM addresses_rename -) - -SELECT * FROM final__base_exchange__addresses \ No newline at end of file diff --git a/models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql b/models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql deleted file mode 100644 index 4418be9..0000000 --- a/models/exchange/base/announcement_segments/base_exchange__announcement_segments.sql +++ /dev/null @@ -1,34 +0,0 @@ --- IMPORTS -WITH announcement_segments AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'announcement_segment') }} -), - - --- LOGIC -announcement_segments_rename AS ( - - SELECT - announcement_segments.announcement_id, - announcement_segments.segment_id, - announcement_segments.created_at, - announcement_segments.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM announcement_segments -), - - --- FINAL -final__base_exchange__announcement_segments AS ( - - SELECT - announcement_id, - segment_id, - created_at, - updated_at, - _dbt_ran_at - - FROM announcement_segments_rename -) - -SELECT * FROM final__base_exchange__announcement_segments \ No newline at end of file diff --git a/models/exchange/base/announcements/base_exchange__announcements.sql b/models/exchange/base/announcements/base_exchange__announcements.sql deleted file mode 100644 index db52a8b..0000000 --- a/models/exchange/base/announcements/base_exchange__announcements.sql +++ /dev/null @@ -1,50 +0,0 @@ --- IMPORTS -WITH announcements AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'announcements') }} -), - - --- LOGIC -announcements_rename AS ( - - SELECT - id, - title, - description, - starting_on AS started_at, - ending_on AS ended_at, - is_all_day, - duration, - is_recurring, - recurrence_pattern, - deleted_at, - created_at, - updated_at, - current_timestamp() as _dbt_ran_at - - FROM announcements -), - - --- FINAL -final__base_exchange__announcements AS ( - - SELECT - id, - title, - description, - started_at, - ended_at, - is_all_day, - duration, - is_recurring, - recurrence_pattern, - deleted_at, - created_at, - updated_at, - current_timestamp() as _dbt_ran_at - - FROM announcements_rename -) - -SELECT * FROM final__base_exchange__announcements \ No newline at end of file diff --git a/models/exchange/base/bank_logs/base_exchange__bank_logs.sql b/models/exchange/base/bank_logs/base_exchange__bank_logs.sql deleted file mode 100644 index ae9755b..0000000 --- a/models/exchange/base/bank_logs/base_exchange__bank_logs.sql +++ /dev/null @@ -1,61 +0,0 @@ --- IMPORTS -WITH bank_logs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'bank_logs') }} -), - - --- LOGIC -bank_logs_rename AS ( - - SELECT - bank_logs.id, - bank_logs.bank_id, - bank_logs.company_id, - bank_logs.reference, - bank_logs.bank_name, - bank_logs.holder_name, - bank_logs.account_no, - bank_logs.bank_branch, - bank_logs.swift AS swift_code, - bank_logs.snap AS cnaps_code, - bank_logs.type AS bank_type, - bank_logs.default AS is_default, - bank_logs.status, - bank_logs.country_id, - bank_logs.deleted_at, - bank_logs.created_at, - bank_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM bank_logs - -), - - --- FINAL -final__base_exchange__bank_logs AS ( - - SELECT - id, - bank_id, - company_id, - reference, - bank_name, - holder_name, - account_no, - bank_branch, - swift_code, - cnaps_code, - bank_type, - is_default, - status, - country_id, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM bank_logs_rename -) - -SELECT * FROM final__base_exchange__bank_logs \ No newline at end of file diff --git a/models/exchange/base/banks/base_exchange__banks.sql b/models/exchange/base/banks/base_exchange__banks.sql deleted file mode 100644 index 0b5ae1f..0000000 --- a/models/exchange/base/banks/base_exchange__banks.sql +++ /dev/null @@ -1,58 +0,0 @@ --- IMPORTS -WITH banks AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'banks') }} -), - - --- LOGIC -banks_rename AS ( - - SELECT - banks.id, - banks.company_id, - banks.reference, - banks.bank_name, - banks.holder_name, - banks.account_no, - banks.bank_branch, - banks.swift AS swift_code, - banks.snap AS cnaps_code, - banks.type AS bank_type, - banks.default AS is_default, - banks.status, - banks.country_id, - banks.deleted_at, - banks.created_at, - banks.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM banks -), - - --- FINAL -final__base_exchange__banks AS ( - - SELECT - id, - company_id, - reference, - bank_name, - holder_name, - account_no, - bank_branch, - swift_code, - cnaps_code, - bank_type, - is_default, - status, - country_id, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM banks_rename -) - -SELECT * FROM final__base_exchange__banks \ No newline at end of file diff --git a/models/exchange/base/booking_logs/base_exchange__booking_logs.sql b/models/exchange/base/booking_logs/base_exchange__booking_logs.sql deleted file mode 100644 index e5800b1..0000000 --- a/models/exchange/base/booking_logs/base_exchange__booking_logs.sql +++ /dev/null @@ -1,55 +0,0 @@ --- IMPORTS -WITH booking_logs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'booking_logs') }} -), - - --- LOGIC -booking_logs_rename AS ( - - SELECT - booking_logs.id, - booking_logs.booking_id, - booking_logs.company_id, - booking_logs.marking AS marking_id, - booking_logs.service_id AS service_type, - booking_logs.bank_id, - booking_logs.fix_amount AS fix_value, - booking_logs.fix_currency_id, - booking_logs.convertible_currency_id AS quote_currency_id, - booking_logs.conversion_currency_id AS base_currency_id, - booking_logs.status, - booking_logs.deleted_at, - booking_logs.created_at, - booking_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM booking_logs - -), - --- FINAL -final_base_exchange__booking_logs AS ( - - SELECT - id, - booking_id, - company_id, - marking_id, - service_type, - bank_id, - fix_value, - fix_currency_id, - quote_currency_id, - base_currency_id, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM booking_logs_rename - -) - -SELECT * FROM final_base_exchange__booking_logs \ No newline at end of file diff --git a/models/exchange/base/bookings/base_exchange__bookings.sql b/models/exchange/base/bookings/base_exchange__bookings.sql deleted file mode 100644 index f5a3072..0000000 --- a/models/exchange/base/bookings/base_exchange__bookings.sql +++ /dev/null @@ -1,53 +0,0 @@ --- IMPORTS -WITH bookings AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'bookings') }} -), - - --- LOGIC -bookings_rename AS ( - - SELECT - bookings.id, - bookings.company_id, - bookings.marking AS marking_id, - bookings.service_id AS service_type, - bookings.bank_id, - bookings.fix_amount AS fix_value, - bookings.fix_currency_id, - bookings.convertible_currency_id AS quote_currency_id, - bookings.conversion_currency_id AS base_currency_id, - bookings.status, - bookings.deleted_at, - bookings.created_at, - bookings.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM bookings - -), - --- FINAL -final_base_exchange__bookings AS ( - - SELECT - id, - company_id, - marking_id, - service_type, - bank_id, - fix_value, - fix_currency_id, - quote_currency_id, - base_currency_id, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM bookings_rename - -) - -SELECT * FROM final_base_exchange__bookings \ No newline at end of file diff --git a/models/exchange/base/companies/base_exchange__companies.sql b/models/exchange/base/companies/base_exchange__companies.sql deleted file mode 100644 index c6566fc..0000000 --- a/models/exchange/base/companies/base_exchange__companies.sql +++ /dev/null @@ -1,47 +0,0 @@ --- IMPORTS -WITH companies AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'companies') }} -), - - --- LOGIC -companies_rename AS ( - - SELECT - companies.id, - companies.name, - companies.reference AS marking_id, - companies.debtor AS account_software_id, - companies.type AS company_type, - companies.business_type, - companies.status, - companies.deleted_at, - companies.created_at, - companies.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM companies -), - - --- FINAL -final__base_exchange__companies AS ( - - SELECT - id, - name, - marking_id, - account_software_id, - company_type, - business_type, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM companies_rename - -) - -SELECT * FROM final__base_exchange__companies \ No newline at end of file diff --git a/models/exchange/base/contacts/base_exchange__contacts.sql b/models/exchange/base/contacts/base_exchange__contacts.sql deleted file mode 100644 index 78421fe..0000000 --- a/models/exchange/base/contacts/base_exchange__contacts.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH contacts AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'contacts') }} -), - - --- LOGIC -companies_rename AS ( - - SELECT - contacts.id, - contacts.company_id, - contacts.country_id, - contacts.reference, - contacts.phone, - contacts.email, - contacts.wechat_id, - contacts.deleted_at, - contacts.created_at, - contacts.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM contacts - -), - - --- FINAL -final__base_exchange__contacts AS ( - - SELECT - id, - company_id, - country_id, - reference, - phone, - email, - wechat_id, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM companies_rename - -) - -SELECT * FROM final__base_exchange__contacts \ No newline at end of file diff --git a/models/exchange/base/countries/base_exchange__countries.sql b/models/exchange/base/countries/base_exchange__countries.sql deleted file mode 100644 index c60a99b..0000000 --- a/models/exchange/base/countries/base_exchange__countries.sql +++ /dev/null @@ -1,41 +0,0 @@ --- IMPORTS -WITH countries AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'countries') }} -), - --- LOGIC -countries_rename AS ( - - SELECT - countries.id, - countries.name, - countries.short_code, - countries.phone_code, - countries.deleted_at, - countries.created_at, - countries.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM countries - -), - - --- FINAL -final__base_exchange__countries AS ( - - SELECT - id, - name, - short_code, - phone_code, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM countries_rename - -) - -SELECT * FROM final__base_exchange__countries \ No newline at end of file diff --git a/models/exchange/base/currency_logs/base_exchange__currency_logs.sql b/models/exchange/base/currency_logs/base_exchange__currency_logs.sql deleted file mode 100644 index 9a549a2..0000000 --- a/models/exchange/base/currency_logs/base_exchange__currency_logs.sql +++ /dev/null @@ -1,39 +0,0 @@ --- IMPORTS -WITH currency_logs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'currency_logs') }} -), - - --- LOGIC -currency_logs_rename AS ( - - SELECT - currency_logs.id, - currency_logs.currency_id, - currency_logs.created_by, - currency_logs.deleted_at, - currency_logs.created_at, - currency_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM currency_logs -), - - --- FINAL -final__base_exchange__currency_logs AS ( - - SELECT - id, - currency_id, - created_by, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM currency_logs_rename - -) - -SELECT * FROM final__base_exchange__currency_logs \ No newline at end of file diff --git a/models/exchange/base/currency_rates/base_exchange__currency_rates.sql b/models/exchange/base/currency_rates/base_exchange__currency_rates.sql deleted file mode 100644 index 488d104..0000000 --- a/models/exchange/base/currency_rates/base_exchange__currency_rates.sql +++ /dev/null @@ -1,43 +0,0 @@ --- IMPORTS -WITH currency_rates AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'currency_rates') }} -), - - --- LOGIC -currency_rates_rename AS ( - - SELECT - currency_rates.id, - currency_rates.currency_id AS quote_currency_id, - currency_rates.selling AS base_to_quote_currency_exchange_rate, - currency_rates.payment_method_type AS payment_method, - currency_rates.service_id AS service_type, - currency_rates.deleted_at, - currency_rates.created_at, - currency_rates.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM currency_rates -), - - --- FINAL -final__base_exchange__currency_rates AS ( - - SELECT - id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - payment_method, - service_type, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM currency_rates_rename - -) - -SELECT * FROM final__base_exchange__currency_rates \ No newline at end of file diff --git a/models/exchange/base/districts/base_exchange__districts.sql b/models/exchange/base/districts/base_exchange__districts.sql deleted file mode 100644 index c193cb6..0000000 --- a/models/exchange/base/districts/base_exchange__districts.sql +++ /dev/null @@ -1,44 +0,0 @@ --- IMPORTS -WITH districts AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'districts') }} -), - - --- LOGIC -districts_rename AS ( - - SELECT - districts.id, - districts.country_id, - districts.state_id, - districts.name, - districts.postcode, - districts.status, - districts.deleted_at, - districts.created_at, - districts.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM districts -), - - --- FINAL -final__base_exchange__districts AS ( - - SELECT - id, - country_id, - state_id, - name, - postcode, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM districts_rename -) - -SELECT * FROM final__base_exchange__districts \ No newline at end of file diff --git a/models/exchange/base/documents/base_exchange__documents.sql b/models/exchange/base/documents/base_exchange__documents.sql deleted file mode 100644 index 59e6f5f..0000000 --- a/models/exchange/base/documents/base_exchange__documents.sql +++ /dev/null @@ -1,53 +0,0 @@ --- IMPORTS -WITH documents AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'documents') }} -), - - --- LOGIC -documents_rename AS ( - - SELECT - documents.id, - documents.owner_type, - documents.owner_id, - documents.document_type, - documents.reference, - documents.status, - documents.issued_date AS issued_at, - documents.expired_date AS expired_at, - documents.approver AS approved_by, - documents.approval_date AS approved_at, - documents.deleted_at, - documents.created_at, - documents.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM documents -), - - --- FINAL -final__base_exchange__documents AS ( - - SELECT - id, - owner_type, - owner_id, - document_type, - reference, - status, - issued_at, - expired_at, - approved_by, - approved_at, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM documents_rename - -) - -SELECT * FROM final__base_exchange__documents \ No newline at end of file diff --git a/models/exchange/base/employees/base_exchange__employees.sql b/models/exchange/base/employees/base_exchange__employees.sql deleted file mode 100644 index b895732..0000000 --- a/models/exchange/base/employees/base_exchange__employees.sql +++ /dev/null @@ -1,32 +0,0 @@ --- IMPORTS -WITH employees AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'employees') }} -), - - --- LOGIC -employees_rename AS ( - - SELECT - employees.company_id, - employees.user_id, - employees.status, - current_timestamp() AS _dbt_ran_at - - FROM employees -), - - --- FINAL -final__base_exchange__employees AS ( - - SELECT - company_id, - user_id, - status, - _dbt_ran_at - - FROM employees_rename -) - -SELECT * FROM final__base_exchange__employees \ No newline at end of file diff --git a/models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql b/models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql deleted file mode 100644 index aa6b770..0000000 --- a/models/exchange/base/failed_jobs/base_exchange__failed_jobs.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH failed_jobs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'failed_jobs') }} -), - - --- LOGIC -failed_jobs_rename AS ( - - SELECT - failed_jobs.id, - failed_jobs.connection, - failed_jobs.queue, - failed_jobs.payload, - failed_jobs.exception, - failed_jobs.failed_at, - current_timestamp() AS _dbt_ran_at - - FROM failed_jobs - -), - - --- FINAL -final__stg_failed_jobs__exchange AS ( - - SELECT - id, - connection, - queue, - payload, - exception, - failed_at, - _dbt_ran_at - - FROM failed_jobs_rename - -) - -SELECT * FROM final__stg_failed_jobs__exchange \ No newline at end of file diff --git a/models/exchange/base/files/base_exchange__files.sql b/models/exchange/base/files/base_exchange__files.sql deleted file mode 100644 index 3cc140e..0000000 --- a/models/exchange/base/files/base_exchange__files.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH files AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'files') }} -), - - --- LOGIC -files_rename AS ( - - SELECT - files.id, - files.document_id, - files.file, - files.file_type, - files.deleted_at, - files.created_at, - files.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM files - -), - - --- FINAL -final__base_exchange__files AS ( - - SELECT - id, - document_id, - file, - file_type, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM files_rename - -) - -SELECT * FROM final__base_exchange__files \ No newline at end of file diff --git a/models/exchange/base/group_transactions/base_exchange__group_transactions.sql b/models/exchange/base/group_transactions/base_exchange__group_transactions.sql deleted file mode 100644 index 628b543..0000000 --- a/models/exchange/base/group_transactions/base_exchange__group_transactions.sql +++ /dev/null @@ -1,34 +0,0 @@ --- IMPORTS -WITH group_transactions AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'group_transactions') }} -), - - --- LOGIC -group_transactions_rename AS ( - - SELECT - group_transactions.id, - group_transactions.group_id, - group_transactions.transaction_id, - current_timestamp() AS _dbt_ran_at - - FROM group_transactions - -), - - --- FINAL -final__base_exchange__group_transactions AS ( - - SELECT - id, - group_id, - transaction_id, - _dbt_ran_at - - FROM group_transactions_rename - -) - -SELECT * FROM final__base_exchange__group_transactions \ No newline at end of file diff --git a/models/exchange/base/groups/base_exchange__groups.sql b/models/exchange/base/groups/base_exchange__groups.sql deleted file mode 100644 index e4dbdd8..0000000 --- a/models/exchange/base/groups/base_exchange__groups.sql +++ /dev/null @@ -1,55 +0,0 @@ --- IMPORTS -WITH groups AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'groups') }} -), - - --- LOGIC -groups_rename AS ( - - SELECT - groups.id, - groups.reference AS marking_id, - groups.issuer AS issued_by, - groups.receiver AS received_by, - groups.amount AS base_amount, - groups.original_amount AS quote_amount, - groups.currency_id AS base_currency_id, - groups.original_currency_id AS quote_currency_id, - groups.currency_rate AS base_to_quote_currency_exchange_rate, - groups.tax AS base_tax, - groups.service_charge AS base_service_charge, - groups.status, - groups.created_at, - groups.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM groups -), - - --- FINAL -final__base_exchange__groups AS ( - - SELECT - id, - marking_id, - issued_by, - received_by, - base_amount, - quote_amount, - base_currency_id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - status, - created_at, - updated_at, - _dbt_ran_at - - FROM groups_rename - -) - -SELECT * FROM final__base_exchange__groups \ No newline at end of file diff --git a/models/exchange/base/jobs/base_exchange__jobs.sql b/models/exchange/base/jobs/base_exchange__jobs.sql deleted file mode 100644 index 4ae12ee..0000000 --- a/models/exchange/base/jobs/base_exchange__jobs.sql +++ /dev/null @@ -1,41 +0,0 @@ --- IMPORTS -WITH jobs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'jobs') }} -), - - --- LOGIC -jobs_rename AS ( - - SELECT - jobs.id, - jobs.queue, - jobs.payload, - jobs.attempts, - TO_TIMESTAMP(jobs.reserved_at) AS reserved_at, - TO_TIMESTAMP(jobs.available_at) AS available_at, - TO_TIMESTAMP(jobs.created_at) AS created_at, - current_timestamp() AS _dbt_ran_at - - FROM jobs -), - - --- FINAL -final__base_exchange__jobs AS ( - - SELECT - id, - queue, - payload, - attempts, - reserved_at, - available_at, - created_at, - _dbt_ran_at - - FROM jobs_rename - -) - -SELECT * FROM final__base_exchange__jobs \ No newline at end of file diff --git a/models/exchange/base/migrations/base_exchange__migrations.sql b/models/exchange/base/migrations/base_exchange__migrations.sql deleted file mode 100644 index c7e3231..0000000 --- a/models/exchange/base/migrations/base_exchange__migrations.sql +++ /dev/null @@ -1,33 +0,0 @@ --- IMPORTS -WITH migrations AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'migrations') }} -), - - --- LOGIC -migrations_rename AS ( - - SELECT - migrations.id, - migrations.migration, - migrations.batch, - current_timestamp() AS _dbt_ran_at - - FROM migrations -), - - --- FINAL -final__base_exchange__migrations AS ( - - SELECT - id, - migration, - batch, - _dbt_ran_at - - FROM migrations_rename - -) - -SELECT * FROM final__base_exchange__migrations \ No newline at end of file diff --git a/models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql b/models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql deleted file mode 100644 index a5b25da..0000000 --- a/models/exchange/base/model_has_roles/base_exchange__model_has_roles.sql +++ /dev/null @@ -1,34 +0,0 @@ --- IMPORTS -WITH model_has_roles AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'model_has_roles') }} -), - - --- LOGIC -migrations_rename AS ( - - SELECT - model_has_roles.role_id, - model_has_roles.model_type, - model_has_roles.model_id, - current_timestamp() AS _dbt_ran_at - - FROM model_has_roles - -), - - --- FINAL -final__base_exchange__model_has_roles AS ( - - SELECT - role_id, - model_type, - model_id, - _dbt_ran_at - - FROM migrations_rename - -) - -SELECT * FROM final__base_exchange__model_has_roles \ No newline at end of file diff --git a/models/exchange/base/notifications/base_exchange__notifications.sql b/models/exchange/base/notifications/base_exchange__notifications.sql deleted file mode 100644 index 620b4c1..0000000 --- a/models/exchange/base/notifications/base_exchange__notifications.sql +++ /dev/null @@ -1,53 +0,0 @@ --- IMPORTS -WITH notifications AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'notifications') }} -), - - --- LOGIC -notifications_rename AS ( - - SELECT - notifications.id, - notifications.title, - notifications.description, - notifications.subject_type, - notifications.subject_id, - notifications.target_type, - notifications.target_id, - notifications.causer_type, - notifications.causer_id, - notifications.status, - notifications.deleted_at, - notifications.created_at, - notifications.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM notifications -), - - --- FINAL -final__base_exchange__notifications AS ( - - SELECT - id, - title, - description, - subject_type, - subject_id, - target_type, - target_id, - causer_type, - causer_id, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM notifications_rename - -) - -SELECT * FROM final__base_exchange__notifications \ No newline at end of file diff --git a/models/exchange/base/password_resets/base_exchange__password_resets.sql b/models/exchange/base/password_resets/base_exchange__password_resets.sql deleted file mode 100644 index d3c5505..0000000 --- a/models/exchange/base/password_resets/base_exchange__password_resets.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH password_resets AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'password_resets') }} -), - - --- LOGIC -password_resets_rename AS ( - - SELECT - password_resets.id, - password_resets.user_id, - password_resets.token, - password_resets.is_expired, - password_resets.is_complete, - password_resets.created_at, - password_resets.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM password_resets - -), - - --- FINAL -final__base_exchange__password_resets AS ( - - SELECT - id, - user_id, - token, - is_expired, - is_complete, - created_at, - updated_at, - _dbt_ran_at - - FROM password_resets_rename - -) - -SELECT * FROM final__base_exchange__password_resets \ No newline at end of file diff --git a/models/exchange/base/permissions/base_exchange__permissions.sql b/models/exchange/base/permissions/base_exchange__permissions.sql deleted file mode 100644 index 0d1ff93..0000000 --- a/models/exchange/base/permissions/base_exchange__permissions.sql +++ /dev/null @@ -1,38 +0,0 @@ --- IMPORTS -WITH permissions AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'permissions') }} -), - - --- LOGIC -permissions_rename AS ( - - SELECT - permissions.id, - permissions.name, - permissions.guard_name, - permissions.created_at, - permissions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM permissions - -), - - --- FINAL -final__base_exchange__permissions AS ( - - SELECT - id, - name, - guard_name, - created_at, - updated_at, - _dbt_ran_at - - FROM permissions_rename - -) - -SELECT * FROM final__base_exchange__permissions \ No newline at end of file diff --git a/models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql b/models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql deleted file mode 100644 index 2867cf1..0000000 --- a/models/exchange/base/role_has_permissions/base_exchange__role_has_permissions.sql +++ /dev/null @@ -1,32 +0,0 @@ --- IMPORTS -WITH role_has_permissions AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'role_has_permissions') }} -), - - --- LOGIC -role_has_permissions_rename AS ( - - SELECT - role_has_permissions.permission_id, - role_has_permissions.role_id, - current_timestamp() AS _dbt_ran_at - - FROM role_has_permissions - -), - - --- FINAL -final__exchange_base__role_has_permissions AS ( - - SELECT - permission_id, - role_id, - _dbt_ran_at - - FROM role_has_permissions_rename - -) - -SELECT * FROM final__exchange_base__role_has_permissions \ No newline at end of file diff --git a/models/exchange/base/roles/base_exchange__roles.sql b/models/exchange/base/roles/base_exchange__roles.sql deleted file mode 100644 index 0f7c8db..0000000 --- a/models/exchange/base/roles/base_exchange__roles.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH roles AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'roles') }} -), - - --- LOGIC -roles_rename AS ( - - SELECT - roles.id, - roles.name, - roles.guard_name, - roles.type, - roles.created_at, - roles.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM roles - -), - - --- FINAL -final__base_exchange__roles AS ( - - SELECT - id, - name, - guard_name, - type, - created_at, - updated_at, - _dbt_ran_at - - FROM roles_rename - -) - -SELECT * FROM final__base_exchange__roles \ No newline at end of file diff --git a/models/exchange/base/segment_companies/base_exchange__segment_companies.sql b/models/exchange/base/segment_companies/base_exchange__segment_companies.sql deleted file mode 100644 index 61867d0..0000000 --- a/models/exchange/base/segment_companies/base_exchange__segment_companies.sql +++ /dev/null @@ -1,43 +0,0 @@ --- IMPORTS -WITH segment_companies AS ( - SELECT * FROM {{ ref('snap_exchange__segment_companies') }} -), - - --- LOGIC -segment_companies_rename AS ( - - SELECT - segment_companies.concat_id, - segment_companies.segment_id, - segment_companies.company_id, - segment_companies.deleted_at, - segment_companies.created_at, - segment_companies.updated_at, - segment_companies.dbt_valid_from, - segment_companies.dbt_valid_to, - current_timestamp() AS _dbt_ran_at - - FROM segment_companies -), - - --- FINAL -final__base_exchange__segment_companies AS ( - - SELECT - concat_id, - segment_id, - company_id, - deleted_at, - created_at, - updated_at, - dbt_valid_from, - dbt_valid_to, - _dbt_ran_at - - FROM segment_companies_rename - -) - -SELECT * FROM final__base_exchange__segment_companies \ No newline at end of file diff --git a/models/exchange/base/segment_constants/base_exchange__segment_constants.sql b/models/exchange/base/segment_constants/base_exchange__segment_constants.sql deleted file mode 100644 index 19adb06..0000000 --- a/models/exchange/base/segment_constants/base_exchange__segment_constants.sql +++ /dev/null @@ -1,43 +0,0 @@ --- IMPORTS -WITH segment_constants AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'segment_constants') }} -), - - --- LOGIC -segment_constants_rename AS ( - - SELECT - segment_constants.id, - segment_constants.segment_id, - segment_constants.name, - segment_constants.reference AS marking_id, - segment_constants.detail, - segment_constants.deleted_at, - segment_constants.created_at, - segment_constants.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM segment_constants -), - - --- FINAL -final_base_exchange__segment_constants AS ( - - SELECT - id, - segment_id, - name, - marking_id, - detail, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM segment_constants_rename - -) - -SELECT * FROM final_base_exchange__segment_constants \ No newline at end of file diff --git a/models/exchange/base/segments/base_exchange__segments.sql b/models/exchange/base/segments/base_exchange__segments.sql deleted file mode 100644 index 451517a..0000000 --- a/models/exchange/base/segments/base_exchange__segments.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH segments AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'segments') }} -), - - --- LOGIC -segments_rename AS ( - - SELECT - segments.id, - segments.name, - segments.type, - segments.status, - segments.deleted_at, - segments.created_at, - segments.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM segments -), - - --- FINAL -final__base_exchange__segments AS ( - - SELECT - id, - name, - type, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM segments_rename -) - -SELECT * FROM final__base_exchange__segments \ No newline at end of file diff --git a/models/exchange/base/service_types/base_exchange__service_types.sql b/models/exchange/base/service_types/base_exchange__service_types.sql deleted file mode 100644 index ce148a5..0000000 --- a/models/exchange/base/service_types/base_exchange__service_types.sql +++ /dev/null @@ -1,39 +0,0 @@ --- IMPORTS -WITH service_types AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'service_types') }} -), - - --- LOGIC -service_types_rename AS ( - - SELECT - service_types.id, - service_types.name, - service_types.status, - service_types.deleted_at, - service_types.created_at, - service_types.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM service_types -), - - --- FINAL -final__base_exchange__service_types AS ( - - SELECT - id, - name, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM service_types_rename - -) - -SELECT * FROM final__base_exchange__service_types \ No newline at end of file diff --git a/models/exchange/base/states/base_exchange__states.sql b/models/exchange/base/states/base_exchange__states.sql deleted file mode 100644 index 202bc57..0000000 --- a/models/exchange/base/states/base_exchange__states.sql +++ /dev/null @@ -1,41 +0,0 @@ --- IMPORTS -WITH states AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'states') }} -), - - --- LOGIC -states_rename AS ( - - SELECT - states.id, - states.country_id, - states.name, - states.status, - states.deleted_at, - states.created_at, - states.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM states -), - - --- FINAL -final__base_exchange__states AS ( - - SELECT - id, - country_id, - name, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM states_rename - -) - -SELECT * FROM final__base_exchange__states \ No newline at end of file diff --git a/models/exchange/base/transaction_details/base_exchange__transaction_details.sql b/models/exchange/base/transaction_details/base_exchange__transaction_details.sql deleted file mode 100644 index edc6071..0000000 --- a/models/exchange/base/transaction_details/base_exchange__transaction_details.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH transaction_details AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'transaction_detail') }} -), - - --- LOGIC -transaction_details_rename AS ( - - SELECT - transaction_details.id, - transaction_details.transaction_id, - transaction_details.product_code, - transaction_details.product_name, - transaction_details.quantity, - transaction_details.price, - transaction_details.amount, - transaction_details.deleted_at, - transaction_details.created_at, - transaction_details.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transaction_details - -), - - --- FINAL -final__base_exchange__transaction_details AS ( - - SELECT - id, - transaction_id, - product_code, - product_name, - quantity, - price, - amount, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transaction_details_rename - -) - -SELECT * FROM final__base_exchange__transaction_details \ No newline at end of file diff --git a/models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql b/models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql deleted file mode 100644 index edc4eef..0000000 --- a/models/exchange/base/transaction_logs/base_exchange__transaction_logs.sql +++ /dev/null @@ -1,73 +0,0 @@ --- IMPORTS -WITH transaction_logs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'transaction_logs') }} -), - - --- LOGIC -transaction_logs_rename AS ( - - SELECT - transaction_logs.id, - transaction_logs.transaction_id, - transaction_logs.owner_type, - transaction_logs.owner_id, - transaction_logs.type AS transaction_type, - transaction_logs.issuer AS issued_by, - transaction_logs.receiver AS received_by, - transaction_logs.recipient_bank_account_id, - transaction_logs.payment_method, - transaction_logs.payment_reference, - transaction_logs.bill_no, - transaction_logs.amount AS base_value, - transaction_logs.original_amount AS quote_value, - transaction_logs.currency_id AS base_currency_id, - transaction_logs.original_currency_id AS quote_currency_id, - transaction_logs.currency_rate AS base_to_quote_currency_exchange_rate, - transaction_logs.tax AS base_tax, - transaction_logs.service_charge AS base_service_charge, - transaction_logs.expires_on AS expired_at, - transaction_logs.status, - transaction_logs.deleted_at, - transaction_logs.created_at, - transaction_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transaction_logs -), - - --- FINAL -final__base_exchange__transaction_logs AS ( - - SELECT - id, - transaction_id, - owner_type, - owner_id, - transaction_type, - issued_by, - received_by, - recipient_bank_account_id, - payment_method, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transaction_logs_rename - -) - -SELECT * FROM final__base_exchange__transaction_logs \ No newline at end of file diff --git a/models/exchange/base/transactions/base_exchange__transactions.sql b/models/exchange/base/transactions/base_exchange__transactions.sql deleted file mode 100644 index 59b3d4f..0000000 --- a/models/exchange/base/transactions/base_exchange__transactions.sql +++ /dev/null @@ -1,71 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'transactions') }} -), - - --- LOGIC -transactions_rename AS ( - - SELECT - transactions.id, - transactions.owner_type, - transactions.owner_id, - transactions.type AS transaction_type, - transactions.issuer AS issued_by, - transactions.receiver AS received_by, - transactions.recipient_bank_account_id, - transactions.payment_method, - transactions.payment_reference, - transactions.bill_no, - transactions.amount AS base_value, - transactions.original_amount AS quote_value, - transactions.currency_id AS base_currency_id, - transactions.original_currency_id AS quote_currency_id, - transactions.currency_rate AS base_to_quote_currency_exchange_rate, - transactions.tax AS base_tax, - transactions.service_charge AS base_service_charge, - transactions.expires_on AS expired_at, - transactions.status, - transactions.deleted_at, - transactions.created_at, - transactions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transactions -), - - --- FINAL -final__base_exchange__transactions AS ( - - SELECT - id, - owner_type, - owner_id, - transaction_type, - issued_by, - received_by, - recipient_bank_account_id, - payment_method, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions_rename - -) - -SELECT * FROM final__base_exchange__transactions \ No newline at end of file diff --git a/models/exchange/base/users/base_exchange__users.sql b/models/exchange/base/users/base_exchange__users.sql deleted file mode 100644 index 6d4caf1..0000000 --- a/models/exchange/base/users/base_exchange__users.sql +++ /dev/null @@ -1,47 +0,0 @@ --- IMPORTS -WITH users AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'users') }} -), - - --- LOGIC -users_rename AS ( - - SELECT - users.id, - users.name, - users.email, - users.type AS user_type, - users.status, - users.remember_token, - users.active_at, - users.deleted_at, - users.created_at, - users.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM users -), - - --- FINAL -final__base_exchange__users AS ( - - SELECT - id, - name, - email, - user_type, - status, - remember_token, - active_at, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM users_rename - -) - -SELECT * FROM final__base_exchange__users \ No newline at end of file diff --git a/models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql b/models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql deleted file mode 100644 index 8f2c21b..0000000 --- a/models/exchange/base/wallet_logs/base_exchange__wallet_logs.sql +++ /dev/null @@ -1,47 +0,0 @@ --- IMPORTS -WITH wallet_logs AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'wallet_logs') }} -), - - --- LOGIC -wallet_logs_rename AS ( - - SELECT - wallet_logs.id, - wallet_logs.wallet_id, - wallet_logs.owner_type, - wallet_logs.owner_id, - wallet_logs.code, - wallet_logs.currency_id, - wallet_logs.amount, - wallet_logs.deleted_at, - wallet_logs.created_at, - wallet_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM wallet_logs -), - - --- FINAL -final__base_exchange__wallet_logs AS ( - - SELECT - id, - wallet_id, - owner_type, - owner_id, - code, - currency_id, - amount, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM wallet_logs_rename - -) - -SELECT * FROM final__base_exchange__wallet_logs \ No newline at end of file diff --git a/models/exchange/base/wallets/base_exchange__wallets.sql b/models/exchange/base/wallets/base_exchange__wallets.sql deleted file mode 100644 index 2f2cb9f..0000000 --- a/models/exchange/base/wallets/base_exchange__wallets.sql +++ /dev/null @@ -1,45 +0,0 @@ --- IMPORTS -WITH wallets AS ( - SELECT * FROM {{ source('src_exchange_mysql', 'wallets') }} -), - - --- LOGIC -wallets_rename AS ( - - SELECT - wallets.id, - wallets.owner_type, - wallets.owner_id, - wallets.code, - wallets.currency_id, - wallets.amount, - wallets.deleted_at, - wallets.created_at, - wallets.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM wallets -), - - --- FINAL -final__base_exchange__wallets AS ( - - SELECT - id, - owner_type, - owner_id, - code, - currency_id, - amount, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM wallets_rename - -) - -SELECT * FROM final__base_exchange__wallets \ No newline at end of file diff --git a/models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql b/models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql deleted file mode 100644 index 7aaeab0..0000000 --- a/models/exchange/intermediate/booking_logs_ger_estimate_rates/int_exchange__booking_logs_get_estimate_rates.sql +++ /dev/null @@ -1,112 +0,0 @@ --- IMPORTS -WITH booking_logs AS ( - SELECT * FROM {{ ref('stg_exchange__booking_logs') }} -), - -currency_rate_logs AS ( - SELECT * FROM {{ ref('stg_exchange__currency_rate_logs') }} -), - - --- LOGIC -booking_logs_calculate_value_join_currency_rate_logs AS ( - - SELECT - booking_logs.id, - booking_logs.booking_id, - booking_logs.company_id, - booking_logs.user_id, - booking_logs.marking_id, - booking_logs.service_type, - booking_logs.service_type_name, - booking_logs.bank_id, - booking_logs.fix_value, - booking_logs.fix_currency_id, - booking_logs.fix_currency_name, - booking_logs.fix_currency_short_code, - - currency_rate_logs.payment_method AS estimate_payment_method, - currency_rate_logs.payment_method_name AS estimate_payment_method_name, - currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate, - - booking_logs.quote_currency_id, - booking_logs.quote_currency_name, - booking_logs.quote_currency_short_code, - IFF(booking_logs.quote_currency_id = booking_logs.fix_currency_id, - fix_value, - ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value, - - booking_logs.base_currency_id, - booking_logs.base_currency_name, - booking_logs.base_currency_short_code, - IFF(booking_logs.base_currency_id = booking_logs.fix_currency_id, - fix_value, - ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value, - - -- If we have quote & base value, both of them are not contain 1 (MYR), then need to redo this function - CASE - WHEN (booking_logs.base_currency_id = '1') - THEN ROUND(estimate_base_value,2) - WHEN (booking_logs.quote_currency_id = '1') - THEN ROUND(estimate_quote_value,2) - ELSE - NULL - END AS estimate_value_rm, - - booking_logs.status, - booking_logs.status_name, - booking_logs.deleted_at, - booking_logs.created_at, - booking_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM booking_logs - - LEFT JOIN currency_rate_logs - ON (booking_logs.created_at >= currency_rate_logs.created_date_from) - AND (booking_logs.created_at < currency_rate_logs.created_date_to) - AND (booking_logs.service_type = currency_rate_logs.service_type) - AND (booking_logs.quote_currency_id = currency_rate_logs.quote_currency_id) - AND (currency_rate_logs.payment_method = '1') -), - - --- FINAL -final__int_booking_logs_get_estimate_rates__exchange AS ( - - SELECT - id, - booking_id, - company_id, - user_id, - marking_id, - service_type, - service_type_name, - bank_id, - fix_value, - fix_currency_id, - fix_currency_name, - fix_currency_short_code, - estimate_payment_method, - estimate_payment_method_name, - estimate_base_to_quote_currency_exchange_rate, - base_currency_id, - base_currency_name, - base_currency_short_code, - estimate_base_value, - estimate_value_rm, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - estimate_quote_value, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM booking_logs_calculate_value_join_currency_rate_logs -) - -SELECT * FROM final__int_booking_logs_get_estimate_rates__exchange \ No newline at end of file diff --git a/models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql b/models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql deleted file mode 100644 index 4dafd07..0000000 --- a/models/exchange/intermediate/companies_level_segment/int_exchange__companies_level_segment.sql +++ /dev/null @@ -1,81 +0,0 @@ --- Import -WITH companies AS ( - SELECT * FROM {{ ref('stg_exchange__companies') }} -), - -segments AS ( - SELECT * FROM {{ ref('stg_exchange__segments') }} -), - -companies_and_segments_relations AS ( - SELECT * FROM {{ ref('stg_exchange__companies_and_segments_relations') }} -), - - --- LOGIC -get_is_migrated_companies AS ( - SELECT - segment_id, - company_id - - FROM companies_and_segments_relations - - WHERE - segment_id = 5 -), - - --- Because we do not have timestamp to get maximum segment_id -get_segment_row_created_sequences AS ( - SELECT - *, - ROW_NUMBER() OVER (ORDER BY NULL) AS row_created_sequence - - FROM companies_and_segments_relations - - WHERE - deleted_at IS NULL - AND - segment_id IN (1,2,3,4) - AND - dbt_valid_to IS NULL -), - - -get_company_latest_segments AS ( - SELECT - *, - row_number() OVER - (PARTITION BY company_id - ORDER BY row_created_sequence DESC) AS is_latest_segment - - FROM get_segment_row_created_sequences - - QUALIFY - is_latest_segment = 1 -), - - --- FINAL -final__int_exchange__companies_level_segment AS ( - SELECT - companies.*, - get_company_latest_segments.segment_id, - segments.name AS segment_name, - IFF(get_is_migrated_companies.company_id IS NOT NULL, 1, 0) AS is_migrated_company - - FROM companies - - LEFT JOIN get_company_latest_segments - ON (companies.id = get_company_latest_segments.company_id) - - LEFT JOIN segments - ON (get_company_latest_segments.segment_id = segments.id) - - LEFT JOIN get_is_migrated_companies - ON (companies.id = get_is_migrated_companies.company_id) - -) - -SELECT * FROM final__int_exchange__companies_level_segment --- select * from get_is_migrated_company \ No newline at end of file diff --git a/models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql b/models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql deleted file mode 100644 index c102d88..0000000 --- a/models/exchange/intermediate/dedupe_user_email_verifications/int_exchange__dedupe_user_email_verifications.sql +++ /dev/null @@ -1,45 +0,0 @@ --- IMPORTS -WITH user_email_verifications AS ( - SELECT * FROM {{ ref('stg_exchange__user_email_verifications')}} -), - - --- LOGIC -dedupe_user_email_verifications AS ( - SELECT - user_id, - - -- Function below search for '1' in is_complete groupconcat string, if found then true, if null than null, else false - IFF(CONTAINS( LISTAGG(is_completed, ',') WITHIN GROUP(ORDER BY is_completed), '1'),'1','0') AS is_completed, - IFF(CONTAINS( LISTAGG(is_activated, ',') WITHIN GROUP(ORDER BY is_activated), '1'),'1','0') AS is_activated, - IFF(CONTAINS( LISTAGG(is_sent, ',') WITHIN GROUP(ORDER BY is_sent), '1'), '1', '0') AS is_sent, - COUNT(*) AS email_sent, - MIN(created_at) AS first_email_created_at, - MAX(created_at) AS last_email_created_at, - current_timestamp() AS _dbt_ran_at - - FROM user_email_verifications - - GROUP BY - user_id - -), - - --- FINAL -final__int_exchange__dedupe_user_email_verifications AS ( - - SELECT - user_id, - is_completed, - is_activated, - is_sent, - email_sent, - first_email_created_at, - last_email_created_at, - _dbt_ran_at - FROM - dedupe_user_email_verifications -) - -SELECT * FROM final__int_exchange__dedupe_user_email_verifications \ No newline at end of file diff --git a/models/exchange/intermediate/int_exchange__latest_company_contacts.sql b/models/exchange/intermediate/int_exchange__latest_company_contacts.sql deleted file mode 100644 index 23b3b7e..0000000 --- a/models/exchange/intermediate/int_exchange__latest_company_contacts.sql +++ /dev/null @@ -1,38 +0,0 @@ ---IMPORT -WITH contacts AS ( - SELECT * FROM {{ ref('stg_exchange__contacts') }} -), - ---LOGIC -get_latest_company_contacts AS ( - SELECT - *, - row_number() OVER - (PARTITION BY company_id - ORDER BY contact_created_datetime DESC) AS last_row_number - - FROM contacts - - QUALIFY - last_row_number = 1 -), - ---FINAL -final__int_exchange__latest_company_contacts AS ( - SELECT - id, - company_id, - country_id, - country_name, - reference, - phone, - email, - wechat_id, - contact_deleted_datetime, - contact_created_datetime, - contact_updated_datetime - - FROM get_latest_company_contacts -) - -SELECT * FROM final__int_exchange__latest_company_contacts diff --git a/models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql b/models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql deleted file mode 100644 index 2c8054e..0000000 --- a/models/exchange/intermediate/transaction_costs_remove_duplicate_row/int_exchange__transaction_costs_remove_duplicate_row.sql +++ /dev/null @@ -1,72 +0,0 @@ --- IMPORT -WITH transaction_costs AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} -), - - --- LOGIC -duplicate_id_need_remove AS ( - SELECT - ID, - TRANSACTION_ORDER_ID, - BASE_VALUE, - QUOTE_VALUE, - BASE_TO_QUOTE_CURRENCY_EXCHANGE_RATE, - STATUS_NAME, - CREATED_AT, - UPDATED_AT, - row_number() OVER - (PARTITION BY transaction_costs.transaction_order_id - ORDER BY transaction_costs.updated_at DESC) AS row_index - - FROM - transaction_costs - - WHERE - deleted_at IS NULL - - QUALIFY - row_index = 2 -), - - --- duplicate_id_need_remove AS ( --- SELECT --- id - --- FROM mark_duplicate_is_not_deleted_id - --- WHERE --- row_index = 2 - --- ), - - -removed_duplicate_id_table AS ( - SELECT - transaction_costs.* - - FROM transaction_costs - - LEFT JOIN duplicate_id_need_remove - ON (transaction_costs.id = duplicate_id_need_remove.id) - WHERE - duplicate_id_need_remove.id IS NULL -), - - ---FINAL -final__int_exchange__transaction_costs_remove_duplicate_row AS ( - SELECT * FROM removed_duplicate_id_table -) - -SELECT * FROM final__int_exchange__transaction_costs_remove_duplicate_row - - - --- SELECT * FROM duplicate_transaction_order_id_which_not_deleted --- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at --- SELECT * FROM duplicate_id_need_remove --- SELECT * FROM removed_duplicate_id_table WHERE deleted_at is not null --- SELECT * FROM mark_duplicate_is_not_deleted_id WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at --- SELECT * FROM removed_duplicate_id_table WHERE TRANSACTION_ORDER_ID IN (118382,118372,42895,116622,116480,43363,40799) order by TRANSACTION_ORDER_ID, updated_at diff --git a/models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql b/models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql deleted file mode 100644 index 51134fc..0000000 --- a/models/exchange/intermediate/transaction_log_orders_get_user_ids/int_exchange__transaction_log_orders_get_user_ids.sql +++ /dev/null @@ -1,28 +0,0 @@ --- IMPORTS -WITH transaction_log_orders AS ( - SELECT * FROM {{ ref('int_exchange__transaction_log_orders_remove_undo_status') }} -), - -bookings AS ( - SELECT * FROM {{ ref('stg_exchange__bookings') }} -), - --- LOGICS -transaction_log_orders_join_bookings AS ( - SELECT - transaction_log_orders.*, - bookings.user_id - - FROM transaction_log_orders - - LEFT JOIN bookings - ON (transaction_log_orders.booking_id = bookings.id) -), - - --- FINAL -final__int_exchange__transaction_log_orders_get_user_company_ids AS ( - SELECT * FROM transaction_log_orders_join_bookings -) - -SELECT * FROM final__int_exchange__transaction_log_orders_get_user_company_ids \ No newline at end of file diff --git a/models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql b/models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql deleted file mode 100644 index 0e6d5cc..0000000 --- a/models/exchange/intermediate/transaction_log_orders_remove_undo_status/int_exchange__transaction_log_orders_remove_undo_status.sql +++ /dev/null @@ -1,26 +0,0 @@ --- IMPORT -WITH transaction_log_orders AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_log_orders') }} -), - - --- LOGIC -transaction_log_orders_mark_undo_status AS ( - SELECT - * - - FROM transaction_log_orders - - QUALIFY - row_number() OVER - (PARTITION BY transaction_log_orders.status, transaction_log_orders.transaction_order_id - ORDER BY transaction_log_orders.updated_at DESC) = 1 -), - - --- FINAL -final__int_exchange__transaction_log_orders_remove_undo_status AS ( - SELECT * FROM transaction_log_orders_mark_undo_status -) - -SELECT * FROM final__int_exchange__transaction_log_orders_remove_undo_status \ No newline at end of file diff --git a/models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql b/models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql deleted file mode 100644 index 6e0ec76..0000000 --- a/models/exchange/intermediate/transaction_orders_get_user_ids/int_exchange__transaction_orders_get_user_ids.sql +++ /dev/null @@ -1,30 +0,0 @@ --- IMPORTS -WITH transaction_orders AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_orders') }} -), - -bookings AS ( - SELECT * FROM {{ ref('stg_exchange__bookings') }} -), - --- LOGICS -transaction_orders_join_bookings AS ( - SELECT - transaction_orders.*, - bookings.user_id - - - - FROM transaction_orders - - LEFT JOIN bookings - ON (transaction_orders.booking_id = bookings.id) -), - - --- FINAL -final__int_exchange__transaction_orders_get_user_company_ids AS ( - SELECT * FROM transaction_orders_join_bookings -) - -SELECT * FROM final__int_exchange__transaction_orders_get_user_company_ids \ No newline at end of file diff --git a/models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql b/models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql deleted file mode 100644 index 1fbbb02..0000000 --- a/models/exchange/marts/reporting/rep_exchange__daily_orders/rep_exchange__daily_orders.sql +++ /dev/null @@ -1,234 +0,0 @@ --- IMPORT -WITH transaction_log_orders AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_log_orders') }} -), - -latest_transaction_orders AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} -), - -booking_logs AS ( - SELECT * FROM {{ ref('fct_exchange__booking_logs') }} -), - -latest_bookings AS ( - SELECT * FROM {{ ref('fct_exchange__bookings') }} -), - -latest_transaction_costs AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_costs') }} -), - -transaction_log_costs AS ( - SELECT * FROM {{ ref('fct_exchange__transaction_log_costs') }} -), - -companies AS ( - SELECT * FROM {{ ref('dim_exchange__companies') }} -), - -dates AS ( - SELECT * FROM {{ ref('dim_exchange__dates') }} -), - - ---LOGIC -transaction_order_logs_status_datetime AS ( - - SELECT - transaction_order_id, - MIN(created_at) AS created_datetime, - MAX(CASE WHEN status = '0' THEN updated_at END) AS pending_submission_datetime, - MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime, - MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime, - MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime, - MAX(CASE WHEN status = '4' THEN updated_at END) AS rejected_datetime, - MAX(CASE WHEN status = '5' THEN updated_at END) AS suspended_datetime - - FROM transaction_log_orders - - GROUP BY - transaction_order_id -), - - -cost_logs_status_datetime AS ( - - SELECT - transaction_order_id, - MIN(created_at) AS created_datetime, - MAX(CASE WHEN status = '1' THEN updated_at END) AS pending_verification_datetime, - MAX(CASE WHEN status = '2' THEN updated_at END) AS approved_datetime, - MAX(CASE WHEN status = '3' THEN updated_at END) AS completed_datetime - - FROM transaction_log_costs - - GROUP BY - transaction_order_id -), - - -latest_transaction_costs_remove_deleted AS ( - SELECT - * - - FROM - latest_transaction_costs - - WHERE - deleted_at IS NULL -), - - -previous_order_date_row_rank AS ( - SELECT - *, - row_number() OVER - (PARTITION BY latest_transaction_orders.company_id - ORDER BY latest_transaction_orders.created_at) AS row_rank_index - - FROM - latest_transaction_orders -), - -get_previous_order_date AS ( - SELECT - previous_order_date_row_rank.id, - previous_order_date_row_rank_clone.created_at AS previous_order_created_datetime - - FROM previous_order_date_row_rank - - LEFT JOIN previous_order_date_row_rank AS previous_order_date_row_rank_clone - ON previous_order_date_row_rank.row_rank_index = previous_order_date_row_rank_clone.row_rank_index + 1 - AND previous_order_date_row_rank.company_id = previous_order_date_row_rank_clone.company_id -), - ---FINAL -final__rep_exchange__daily_transactions AS ( - SELECT - latest_transaction_orders.id, - latest_transaction_orders.booking_id, - - latest_bookings.marking_id AS booking_marking_id, - latest_bookings.service_type, - latest_bookings.service_type_name, - latest_bookings.booking_created_datetime, - - latest_transaction_orders.seller_company_id, - latest_transaction_orders.bank_id, - latest_transaction_orders.user_id, - latest_transaction_orders.company_id, - - companies.marking_id AS company_marking_id, - companies.company_type, - companies.company_type_name, - companies.business_type AS company_business_type, - companies.business_type_name AS company_business_type_name, - companies.segment_id AS company_segment_id, - companies.segment_name AS company_segment_name, - companies.is_migrated_company AS company_is_migrated_company, - companies.country_id AS company_country_id, - companies.country_name AS company_country_name, - companies.state_id AS company_state_id, - companies.state_name AS company_state_name, - companies.district_id AS company_district_id, - companies.district_name AS company_district_name, - companies.postcode AS company_postcode, - companies.wallet_id AS company_wallet_id, - companies.have_wallet AS company_have_wallet, - -- i stay here - latest_bookings.is_first_time_user AS is_first_time_booking_user, - latest_bookings.is_first_time_company AS is_first_time_booking_company, - latest_transaction_orders.is_first_time_user, - latest_transaction_orders.is_first_time_company, - - latest_transaction_orders.status, - latest_transaction_orders.status_name, - latest_transaction_orders.transaction_type, - latest_transaction_orders.transaction_type_name, - latest_transaction_orders.payment_method, - latest_transaction_orders.payment_method_name, - latest_transaction_orders.base_currency_id, - latest_transaction_orders.base_currency_name, - latest_transaction_orders.quote_currency_id, - latest_transaction_orders.quote_currency_name, - latest_transaction_orders.base_to_quote_currency_exchange_rate, - latest_transaction_orders.base_value, - latest_transaction_orders.quote_value, - latest_transaction_orders.transaction_value_rm, - latest_transaction_orders.transaction_base_tax, - latest_transaction_orders.transaction_tax_rm, - latest_transaction_orders.base_service_charge, - latest_transaction_orders.transaction_service_charge_rm, - - get_previous_order_date.previous_order_created_datetime AS previous_order_created_datetime, - transaction_order_logs_status_datetime.created_datetime AS order_created_datetime, - order_created_dates.date_actual AS order_created_date, - order_created_dates.first_day_of_week AS order_created_week, - - transaction_order_logs_status_datetime.pending_submission_datetime AS order_pending_submission_datetime, - transaction_order_logs_status_datetime.pending_verification_datetime AS order_pending_verification_datetime, - transaction_order_logs_status_datetime.approved_datetime AS order_approved_datetime, - transaction_order_logs_status_datetime.completed_datetime AS order_completed_datetime, - transaction_order_logs_status_datetime.rejected_datetime AS order_rejected_datetime, - transaction_order_logs_status_datetime.suspended_datetime AS order_suspended_datetime, - - latest_transaction_costs_remove_deleted.id AS cost_id, - latest_transaction_costs_remove_deleted.supplier_company_id AS cost_supplier_company_id, - latest_transaction_costs_remove_deleted.status AS cost_status, - latest_transaction_costs_remove_deleted.status_name AS cost_status_name, - latest_transaction_costs_remove_deleted.base_currency_id AS cost_base_currency_id, - latest_transaction_costs_remove_deleted.base_currency_name AS cost_base_currency_name, - latest_transaction_costs_remove_deleted.quote_currency_id AS cost_quote_currency_id, - latest_transaction_costs_remove_deleted.quote_currency_name AS cost_quote_currency_name, - latest_transaction_costs_remove_deleted.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate, - latest_transaction_costs_remove_deleted.base_value AS cost_base_value, - latest_transaction_costs_remove_deleted.quote_value AS cost_quote_value, - latest_transaction_costs_remove_deleted.transaction_value_rm AS cost_transaction_value_rm, - latest_transaction_costs_remove_deleted.transaction_base_tax AS cost_transaction_base_tax, - latest_transaction_costs_remove_deleted.transaction_tax_rm AS cost_transaction_tax_rm, - latest_transaction_costs_remove_deleted.base_service_charge AS cost_base_service_charge, - latest_transaction_costs_remove_deleted.transaction_service_charge_rm AS cost_transaction_service_charge_rm, - - cost_logs_status_datetime.created_datetime AS cost_created_datetime, - cost_logs_status_datetime.pending_verification_datetime AS cost_pending_verification_datetime, - cost_logs_status_datetime.approved_datetime AS cost_approved_datetime, - cost_logs_status_datetime.completed_datetime AS cost_completed_datetime - - FROM latest_transaction_orders - - LEFT JOIN latest_bookings - ON (latest_transaction_orders.booking_id = latest_bookings.id) - - LEFT JOIN transaction_order_logs_status_datetime - ON (latest_transaction_orders.id = transaction_order_logs_status_datetime.transaction_order_id) - - LEFT JOIN latest_transaction_costs_remove_deleted - ON (latest_transaction_orders.id = latest_transaction_costs_remove_deleted.transaction_order_id) - - LEFT JOIN cost_logs_status_datetime - ON (latest_transaction_orders.id = cost_logs_status_datetime.transaction_order_id) - - LEFT JOIN companies - ON (latest_transaction_orders.company_id = companies.id) - - LEFT JOIN get_previous_order_date - ON(latest_transaction_orders.id = get_previous_order_date.id) - - LEFT JOIN dates AS order_created_dates - ON (date(transaction_order_logs_status_datetime.created_datetime) = order_created_dates.date_day ) - - ORDER BY latest_transaction_orders.updated_at ASC -) - - -SELECT * FROM final__rep_exchange__daily_transactions --- @YAM: not unique order id ---(2007, 40808, 2013, 40920, 41101, 125169, 18604, 18552, 18550, 35795, 117444, 72332, 40798, 51174, 18553, 2010, 66195, 89413, 1678, 71516, 5673, 78366, 66208, 18546, 1672, 1684, 18579, 66202, 132074, 18554, 43249, 40804, 86259, 4881, 2311, 103084, 18549, 41097, 106321, 17634, 66207, 18582, 1661, 63970) and so on.... - - --- (OMAIR) --- SELECT booking_id, count(*) FROM booking_logs WHERE status= 3 GROUP BY booking_id ORDER BY count(*) desc --- SELECT booking_id, company_id, user_id, fix_value, status_name, updated_at FROM booking_logs WHERE booking_id IN (18901,18618,22960) ORDER BY booking_id, updated_at --- SELECT * FROM booking_logs WHERE booking_id = 22173 ORDER BY updated_at - \ No newline at end of file diff --git a/models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml b/models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml deleted file mode 100644 index f1aa2f9..0000000 --- a/models/exchange/marts/reporting/rep_exchange__daily_orders/test_exchange__daily_orders.yml +++ /dev/null @@ -1,211 +0,0 @@ -version: 2 - -models: - - name: rep_exchange__daily_orders - columns: - - name: id - description: transaction orders id - tests: - - unique - - not_null - - - name: booking_id - description: transaction orders id - tests: - - not_null - - relationships: - to: ref('base_exchange__bookings') - field: id - - - name: service_type - description: type of service - tests: - - not_null - - - name: service_type_name - description: name of type of service - tests: - - not_null - - - name: booking_created_datetime - description: date of boking created - tests: - - not_null - - - name: seller_company_id - description: seller company id - tests: - - not_null - - - name: bank_id - description: bank id - tests: - - not_null - - - name: company_id - description: user id - tests: - - not_null - - - name: company_marking_id - description: company marking id - tests: - - relationships: - to: ref('base_exchange__companies') - field: marking_id - - - name: company_type - description: type of company - tests: - - not_null - - - name: company_type_name - description: name type of company - tests: - - accepted_values: - values: - - PERSONAL - - CORPORATE - - - name: company_business_type - description: company business type - tests: - - not_null - - - name: company_business_type_name - description: company business type name - tests: - - accepted_values: - values: - - IMPORTER - - - name: company_segment_id - description: company segment id - tests: - - not_null - - - name: company_segment_name - description: company segment name - tests: - - accepted_values: - values: - - PLATINIUM MEMBER - - GOLD MEMBER - - STANDARD SEGMENT - - - name: cost_id - description: cost id - tests: - - relationships: - to: ref('base_exchange__transactions') - field: id - - - name: base_value - description: test base value combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: quote_value - description: test quote value combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: transaction_value_rm - description: transaction value rm combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: transaction_base_tax - description: test transaction base tax combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: base_service_charge - description: test base service charge combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: transaction_tax_rm - description: test transaction tax rm combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_base_value - description: test cost base value combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_base_to_quote_currency_exchange_rate - description: test cost base to quote currency exchange rate combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_quote_value - description: test cost quote value combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_transaction_value_rm - description: test COST TRANSACTION VALUE RM combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_transaction_tax_rm - description: test COST TRANSACTION TAX RM combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_base_service_charge - description: test COST BASE SERVICE CHARGE combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: cost_transaction_service_charge_rm - description: test COST TRANSACTION SERVICE CHARGE_RM combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - - name: transaction_service_charge_rm - description: test transaction service charge rm combine with dbt_utils - tests: - - not_null - - dbt_utils.expression_is_true: - expression: " >= 0" - - tests: - - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: - column_A: order_completed_datetime - column_B: order_created_datetime - or_equal: True - tests: - - dbt_utils.expression_is_true: - expression: "cost_transaction_value_rm + cost_transaction_tax_rm + cost_transaction_service_charge_rm < 2*(transaction_value_rm + transaction_base_tax + transaction_tax_rm)" - severity: warn - -# FRESHNESS SOURCE diff --git a/models/exchange/marts/warehouse/dim_exchange__companies.sql b/models/exchange/marts/warehouse/dim_exchange__companies.sql deleted file mode 100644 index f115879..0000000 --- a/models/exchange/marts/warehouse/dim_exchange__companies.sql +++ /dev/null @@ -1,178 +0,0 @@ --- IMPORTS -WITH companies AS ( - SELECT * FROM {{ ref('int_exchange__companies_level_segment') }} -), - -company_addresses AS ( - SELECT * FROM {{ ref('stg_exchange__company_addresses') }} -), - -current_wallets AS ( - SELECT * FROM {{ ref('stg_exchange__current_wallets') }} -), - -companies_and_users_relations AS ( - SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }} -), - -company_contacts AS ( - SELECT * FROM {{ ref('int_exchange__latest_company_contacts') }} -), - -documents AS ( - SELECT * FROM {{ ref('stg_exchange__documents') }} -), - --- LOGICS -latest_billing_company_addresses AS ( - SELECT - *, - - -- 1 will be latest updated address, rank by updated_at date - row_number() OVER - (PARTITION BY company_addresses.company_id - ORDER BY company_addresses.updated_at DESC) AS lastest_address_rank - - FROM company_addresses - - WHERE - is_billing_address = 1 - - QUALIFY - lastest_address_rank = 1 -), - - -company_current_wallets AS ( - SELECT - id, - owner_id AS company_id, - amount, - currency_id, - currency_name, - created_at, - updated_at - - FROM current_wallets - - WHERE - owner_type = 'App\\Models\\Company' -), - - -count_company_user AS ( - SELECT - company_id, - COUNT(user_id) AS count_user - - FROM companies_and_users_relations - - GROUP BY company_id -), - -latest_company_upload_identity_documents AS ( - SELECT - id, - owner_type, - owner_id AS company_id, - document_type AS identity_document_type_name, - reference AS identity_number, - status, - status_name, - approved_at, - created_at, - - row_number() OVER - (PARTITION BY company_id - ORDER BY created_at DESC) AS latest_row_number - - FROM documents - - WHERE - owner_type = 'App\\Models\\Company' - AND - document_type IN ('IDENTITY_CARD','SSM_REGISTRATION') - AND - status = '2' - - QUALIFY - latest_row_number = '1' - -), - --- FINAL -final__dim_exchange__companies AS ( - SELECT - companies.id, - companies.name, - companies.marking_id, - companies.account_software_id, - companies.company_type, - companies.company_type_name, - companies.business_type, - companies.business_type_name, - companies.segment_id, - companies.segment_name, - companies.is_migrated_company, - companies.status, - companies.status_name, - companies.created_at, - - company_contacts.reference AS company_contact_reference, - company_contacts.phone AS company_contact_phone, - company_contacts.email AS company_contact_email, - company_contacts.wechat_id AS company_contact_wechat_id, - - latest_billing_company_addresses.id AS address_id, - latest_billing_company_addresses.country_id, - latest_billing_company_addresses.country_name, - latest_billing_company_addresses.country_short_code, - latest_billing_company_addresses.state_id, - latest_billing_company_addresses.state_name, - latest_billing_company_addresses.district_id, - latest_billing_company_addresses.district_name, - latest_billing_company_addresses.postcode, - latest_billing_company_addresses.address_line_one, - latest_billing_company_addresses.address_line_two, - latest_billing_company_addresses.is_billing_address, - latest_billing_company_addresses.deleted_at AS address_deleted_at, - latest_billing_company_addresses.created_at AS address_created_at, - latest_billing_company_addresses.updated_at AS address_updated_at, - - company_current_wallets.id AS wallet_id, - IFF(company_current_wallets.id IS NOT NULL, 1, 0) AS have_wallet, - company_current_wallets.amount AS wallet_balance, - company_current_wallets.currency_id AS wallet_currency_id, - company_current_wallets.currency_name AS wallet_currency_name, - company_current_wallets.created_at AS wallet_created_at, - company_current_wallets.updated_at AS wallet_updated_at, - - latest_company_upload_identity_documents.identity_document_type_name, - latest_company_upload_identity_documents.identity_number, - latest_company_upload_identity_documents.status AS identity_status, - latest_company_upload_identity_documents.status_name AS identity_status_name, - latest_company_upload_identity_documents.approved_at AS identity_approved_datetime, - latest_company_upload_identity_documents.created_at AS identity_created_datetime, - - count_company_user.count_user AS number_of_user - - FROM companies - - LEFT JOIN latest_billing_company_addresses - ON (companies.id = latest_billing_company_addresses.company_id) - - LEFT JOIN company_current_wallets - ON (companies.id = company_current_wallets.company_id) - - LEFT JOIN count_company_user - ON (companies.id = count_company_user.company_id) - - LEFT JOIN company_contacts - ON (companies.id = company_contacts.company_id) - - LEFT JOIN latest_company_upload_identity_documents - ON (companies.id = latest_company_upload_identity_documents.company_id) -) - - -SELECT * FROM final__dim_exchange__companies diff --git a/models/exchange/marts/warehouse/dim_exchange__dates.sql b/models/exchange/marts/warehouse/dim_exchange__dates.sql deleted file mode 100644 index 5ee656f..0000000 --- a/models/exchange/marts/warehouse/dim_exchange__dates.sql +++ /dev/null @@ -1,202 +0,0 @@ --- Code and forum link refer: https://gitlab.com/gitlab-data/analytics/-/blob/master/transform/snowflake-dbt/models/sources/date/date_details_source.sql https://stackoverflow.com/questions/71016585/dbt-use-dbt-modeling-to-insert-rows-in-a-table-like-date-dimension-table-in-az - --- Import - --- @NOTE TO GETSON, join this into calculated -WITH holiday_and_non_working_day_lists AS ( - SELECT * FROM {{ ref('seed_exchange__holiday_and_non_working_day_lists') }} -), - --- LOGIC -date_spine AS ( - - {{ dbt_utils.date_spine( - start_date="to_date('01/01/2015', 'mm/dd/yyyy')", - datepart="day", - end_date="dateadd(year, 40, current_date)" - ) - }} - -), - -calculated as ( - - SELECT - date_day, - date_day AS date_actual, - - DAYNAME(date_day) AS day_name, - - DATE_PART('month', date_day) AS month_actual, - DATE_PART('year', date_day) AS year_actual, - - DATE_PART(quarter, date_day) AS quarter_actual, - - CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7 - ELSE DATE_PART(dayofweek, date_day) END AS day_of_week, - - CASE WHEN day_name = 'Mon' THEN date_day - ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week, - - CASE WHEN day_name = 'Mon' THEN WEEK(date_day) - ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column - - CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1' - THEN '1' - ELSE week_of_year_temp END AS week_of_year, - - DATE_PART('day', date_day) AS day_of_month, - - ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter, - ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year, - - -- @TODO @GERSON ,done - -- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023) - CASE WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual - when DATE_PART('month', DATEADD( month,-1, date_day) ) =12 THEN year_actual-1 - - ELSE (year_actual+1) END AS fiscal_year, - - -- @TODO @GERSON , done - -- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4) - CASE WHEN month_actual in (2,3,4) THEN '1' - WHEN month_actual in (5,6,7) THEN '2' - WHEN month_actual in (8,9,10) THEN '3' - WHEN month_actual in (11,12) THEN '4' - WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4' - -- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4' - ELSE '4' END AS fiscal_quarter, - - -- @TODO @GERSON ,done - ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter, - ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year, - - TO_CHAR(date_day, 'MMMM') AS month_name, - - TRUNC(date_day, 'Month') AS first_day_of_month, - LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month, - - FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year, - LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year, - - FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter, - LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter, - - -- @TODO @GERSON ,done - -- Change this 2 too - FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter, - LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter, - - -- @TODO @GERSON,done - FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year, - LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year, - - -- @TODO @GERSON ,done - DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year, - - -- @TODO @GERSON ,done - CASE WHEN EXTRACT('month', date_day) = 1 THEN 12 - ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year, - - LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week, - - (year_actual || '-Q' || fiscal_quarter) AS quarter_name, - - -- @TODO @GERSON,done - (fiscal_year || '-' || DECODE(fiscal_quarter, - 1, 'Q1', - 2, 'Q2', - 3, 'Q3', - 4, 'Q4')) AS fiscal_quarter_name, - ('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy, - DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute, - --here check - fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name, - ('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy, - - -- @TODO @GERSON - -- Use the Seed, we will not doing the hard code in SQL, we use CSV to refer the holiday and desc - h.holiday_desc, - h.is_holiday, - h.holiday_race, - h.holiday_country, - h.holiday_state, - h.is_non_working_day, - -- @TODO @GERSON - DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter, - IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter, - DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year, - IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year, - - DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa, - DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings - - FROM date_spine s - - LEFT JOIN holiday_and_non_working_day_lists h - ON s.date_day = to_date(h.date, 'dd/mm/yyyy') - - QUALIFY date_actual >= '2016-01-01' - - ORDER BY date_day - - -), - -final AS ( - - SELECT - date_day, - date_actual, - day_name, - month_actual, - year_actual, - quarter_actual, - day_of_week, - first_day_of_week, - week_of_year, - day_of_month, - day_of_quarter, - day_of_year, - fiscal_year, - fiscal_quarter, - day_of_fiscal_quarter, - day_of_fiscal_year, - month_name, - first_day_of_month, - last_day_of_month, - first_day_of_year, - last_day_of_year, - first_day_of_quarter, - last_day_of_quarter, - first_day_of_fiscal_quarter, - last_day_of_fiscal_quarter, - first_day_of_fiscal_year, - last_day_of_fiscal_year, - week_of_fiscal_year, - month_of_fiscal_year, - last_day_of_week, - quarter_name, - fiscal_quarter_name, - fiscal_quarter_name_fy, - fiscal_quarter_number_absolute, - fiscal_month_name, - fiscal_month_name_fy, - holiday_desc, - is_holiday, - holiday_race, - holiday_country, - holiday_state, - is_non_working_day, - last_month_of_fiscal_quarter, - is_first_day_of_last_month_of_fiscal_quarter, - last_month_of_fiscal_year, - is_first_day_of_last_month_of_fiscal_year, - snapshot_date_fpa, - snapshot_date_billings, - current_timestamp() as _dbt_run_at - FROM calculated - -) - -SELECT * FROM final \ No newline at end of file diff --git a/models/exchange/marts/warehouse/dim_exchange__users.sql b/models/exchange/marts/warehouse/dim_exchange__users.sql deleted file mode 100644 index 7275de2..0000000 --- a/models/exchange/marts/warehouse/dim_exchange__users.sql +++ /dev/null @@ -1,45 +0,0 @@ --- IMPORTS -WITH users AS ( - SELECT * FROM {{ ref('stg_exchange__users') }} -), - -dedupe_user_email_verifications AS ( - SELECT * FROM {{ ref('int_exchange__dedupe_user_email_verifications') }} -), - --- LOGICS -users_join_dedupe_user_email_verifications AS ( - SELECT - users.id, - users.name, - users.email, - users.user_type, - users.user_type_name, - users.status, - users.status_name, - users.company_id, - users.created_at, - users.deleted_at, - users.updated_at, - dedupe_user_email_verifications.is_sent AS email_verification_is_sent, - dedupe_user_email_verifications.is_completed AS email_verification_is_completed, - dedupe_user_email_verifications.is_activated AS email_verification_is_activated, - dedupe_user_email_verifications.email_sent AS email_verification_email_sent, - first_email_created_at AS email_verification_first_email_created_at, - last_email_created_at AS email_verification_last_email_created_at - - - FROM users - - JOIN dedupe_user_email_verifications - ON (users.id = dedupe_user_email_verifications.user_id) -), - - --- FINAL -final__dim_exchange__users AS ( - SELECT * FROM users_join_dedupe_user_email_verifications -) - -SELECT * -FROM final__dim_exchange__users diff --git a/models/exchange/marts/warehouse/fct_exchange__booking_logs.sql b/models/exchange/marts/warehouse/fct_exchange__booking_logs.sql deleted file mode 100644 index 11537d6..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__booking_logs.sql +++ /dev/null @@ -1,81 +0,0 @@ --- IMPORTS -WITH booking_logs AS ( - SELECT * FROM {{ ref('int_exchange__booking_logs_get_estimate_rates') }} -), - - --- LOGICS -first_time_user_booking_id AS ( - SELECT - booking_id, - user_id, - created_at, - row_number() OVER - (PARTITION BY booking_logs.user_id - ORDER BY booking_logs.created_at) AS cummulative_count_booking - - FROM - booking_logs - - GROUP BY - booking_id, - user_id, - created_at - - QUALIFY - cummulative_count_booking = 1 -), - - -first_time_company_booking_id AS ( - SELECT - booking_id, - company_id, - created_at, - row_number() OVER - (PARTITION BY booking_logs.company_id - ORDER BY booking_logs.created_at) AS cummulative_count_booking - - FROM - booking_logs - - GROUP BY - booking_id, - company_id, - created_at - - QUALIFY - cummulative_count_booking = 1 -), - - -booking_logs_lists AS ( - SELECT - booking_logs.*, - IFF(first_time_user_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_user, - IFF(first_time_company_booking_id.booking_id IS NOT NULL, 1, 0) AS is_first_time_company - - FROM - booking_logs - - LEFT JOIN first_time_user_booking_id - ON (booking_logs.booking_id = first_time_user_booking_id.booking_id) - - LEFT JOIN first_time_company_booking_id - ON (booking_logs.booking_id = first_time_company_booking_id.booking_id) - - ORDER BY - created_at -), - - --- FINAL -final_fct_exchange__booking_logs AS ( - SELECT - * - - FROM booking_logs_lists -) - -SELECT * FROM final_fct_exchange__booking_logs - diff --git a/models/exchange/marts/warehouse/fct_exchange__bookings.sql b/models/exchange/marts/warehouse/fct_exchange__bookings.sql deleted file mode 100644 index 34d460e..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__bookings.sql +++ /dev/null @@ -1,60 +0,0 @@ --- IMPORTS -WITH bookings AS ( - SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_rates') }} -), - - --- LOGICS -bookings_lists AS ( - SELECT - bookings.*, - - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY bookings.company_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1 - ELSE 0 - END AS is_first_time_company, - - CASE - WHEN bookings.status = '3' - AND - MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL)) - OVER (PARTITION BY bookings.company_id) - = bookings.booking_created_datetime - THEN 1 - ELSE 0 - END AS is_first_time_company_completed, - - - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY bookings.user_id ORDER BY bookings.booking_created_datetime) = 1 THEN 1 - ELSE 0 - END AS is_first_time_user, - - CASE - WHEN bookings.status = '3' - AND - MIN(IFF(bookings.status = '3', bookings.booking_created_datetime, NULL)) - OVER (PARTITION BY bookings.user_id) - = bookings.booking_created_datetime - THEN 1 - ELSE 0 - END AS is_first_time_user_completed - - FROM - bookings - - ORDER BY - booking_created_datetime -), - - --- FINAL -final_fct_exchange__bookings AS ( - SELECT - * - - FROM bookings_lists -) - -SELECT * FROM final_fct_exchange__bookings - diff --git a/models/exchange/marts/warehouse/fct_exchange__email_verifications.sql b/models/exchange/marts/warehouse/fct_exchange__email_verifications.sql deleted file mode 100644 index e53d802..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__email_verifications.sql +++ /dev/null @@ -1,48 +0,0 @@ ---IMPORT -WITH user_email_verifications AS ( - SELECT * FROM {{ ref('stg_exchange__user_email_verifications') }} -), - -companies_and_users_relations AS ( - SELECT * FROM {{ ref('stg_exchange__companies_and_users_relations') }} -), - - ---LOGIC -user_email_verifications_get_company_id AS ( - SELECT - user_email_verifications.id, - user_email_verifications.user_id, - companies_and_users_relations.company_id, - user_email_verifications.email, - user_email_verifications.is_sent, - user_email_verifications.is_completed, - user_email_verifications.is_activated, - user_email_verifications.created_at, - user_email_verifications.updated_at - - FROM user_email_verifications - - LEFT JOIN companies_and_users_relations - ON (user_email_verifications.user_id = companies_and_users_relations.user_id) -), - - ---FINAL -final__fct_exchange__company_email_verifications AS ( - SELECT - id, - user_id, - company_id, - email, - is_sent, - is_completed, - is_activated, - created_at AS verification_email_created_datetime, - updated_at AS verification_email_updated_datetime - - FROM user_email_verifications_get_company_id -) - - -SELECT * FROM final__fct_exchange__company_email_verifications \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql deleted file mode 100644 index 3a42556..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__transaction_costs.sql +++ /dev/null @@ -1,60 +0,0 @@ --- IMPORTS -WITH transaction_costs AS ( - SELECT * FROM {{ ref('int_exchange__transaction_costs_remove_duplicate_row') }} -), - - --- LOGICS -transaction_costs_lists AS ( - SELECT - id, - transaction_order_id, - supplier_company_id, - bank_id, - buyer_company_id, - status, - status_name, - transaction_type, - transaction_type_name, - payment_method, - payment_method_name, - base_currency_id, - base_currency_name, - quote_currency_id, - quote_currency_name, - base_to_quote_currency_exchange_rate, - - base_value, - quote_value, - -- use for future if base currency id other from '1' - IFF(base_currency_id = '1', - base_value, - 99999999) AS transaction_value_rm, - - base_tax AS transaction_base_tax, - -- use for future if base currency id other from '1' - IFF(base_currency_id = '1', - base_tax, - 99999999) AS transaction_tax_rm, - - base_service_charge, - -- use for future if base currency id other from '1' - IFF(base_currency_id = '1', - base_service_charge, - 99999999) AS transaction_service_charge_rm, - - - deleted_at, - created_at, - updated_at - - FROM transaction_costs -), - - --- FINAL -final__fct_exchange__transaction_costs AS ( - SELECT * FROM transaction_costs_lists -) - -SELECT * FROM final__fct_exchange__transaction_costs \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql deleted file mode 100644 index 3bd04f1..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__transaction_log_costs.sql +++ /dev/null @@ -1,61 +0,0 @@ --- IMPORTS -WITH transaction_log_costs AS ( - SELECT * FROM {{ ref('stg_exchange__transaction_log_costs') }} -), - - --- LOGICS -transaction_log_costs_lists AS ( - SELECT - id, - transaction_cost_id, - transaction_order_id, - supplier_company_id, - bank_id, - buyer_company_id, - status, - status_name, - transaction_type, - transaction_type_name, - payment_method, - payment_method_name, - base_currency_id, - base_currency_name, - quote_currency_id, - quote_currency_name, - base_to_quote_currency_exchange_rate, - - base_value, - quote_value, - -- use for future if base currency id other from '1' - IFF(base_currency_id = '1', - base_value, - 99999999) AS transaction_value_rm, - - base_tax AS transaction_base_tax, - -- use for future if base currency id other from '1' - IFF(base_currency_id = '1', - base_tax, - 99999999) AS transaction_tax_rm, - - base_service_charge, - -- use for future if base currency id other from '1' - IFF(base_currency_id = '1', - base_service_charge, - 99999999) AS transaction_service_charge_rm, - - - deleted_at, - created_at, - updated_at - - FROM transaction_log_costs -), - - --- FINAL -final__fct_exchange__transaction_log_costs AS ( - SELECT * FROM transaction_log_costs_lists -) - -SELECT * FROM final__fct_exchange__transaction_log_costs \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql deleted file mode 100644 index 47a0438..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__transaction_log_orders.sql +++ /dev/null @@ -1,116 +0,0 @@ --- IMPORTS -WITH transaction_log_orders AS ( - SELECT * FROM {{ ref('int_exchange__transaction_log_orders_get_user_ids') }} -), - - --- LOGICS -first_time_user_transaction_order_id AS ( - SELECT - transaction_order_id, - user_id, - created_at, - row_number() OVER - (PARTITION BY transaction_log_orders.user_id - ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking - - FROM - transaction_log_orders - - GROUP BY - transaction_order_id, - user_id, - created_at - - QUALIFY - cummulative_count_booking = 1 -), - - -first_time_company_transaction_order_id AS ( - SELECT - transaction_order_id, - company_id, - created_at, - row_number() OVER - (PARTITION BY transaction_log_orders.company_id - ORDER BY transaction_log_orders.created_at) AS cummulative_count_booking - - FROM - transaction_log_orders - - GROUP BY - transaction_order_id, - company_id, - created_at - - QUALIFY - cummulative_count_booking = 1 -), - - -transaction_log_orders_lists AS ( - SELECT - transaction_log_orders.id, - transaction_log_orders.transaction_order_id, - transaction_log_orders.booking_id, - transaction_log_orders.seller_company_id, - transaction_log_orders.bank_id, - transaction_log_orders.company_id, - transaction_log_orders.user_id, - transaction_log_orders.status, - transaction_log_orders.status_name, - transaction_log_orders.transaction_type, - transaction_log_orders.transaction_type_name, - transaction_log_orders.payment_method, - transaction_log_orders.payment_method_name, - transaction_log_orders.base_currency_id, - transaction_log_orders.base_currency_name, - transaction_log_orders.quote_currency_id, - transaction_log_orders.quote_currency_name, - transaction_log_orders.base_to_quote_currency_exchange_rate, - - transaction_log_orders.base_value, - transaction_log_orders.quote_value, - -- use for future if base currency id other from '1' - IFF(transaction_log_orders.base_currency_id = '1', - transaction_log_orders.base_value, - 99999999) AS transaction_value_rm, - - transaction_log_orders.base_tax AS transaction_base_tax, - -- use for future if base currency id other from '1' - IFF(transaction_log_orders.base_currency_id = '1', - transaction_log_orders.base_tax, - 99999999) AS transaction_tax_rm, - - base_service_charge, - -- use for future if base currency id other from '1' - IFF(transaction_log_orders.base_currency_id = '1', - transaction_log_orders.base_service_charge, - 99999999) AS transaction_service_charge_rm, - - - transaction_log_orders.deleted_at, - transaction_log_orders.created_at, - transaction_log_orders.updated_at, - - IFF(first_time_user_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_user, - IFF(first_time_company_transaction_order_id.transaction_order_id IS NOT NULL, 1, 0) AS is_first_time_company - - - FROM transaction_log_orders - - LEFT JOIN first_time_user_transaction_order_id - ON (transaction_log_orders.transaction_order_id = first_time_user_transaction_order_id.transaction_order_id) - - LEFT JOIN first_time_company_transaction_order_id - ON (transaction_log_orders.transaction_order_id = first_time_company_transaction_order_id.transaction_order_id) -), - - --- FINAL -final__fct_exchange__transaction_log_orders AS ( - SELECT * FROM transaction_log_orders_lists -) - -SELECT * FROM final__fct_exchange__transaction_log_orders \ No newline at end of file diff --git a/models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql b/models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql deleted file mode 100644 index 80b8521..0000000 --- a/models/exchange/marts/warehouse/fct_exchange__transaction_orders.sql +++ /dev/null @@ -1,92 +0,0 @@ --- IMPORTS -WITH transaction_orders AS ( - SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} -), - - --- LOGICS -transaction_orders_lists AS ( - SELECT - transaction_orders.id, - transaction_orders.booking_id, - transaction_orders.seller_company_id, - transaction_orders.bank_id, - transaction_orders.user_id, - transaction_orders.company_id, - transaction_orders.status, - transaction_orders.status_name, - transaction_orders.transaction_type, - transaction_orders.transaction_type_name, - transaction_orders.payment_method, - transaction_orders.payment_method_name, - transaction_orders.base_currency_id, - transaction_orders.base_currency_name, - transaction_orders.quote_currency_id, - transaction_orders.quote_currency_name, - transaction_orders.base_to_quote_currency_exchange_rate, - - transaction_orders.base_value, - transaction_orders.quote_value, - -- use for future if base currency id other from '1' - IFF(transaction_orders.base_currency_id = '1', - transaction_orders.base_value, - 99999999) AS transaction_value_rm, - - transaction_orders.base_tax AS transaction_base_tax, - -- use for future if base currency id other from '1' - IFF(transaction_orders.base_currency_id = '1', - transaction_orders.base_tax, - 99999999) AS transaction_tax_rm, - - transaction_orders.base_service_charge, - -- use for future if base currency id other from '1' - IFF(transaction_orders.base_currency_id = '1', - transaction_orders.base_service_charge, - 99999999) AS transaction_service_charge_rm, - - - transaction_orders.deleted_at, - transaction_orders.created_at, - transaction_orders.updated_at, - - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.company_id ORDER BY transaction_orders.created_at) = 1 THEN 1 - ELSE 0 - END AS is_first_time_company, - - CASE - WHEN transaction_orders.status = '3' - AND - MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL)) - OVER (PARTITION BY transaction_orders.company_id) - = transaction_orders.created_at - THEN 1 - ELSE 0 - END AS is_first_time_company_completed, - - - CASE - WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.user_id ORDER BY transaction_orders.created_at) = 1 THEN 1 - ELSE 0 - END AS is_first_time_user, - - CASE - WHEN transaction_orders.status = '3' - AND - MIN(IFF(transaction_orders.status = '3', transaction_orders.created_at, NULL)) - OVER (PARTITION BY transaction_orders.user_id) - = transaction_orders.created_at - THEN 1 - ELSE 0 - END AS is_first_time_user_completed - - FROM transaction_orders -), - - --- FINAL -final__fct_exchange__transaction_orders AS ( - SELECT * FROM transaction_orders_lists -) - -SELECT * FROM final__fct_exchange__transaction_orders \ No newline at end of file diff --git a/models/exchange/marts/warehouse/test_exchange__transaction_orders.yml b/models/exchange/marts/warehouse/test_exchange__transaction_orders.yml deleted file mode 100644 index e69de29..0000000 diff --git a/models/exchange/marts/warehouse/test_exchange__users.yml b/models/exchange/marts/warehouse/test_exchange__users.yml deleted file mode 100644 index e69de29..0000000 diff --git a/models/exchange/source/src_activity_log__exchange.yml b/models/exchange/source/src_activity_log__exchange.yml deleted file mode 100644 index 2d2e5ca..0000000 --- a/models/exchange/source/src_activity_log__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: activity_log - description: Raw `activity_log` table from Snowflake - columns: - - name: id - description: Primary key for 'activity_log' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_addresses__exchange.yml b/models/exchange/source/src_addresses__exchange.yml deleted file mode 100644 index a92717e..0000000 --- a/models/exchange/source/src_addresses__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: addresses - description: Raw `addresses` table from Snowflake - columns: - - name: id - description: Primary key for 'addresses' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_announcement_segment__exchange.yml b/models/exchange/source/src_announcement_segment__exchange.yml deleted file mode 100644 index 82e4e04..0000000 --- a/models/exchange/source/src_announcement_segment__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: announcement_segment - description: Raw `announcement_segment` table from Snowflake - columns: - - name: announcement_id - description: Primary key for 'announcement_segment' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_announcements__exchange.yml b/models/exchange/source/src_announcements__exchange.yml deleted file mode 100644 index af6824a..0000000 --- a/models/exchange/source/src_announcements__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: announcements - description: Raw `announcements` table from Snowflake - columns: - - name: id - description: Primary key for 'announcements' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_bank_logs__exchange.yml b/models/exchange/source/src_bank_logs__exchange.yml deleted file mode 100644 index d0d9f03..0000000 --- a/models/exchange/source/src_bank_logs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: bank_logs - description: Raw `bank_logs` table from Snowflake - columns: - - name: id - description: Primary key for 'bank_logs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_banks__exchange.yml b/models/exchange/source/src_banks__exchange.yml deleted file mode 100644 index 94e3888..0000000 --- a/models/exchange/source/src_banks__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: banks - description: Raw `banks` table from Snowflake - columns: - - name: id - description: Primary key for 'banks' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_booking_logs__exchange.yml b/models/exchange/source/src_booking_logs__exchange.yml deleted file mode 100644 index 09c48ad..0000000 --- a/models/exchange/source/src_booking_logs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: booking_logs - description: Raw `bookings` table from Snowflake - columns: - - name: id - description: Primary key for 'booking_logs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_bookings__exchange.yml b/models/exchange/source/src_bookings__exchange.yml deleted file mode 100644 index c53015c..0000000 --- a/models/exchange/source/src_bookings__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: bookings - description: Raw `bookings` table from Snowflake - columns: - - name: id - description: Primary key for 'bookings' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_companies__exchange.yml b/models/exchange/source/src_companies__exchange.yml deleted file mode 100644 index e269b49..0000000 --- a/models/exchange/source/src_companies__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: companies - description: Raw `companies` table from Snowflake - columns: - - name: id - description: Primary key for 'companies' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_contacts__exchange.yml b/models/exchange/source/src_contacts__exchange.yml deleted file mode 100644 index 1042e39..0000000 --- a/models/exchange/source/src_contacts__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: contacts - description: Raw `contacts` table from Snowflake - columns: - - name: id - description: Primary key for 'contacts' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_countries__exchange.yml b/models/exchange/source/src_countries__exchange.yml deleted file mode 100644 index 12131cf..0000000 --- a/models/exchange/source/src_countries__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: countries - description: Raw `countries` table from Snowflake - columns: - - name: id - description: Primary key for 'countries' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currencies__exchange.yml b/models/exchange/source/src_currencies__exchange.yml deleted file mode 100644 index 7fe1842..0000000 --- a/models/exchange/source/src_currencies__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: currencies - description: Raw `currencies` table from Snowflake - columns: - - name: id - description: Primary key for 'currencies' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currency_logs__exchange.yml b/models/exchange/source/src_currency_logs__exchange.yml deleted file mode 100644 index 21ecaad..0000000 --- a/models/exchange/source/src_currency_logs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: currency_logs - description: Raw `currency_logs` table from Snowflake - columns: - - name: id - description: Primary key for 'currency_logs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currency_rate_logs.yml b/models/exchange/source/src_currency_rate_logs.yml deleted file mode 100644 index 3a25613..0000000 --- a/models/exchange/source/src_currency_rate_logs.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: currency_rate_logs - description: Raw `currency_rate_logs` table from Snowflake - columns: - - name: id - description: Primary key for 'currency_rate_logs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_currency_rates__exchange.yml b/models/exchange/source/src_currency_rates__exchange.yml deleted file mode 100644 index 82ce730..0000000 --- a/models/exchange/source/src_currency_rates__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: currency_rates - description: Raw `currency_rates` table from Snowflake - columns: - - name: id - description: Primary key for 'currency_rates' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_districts__exchange.yml b/models/exchange/source/src_districts__exchange.yml deleted file mode 100644 index 7b7754f..0000000 --- a/models/exchange/source/src_districts__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: districts - description: Raw `districts` table from Snowflake - columns: - - name: id - description: Primary key for 'districts' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_documents__exchange.yml b/models/exchange/source/src_documents__exchange.yml deleted file mode 100644 index b8fbfc3..0000000 --- a/models/exchange/source/src_documents__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: documents - description: Raw `documents` table from Snowflake - columns: - - name: id - description: Primary key for 'documents' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_employees__exchange.yml b/models/exchange/source/src_employees__exchange.yml deleted file mode 100644 index 05355a8..0000000 --- a/models/exchange/source/src_employees__exchange.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: employees - description: Raw `employees` table from Snowflake - columns: - - name: company_id - description: Primary key for 'employees' - - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_failed_jobs__exchange.yml b/models/exchange/source/src_failed_jobs__exchange.yml deleted file mode 100644 index 0552305..0000000 --- a/models/exchange/source/src_failed_jobs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: failed_jobs - description: Raw `failed_jobs` table from Snowflake - columns: - - name: id - description: Primary key for 'failed_jobs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_files__exchange.yml b/models/exchange/source/src_files__exchange.yml deleted file mode 100644 index 20588e9..0000000 --- a/models/exchange/source/src_files__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: files - description: Raw `files` table from Snowflake - columns: - - name: id - description: Primary key for 'files' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_group_transactions__exchange.yml b/models/exchange/source/src_group_transactions__exchange.yml deleted file mode 100644 index 8585e51..0000000 --- a/models/exchange/source/src_group_transactions__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: group_transactions - description: Raw `group_transactions` table from Snowflake - columns: - - name: id - description: Primary key for 'group_transactions' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_groups__exchange.yml b/models/exchange/source/src_groups__exchange.yml deleted file mode 100644 index 9e398b2..0000000 --- a/models/exchange/source/src_groups__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: groups - description: Raw `groups` table from Snowflake - columns: - - name: id - description: Primary key for 'groups' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_jobs__exchange.yml b/models/exchange/source/src_jobs__exchange.yml deleted file mode 100644 index 6dc16f9..0000000 --- a/models/exchange/source/src_jobs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: jobs - description: Raw `jobs` table from Snowflake - columns: - - name: id - description: Primary key for 'jobs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_migrations__exchange.yml b/models/exchange/source/src_migrations__exchange.yml deleted file mode 100644 index c9bc6c1..0000000 --- a/models/exchange/source/src_migrations__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: migrations - description: Raw `migrations` table from Snowflake - columns: - - name: id - description: Primary key for 'migrations' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_model_has_roles__exchange.yml b/models/exchange/source/src_model_has_roles__exchange.yml deleted file mode 100644 index 86b3736..0000000 --- a/models/exchange/source/src_model_has_roles__exchange.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: model_has_roles - description: Raw `model_has_roles` table from Snowflake - columns: - - name: role_id - description: Primary key for 'model_has_roles' - - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_notifications__exchange.yml b/models/exchange/source/src_notifications__exchange.yml deleted file mode 100644 index 856a463..0000000 --- a/models/exchange/source/src_notifications__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: notifications - description: Raw `notifications` table from Snowflake - columns: - - name: id - description: Primary key for 'notifications' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_password_resets__exchange.yml b/models/exchange/source/src_password_resets__exchange.yml deleted file mode 100644 index c1c3a4c..0000000 --- a/models/exchange/source/src_password_resets__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: password_resets - description: Raw `password_resets` table from Snowflake - columns: - - name: id - description: Primary key for 'password_resets' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_permissions__exchange.yml b/models/exchange/source/src_permissions__exchange.yml deleted file mode 100644 index 201fa3d..0000000 --- a/models/exchange/source/src_permissions__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: permissions - description: Raw `permissions` table from Snowflake - columns: - - name: id - description: Primary key for 'permissions' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_role_has_permissions__exchange.yml b/models/exchange/source/src_role_has_permissions__exchange.yml deleted file mode 100644 index a0b1827..0000000 --- a/models/exchange/source/src_role_has_permissions__exchange.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: role_has_permissions - description: Raw `role_has_permissions` table from Snowflake - columns: - - name: permission_id - description: Primary key for 'role_has_permissions' - - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_roles__exchange.yml b/models/exchange/source/src_roles__exchange.yml deleted file mode 100644 index b4f4b0c..0000000 --- a/models/exchange/source/src_roles__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: roles - description: Raw `roles` table from Snowflake - columns: - - name: id - description: Primary key for 'roles' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_segment_companies__exchange.yml b/models/exchange/source/src_segment_companies__exchange.yml deleted file mode 100644 index accbcf0..0000000 --- a/models/exchange/source/src_segment_companies__exchange.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: segment_companies - description: Raw `segment_companies` table from Snowflake - columns: - - name: segment_id - description: Primary key for 'segment_companies' - - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_segment_constants__exchange.yml b/models/exchange/source/src_segment_constants__exchange.yml deleted file mode 100644 index 9c2475e..0000000 --- a/models/exchange/source/src_segment_constants__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: segment_constants - description: Raw `segment_constants` table from Snowflake - columns: - - name: id - description: Primary key for 'segment_constants' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_segments__exchange.yml b/models/exchange/source/src_segments__exchange.yml deleted file mode 100644 index 7323579..0000000 --- a/models/exchange/source/src_segments__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: segments - description: Raw `segments` table from Snowflake - columns: - - name: id - description: Primary key for 'segments' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_service_types__exchange.yml b/models/exchange/source/src_service_types__exchange.yml deleted file mode 100644 index bbac54c..0000000 --- a/models/exchange/source/src_service_types__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: service_types - description: Raw `service_types` table from Snowflake - columns: - - name: id - description: Primary key for 'service_types' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_states__exchange.yml b/models/exchange/source/src_states__exchange.yml deleted file mode 100644 index 881b421..0000000 --- a/models/exchange/source/src_states__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: states - description: Raw `states` table from Snowflake - columns: - - name: id - description: Primary key for 'states' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_transaction_detail__exchange.yml b/models/exchange/source/src_transaction_detail__exchange.yml deleted file mode 100644 index 8bdfe05..0000000 --- a/models/exchange/source/src_transaction_detail__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: transaction_detail - description: Raw `transaction_detail` table from Snowflake - columns: - - name: id - description: Primary key for 'transaction_detail' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_transaction_logs__exchange.yml b/models/exchange/source/src_transaction_logs__exchange.yml deleted file mode 100644 index 1802834..0000000 --- a/models/exchange/source/src_transaction_logs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: transaction_logs - description: Raw `transaction_logs` table from Snowflake - columns: - - name: id - description: Primary key for 'transaction_logs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_transactions__exchange.yml b/models/exchange/source/src_transactions__exchange.yml deleted file mode 100644 index 4ae35ef..0000000 --- a/models/exchange/source/src_transactions__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: transactions - description: Raw `transactions` table from Snowflake - columns: - - name: id - description: Primary key for 'transactions' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_user_email_verifications__exchange.yml b/models/exchange/source/src_user_email_verifications__exchange.yml deleted file mode 100644 index 7b6ed93..0000000 --- a/models/exchange/source/src_user_email_verifications__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: user_email_verifications - description: Raw `user_email_verifications` table from Snowflake - columns: - - name: id - description: Primary key for 'user_email_verifications' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_users__exchange.yml b/models/exchange/source/src_users__exchange.yml deleted file mode 100644 index 7b31631..0000000 --- a/models/exchange/source/src_users__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: users - description: Raw `users` table from Snowflake - columns: - - name: id - description: Primary key for 'users' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_wallet_logs__exchange.yml b/models/exchange/source/src_wallet_logs__exchange.yml deleted file mode 100644 index c4da2bf..0000000 --- a/models/exchange/source/src_wallet_logs__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: wallet_logs - description: Raw `wallet_logs` table from Snowflake - columns: - - name: id - description: Primary key for 'wallet_logs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/source/src_wallets__exchange.yml b/models/exchange/source/src_wallets__exchange.yml deleted file mode 100644 index 62094bc..0000000 --- a/models/exchange/source/src_wallets__exchange.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_exchange_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: EXCHANGE_MYSQL - - tables: - - name: wallets - description: Raw `wallets` table from Snowflake - columns: - - name: id - description: Primary key for 'wallets' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql b/models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql deleted file mode 100644 index 3e83c4a..0000000 --- a/models/exchange/staging/activity_logs/stg_exchange__activity_logs.sql +++ /dev/null @@ -1,46 +0,0 @@ --- IMPORTS -WITH activity_logs AS ( - SELECT * FROM {{ ref('base_exchange__activity_logs') }} -), - - --- LOGIC -activity_logs_listout AS ( - - SELECT - activity_logs.id, - activity_logs.log_name, - activity_logs.description, - activity_logs.subject_id, - activity_logs.subject_type, - activity_logs.causer_id, - activity_logs.causer_type, - activity_logs.properties, - activity_logs.created_at, - activity_logs.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM activity_logs -), - - --- FINAL -final__stg_exchange__activity_logs AS ( - - SELECT - id, - log_name, - description, - subject_id, - subject_type, - causer_id, - causer_type, - properties, - created_at, - updated_at, - _dbt_ran_at - - FROM activity_logs_listout -) - -SELECT * FROM final__stg_exchange__activity_logs \ No newline at end of file diff --git a/models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql b/models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql deleted file mode 100644 index ca8ceea..0000000 --- a/models/exchange/staging/announcement_segments/stg_exchange__announcement_segments.sql +++ /dev/null @@ -1,44 +0,0 @@ --- IMPORTS -WITH announcement_segments AS ( - SELECT * FROM {{ ref('base_exchange__announcement_segments') }} -), - -segments AS ( - SELECT * FROM {{ ref('base_exchange__segments') }} -), - --- LOGIC -announcement_segments_join_segments AS ( - - SELECT - announcement_segments.announcement_id, - - announcement_segments.segment_id, - segments.name AS segment_name, - - announcement_segments.created_at, - announcement_segments.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM announcement_segments - - LEFT JOIN segments - ON (announcement_segments.segment_id = segments.id) -), - - --- FINAL -final__stg_exchange__announcement_segments AS ( - - SELECT - announcement_id, - segment_id, - segment_name, - created_at, - updated_at, - _dbt_ran_at - - FROM announcement_segments_join_segments -) - -SELECT * FROM final__stg_exchange__announcement_segments \ No newline at end of file diff --git a/models/exchange/staging/announcements/stg_exchange__announcements.sql b/models/exchange/staging/announcements/stg_exchange__announcements.sql deleted file mode 100644 index b2a3846..0000000 --- a/models/exchange/staging/announcements/stg_exchange__announcements.sql +++ /dev/null @@ -1,50 +0,0 @@ --- IMPORTS -WITH announcements AS ( - SELECT * FROM {{ ref('base_exchange__announcements') }} -), - - --- LOGIC -announcements_listout AS ( - - SELECT - id, - title, - description, - started_at, - ended_at, - is_all_day, - duration, - is_recurring, - recurrence_pattern, - deleted_at, - created_at, - updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM announcements -), - - --- FINAL -final__stg_exchange__announcements AS ( - - SELECT - id, - title, - description, - started_at, - ended_at, - is_all_day, - duration, - is_recurring, - recurrence_pattern, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM announcements_listout -) - -SELECT * FROM final__stg_exchange__announcements \ No newline at end of file diff --git a/models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql b/models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql deleted file mode 100644 index 0c50631..0000000 --- a/models/exchange/staging/bank_logs/stg_exchange__bank_logs.sql +++ /dev/null @@ -1,93 +0,0 @@ --- IMPORTS -WITH bank_logs AS ( - SELECT * FROM {{ ref('base_exchange__bank_logs') }} -), - -bank_types AS ( - SELECT * FROM {{ ref('seed_exchange__bank_types') }} - WHERE category = 'REPHRASE' -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - -countries AS ( - SELECT * FROM {{ ref('base_exchange__countries') }} -), - --- LOGIC -bank_logs_join_status_contries AS ( - - SELECT - bank_logs.id, - bank_logs.bank_id, - bank_logs.company_id, - bank_logs.reference, - bank_logs.bank_name, - bank_logs.holder_name, - bank_logs.account_no, - bank_logs.bank_branch, - bank_logs.swift_code, - bank_logs.cnaps_code, - - bank_logs.bank_type, - bank_types.name AS bank_type_name, - - bank_logs.is_default, - - bank_logs.status, - status.name AS status_name, - - bank_logs.country_id, - countries.name AS country_name, - - bank_logs.deleted_at, - bank_logs.created_at, - bank_logs.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM bank_logs - - LEFT JOIN bank_types - ON (bank_logs.bank_type = bank_types.id) - - LEFT JOIN status - ON (bank_logs.status = status.id) - - LEFT JOIN countries - ON (bank_logs.country_id = countries.id) -), - - --- FINAL -final__stg_bank_logs__exchange AS ( - - SELECT - id, - bank_id, - company_id, - reference, - bank_name, - holder_name, - account_no, - bank_branch, - swift_code, - cnaps_code, - bank_type, - bank_type_name, - is_default, - status, - status_name, - country_id, - country_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM bank_logs_join_status_contries -) - -SELECT * FROM final__stg_bank_logs__exchange \ No newline at end of file diff --git a/models/exchange/staging/banks/stg_exchange__banks.sql b/models/exchange/staging/banks/stg_exchange__banks.sql deleted file mode 100644 index 243ae3c..0000000 --- a/models/exchange/staging/banks/stg_exchange__banks.sql +++ /dev/null @@ -1,91 +0,0 @@ --- IMPORTS -WITH banks AS ( - SELECT * FROM {{ ref('base_exchange__banks') }} -), - -bank_types AS ( - SELECT * FROM {{ ref('seed_exchange__bank_types') }} - WHERE category = 'REPHRASE' -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - -countries AS ( - SELECT * FROM {{ ref('base_exchange__countries') }} -), - --- LOGIC -banks_join_status_contries AS ( - - SELECT - banks.id, - banks.company_id, - banks.reference, - banks.bank_name, - banks.holder_name, - banks.account_no, - banks.bank_branch, - banks.swift_code, - banks.cnaps_code, - - banks.bank_type, - bank_types.name AS bank_type_name, - - banks.is_default, - - banks.status, - status.name AS status_name, - - banks.country_id, - countries.name AS country_name, - - banks.deleted_at, - banks.created_at, - banks.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM banks - - LEFT JOIN bank_types - ON (banks.bank_type = bank_types.id) - - LEFT JOIN status - ON (banks.status = status.id) - - LEFT JOIN countries - ON (banks.country_id = countries.id) -), - - --- FINAL -final__stg_exchange__banks AS ( - - SELECT - id, - company_id, - reference, - bank_name, - holder_name, - account_no, - bank_branch, - swift_code, - cnaps_code, - bank_type, - bank_type_name, - is_default, - status, - status_name, - country_id, - country_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM banks_join_status_contries -) - -SELECT * FROM final__stg_exchange__banks \ No newline at end of file diff --git a/models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql b/models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql deleted file mode 100644 index a65d7ec..0000000 --- a/models/exchange/staging/booking_logs/stg_exchange__booking_logs.sql +++ /dev/null @@ -1,165 +0,0 @@ --- IMPORTS -WITH booking_logs AS ( - SELECT * FROM {{ ref('base_exchange__booking_logs') }} -), - -bookings AS ( - SELECT * FROM {{ ref('base_exchange__bookings') }} -), - -activity_logs AS ( - SELECT * FROM {{ ref('base_exchange__activity_logs') }} -), - -service_types AS ( - SELECT * FROM {{ ref('base_exchange__service_types') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -bookings_get_hash_ids AS ( - SELECT - MD5_NUMBER_LOWER64(CONCAT(id, created_at)) AS id, - id AS booking_id, - company_id, - marking_id, - service_type, - bank_id, - fix_value, - fix_currency_id, - quote_currency_id, - base_currency_id, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM bookings -), - - -get_booking_user_ids AS ( - - SELECT - subject_id AS booking_id, - causer_id AS user_id - FROM - activity_logs - WHERE - subject_type='App\\Models\\Booking' - AND - description='created' - -), - - -booking_logs_unions AS ( - SELECT * FROM bookings_get_hash_ids - UNION - SELECT * FROM booking_logs -), - - -booking_logs_unions_join_service_types_currencies_status_get_booking_user_ids AS ( - - SELECT - booking_logs_unions.id, - booking_logs_unions.booking_id, - booking_logs_unions.company_id, - - get_booking_user_ids.user_id, - - booking_logs_unions.marking_id, - - booking_logs_unions.service_type, - service_types.name AS service_type_name, - - booking_logs_unions.bank_id, - booking_logs_unions.fix_value, - - booking_logs_unions.fix_currency_id, - fix_currencies.name AS fix_currency_name, - fix_currencies.short_code AS fix_currency_short_code, - - booking_logs_unions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - - booking_logs_unions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - - booking_logs_unions.status, - status.name AS status_name, - - booking_logs_unions.deleted_at, - booking_logs_unions.created_at, - booking_logs_unions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM booking_logs_unions - - LEFT JOIN get_booking_user_ids - ON (booking_logs_unions.booking_id = get_booking_user_ids.booking_id) - - LEFT JOIN service_types - ON (booking_logs_unions.service_type = service_types.id) - - LEFT JOIN currencies AS fix_currencies - ON (booking_logs_unions.fix_currency_id = fix_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (booking_logs_unions.quote_currency_id = quote_currencies.id) - - LEFT JOIN currencies AS base_currencies - ON (booking_logs_unions.base_currency_id = base_currencies.id) - - LEFT JOIN status - ON (booking_logs_unions.status = status.id) -), - - --- FINAL -final__stg_exchange__booking_logs AS ( - - SELECT - id, - booking_id, - company_id, - user_id, - marking_id, - service_type, - service_type_name, - bank_id, - fix_value, - fix_currency_id, - fix_currency_name, - fix_currency_short_code, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - base_currency_id, - base_currency_name, - base_currency_short_code, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM booking_logs_unions_join_service_types_currencies_status_get_booking_user_ids - -) - -SELECT * FROM final__stg_exchange__booking_logs \ No newline at end of file diff --git a/models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md b/models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md deleted file mode 100644 index 9ccfc2f..0000000 --- a/models/exchange/staging/bookings/doc_generic_test_stg_bookings__exchange.md +++ /dev/null @@ -1,15 +0,0 @@ -{% docs generic_test_stg_bookings_status__exchange %} -Using seeds csv to transform - -| STATUS | STATUS_DISPLAYNAME | definition | -|----------------|----------------------------|--------------------------------------------------| -| 0 | PENDING_SUBMISSION | Order placed, not yet shipped | -| 1 | PENDING_VERIFICATION | Order has been shipped, not yet been delivered | -| 2 | APPROVED | Order has been received by customers | -| 3 | COMPLETED | Order has been received by customers | -| 4 | REJECTED | Order has been received by customers | -| 5 | SUSPENDED | Order has been received by customers | -| 6 | EXPIRED | Order has been received by customers | -| 7 | REFUNDED | Order has been received by customers | -| ELSE | | Refer to seeds for define the status | -{% enddocs %} \ No newline at end of file diff --git a/models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml b/models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml deleted file mode 100644 index 347d3fa..0000000 --- a/models/exchange/staging/bookings/generic_test_stg_bookings__exchange.yml +++ /dev/null @@ -1,100 +0,0 @@ -# version: 2 - -# models: -# - name: stg_exchange__bookings -# columns: -# - name: ID -# description: PK ID -# tests: -# - unique -# - not_null - -# - name: COMPANY_ID -# description: FK, for source `src_companies` -# tests: -# - not_null -# - relationships: -# to: ref('stg_companies') -# field: COMPANY_ID - -# - name: HUMAN_READABLE_MARKING_ID -# description: Auto generate unique key -# tests: -# - unique -# - not_null - -# - name: SERVICE_ID -# description: FK, for source `src_service_types` -# tests: -# - not_null -# - relationships: -# to: ref('stg_exchange__service_types') -# field: SERVICE_TYPE_ID - -# - name: BANK_ID -# description: FK, for source `src_banks` -# tests: -# - not_null -# - relationships: -# to: ref('stg_banks') -# field: BANK_ID - -# - name: FIX_AMOUNT -# description: |- -# Amount base on 'FIX_CURRENCY_ID' -# Contain singular test -# Purpose of singular test:- It must only contain positive number & greater than 0 -# Location singular test: tests/exchange/staging/test_stg_bookings_fixAmount_positive.sql -# tests: -# - not_null - -# - name: FIX_CURRENCY_ID -# description: FK, for source `src_currencies` -# tests: -# - not_null -# - relationships: -# to: ref('stg_currencies') -# field: CURRENCY_ID - -# - name: QUOTE_CURRENCY_ID -# description: FK, for source `src_currencies`, currency that user want change into -# tests: -# - not_null -# - relationships: -# to: ref('stg_currencies') -# field: CURRENCY_ID - -# - name: BASE_CURRENCY_ID -# description: FK, for source `src_currencies`, currency that user use to change to other currency -# tests: -# - not_null -# - relationships: -# to: ref('stg_currencies') -# field: CURRENCY_ID - -# - name: STATUS -# description: "{{ doc('generic_test_stg_bookings_status__exchange') }}" -# tests: -# - not_null -# - accepted_values: -# values: [0, 1, 2, 3, 4, 5, 6, 7] - -# - name: DELETED_AT -# description: Database row deleted datetime, if it occur, means row has been deleted, can be NULL -# tests: - -# - name: CREATED_AT -# description: Database row created datetime -# tests: -# - not_null - -# - name: UPDATED_AT -# description: Database row updated datetime, can be same as 'CREATED_AT' -# tests: -# - not_null - -# - name: _SOURCE_LOADED_AT -# description: Python extract datetime - -# - name: _DBT_RUN_AT -# description: Dbt run query datetime \ No newline at end of file diff --git a/models/exchange/staging/bookings/stg_exchange__bookings.sql b/models/exchange/staging/bookings/stg_exchange__bookings.sql deleted file mode 100644 index f4decd9..0000000 --- a/models/exchange/staging/bookings/stg_exchange__bookings.sql +++ /dev/null @@ -1,128 +0,0 @@ --- IMPORTS -WITH bookings AS ( - SELECT * FROM {{ ref('base_exchange__bookings') }} -), - -activity_logs AS ( - SELECT * FROM {{ ref('base_exchange__activity_logs') }} -), - -service_types AS ( - SELECT * FROM {{ ref('base_exchange__service_types') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -booking_get_user_ids AS ( - - SELECT - subject_id AS booking_id, - causer_id AS user_id - FROM - activity_logs - WHERE - subject_type='App\\Models\\Booking' - AND - description='created' - -), - -bookings_join_service_types_currencies_status_booking_get_user_ids AS ( - - SELECT - bookings.id, - bookings.company_id, - - booking_get_user_ids.user_id, - - bookings.marking_id, - - bookings.service_type, - service_types.name AS service_type_name, - - bookings.bank_id, - bookings.fix_value, - - bookings.fix_currency_id, - fix_currencies.name AS fix_currency_name, - fix_currencies.short_code AS fix_currency_short_code, - - bookings.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - - bookings.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - - bookings.status, - status.name AS status_name, - - bookings.deleted_at AS booking_deleted_datetime, - bookings.created_at AS booking_created_datetime, - bookings.updated_at AS booking_updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM bookings - - LEFT JOIN booking_get_user_ids - ON (bookings.id = booking_get_user_ids.booking_id) - - LEFT JOIN service_types - ON (bookings.service_type = service_types.id) - - LEFT JOIN currencies AS fix_currencies - ON (bookings.fix_currency_id = fix_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (bookings.quote_currency_id = quote_currencies.id) - - LEFT JOIN currencies AS base_currencies - ON (bookings.base_currency_id = base_currencies.id) - - LEFT JOIN status - ON (bookings.status = status.id) -), - --- FINAL -final__stg_exchange__bookings AS ( - - SELECT - id, - company_id, - user_id, - marking_id, - service_type, - service_type_name, - bank_id, - fix_value, - fix_currency_id, - fix_currency_name, - fix_currency_short_code, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - base_currency_id, - base_currency_name, - base_currency_short_code, - status, - status_name, - booking_deleted_datetime, - booking_created_datetime, - booking_updated_datetime, - _dbt_ran_at - - FROM bookings_join_service_types_currencies_status_booking_get_user_ids - -) - -SELECT * FROM final__stg_exchange__bookings \ No newline at end of file diff --git a/models/exchange/staging/companies/stg_exchange__companies.sql b/models/exchange/staging/companies/stg_exchange__companies.sql deleted file mode 100644 index 4eaaafe..0000000 --- a/models/exchange/staging/companies/stg_exchange__companies.sql +++ /dev/null @@ -1,80 +0,0 @@ --- IMPORTS -WITH companies AS ( - SELECT * FROM {{ ref('base_exchange__companies') }} -), - -company_types AS ( - SELECT * FROM {{ ref('seed_exchange__company_types') }} - WHERE category = '2_AS_CORPORATE' -), - -business_types AS ( - SELECT * FROM {{ ref('seed_exchange__business_types') }} - WHERE category = 'DEFAULT' -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -companies_rename_join_company_types_business_types_status AS ( - - SELECT - companies.id, - companies.name, - companies.marking_id, - companies.account_software_id, - - companies.company_type, - company_types.name AS company_type_name, - - companies.business_type, - business_types.name AS business_type_name, - - companies.status, - status.name AS status_name, - - companies.deleted_at, - companies.created_at, - companies.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM companies - - LEFT JOIN company_types - ON (companies.company_type = company_types.id) - - LEFT JOIN business_types - ON (companies.business_type = business_types.id) - - LEFT JOIN status - ON (companies.status = status.id) -), - --- FINAL -final__stg_companies__exchange AS ( - - SELECT - id, - name, - marking_id, - account_software_id, - company_type, - company_type_name, - business_type, - business_type_name, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM companies_rename_join_company_types_business_types_status - -) - -SELECT * FROM final__stg_companies__exchange \ No newline at end of file diff --git a/models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql b/models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql deleted file mode 100644 index 96fd03f..0000000 --- a/models/exchange/staging/companies_and_segments_relations/stg_exchange__companies_and_segments_relations.sql +++ /dev/null @@ -1,43 +0,0 @@ --- IMPORTS -WITH segment_companies AS ( - SELECT * FROM {{ ref('base_exchange__segment_companies') }} -), - - --- LOGIC -segment_companies_list AS ( - - SELECT - segment_companies.concat_id, - segment_companies.segment_id, - segment_companies.company_id, - segment_companies.deleted_at, - segment_companies.created_at, - segment_companies.updated_at, - segment_companies.dbt_valid_from, - segment_companies.dbt_valid_to, - current_timestamp() AS _dbt_ran_at - - FROM segment_companies -), - - --- FINAL -final__stg_exchange__segment_companies AS ( - - SELECT - concat_id, - segment_id, - company_id, - deleted_at, - created_at, - updated_at, - dbt_valid_from, - dbt_valid_to, - _dbt_ran_at - - FROM segment_companies_list - -) - -SELECT * FROM final__stg_exchange__segment_companies \ No newline at end of file diff --git a/models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql b/models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql deleted file mode 100644 index 857cece..0000000 --- a/models/exchange/staging/companies_and_users_relations/stg_exchange__companies_and_users_relations.sql +++ /dev/null @@ -1,46 +0,0 @@ --- IMPORTS -WITH employees AS ( - SELECT * FROM {{ ref('base_exchange__employees') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -employees_join_status AS ( - - SELECT - employees.company_id, - employees.user_id, - - employees.status, - status.name AS status_name, - - current_timestamp() AS _dbt_ran_at - - FROM employees - - LEFT JOIN status - ON (employees.status = status.id) - -), - - --- FINAL -final__stg_exchange__companies_and_users_relations AS ( - - SELECT - company_id, - user_id, - status, - status_name, - _dbt_ran_at - - FROM employees_join_status - -) - -SELECT * FROM final__stg_exchange__companies_and_users_relations \ No newline at end of file diff --git a/models/exchange/staging/contacts/stg_exchange__contacts.sql b/models/exchange/staging/contacts/stg_exchange__contacts.sql deleted file mode 100644 index 5a679c6..0000000 --- a/models/exchange/staging/contacts/stg_exchange__contacts.sql +++ /dev/null @@ -1,59 +0,0 @@ --- IMPORTS -WITH contacts AS ( - SELECT * FROM {{ ref('base_exchange__contacts') }} -), - -countries AS ( - SELECT * FROM {{ ref('base_exchange__countries') }} -), - - --- LOGIC -companies_join_countries_status AS ( - - SELECT - contacts.id, - contacts.company_id, - - contacts.country_id, - countries.name AS country_name, - - contacts.reference, - contacts.phone, - contacts.email, - contacts.wechat_id, - contacts.deleted_at, - contacts.created_at, - contacts.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM contacts - - LEFT JOIN countries - ON (contacts.country_id = countries.id) - -), - - --- FINAL -final__stg_exchange__contacts AS ( - - SELECT - id, - company_id, - country_id, - country_name, - reference, - phone, - email, - wechat_id, - deleted_at AS contact_deleted_datetime, - created_at AS contact_created_datetime, - updated_at AS contact_updated_datetime, - _dbt_ran_at - - FROM companies_join_countries_status - -) - -SELECT * FROM final__stg_exchange__contacts \ No newline at end of file diff --git a/models/exchange/staging/countries/stg_exchange__countries.sql b/models/exchange/staging/countries/stg_exchange__countries.sql deleted file mode 100644 index 72318a5..0000000 --- a/models/exchange/staging/countries/stg_exchange__countries.sql +++ /dev/null @@ -1,41 +0,0 @@ --- IMPORTS -WITH countries AS ( - SELECT * FROM {{ ref('base_exchange__countries') }} -), - --- LOGIC -countries_rename AS ( - - SELECT - countries.id, - countries.name, - countries.short_code, - countries.phone_code, - countries.deleted_at, - countries.created_at, - countries.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM countries - -), - - --- FINAL -final__stg_countries__exchange AS ( - - SELECT - id, - name, - short_code, - phone_code, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM countries_rename - -) - -SELECT * FROM final__stg_countries__exchange \ No newline at end of file diff --git a/models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql b/models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql deleted file mode 100644 index 8872783..0000000 --- a/models/exchange/staging/currency_logs/stg_exchange__currency_logs.sql +++ /dev/null @@ -1,55 +0,0 @@ --- IMPORTS -WITH currency_logs AS ( - SELECT * FROM {{ ref('base_exchange__currency_logs') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - - - --- LOGIC -currency_logs_join_currencies AS ( - - SELECT - currency_logs.id, - - currency_logs.currency_id, - currencies.name AS currency_name, - currencies.short_code AS currency_short_code, - currencies.symbol AS currency_symbol, - - currency_logs.created_by, - currencies.deleted_at, - currencies.created_at, - currencies.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM currency_logs - - LEFT JOIN currencies - ON (currency_logs.currency_id = currencies.id) -), - - --- FINAL -final__stg_exchange__currency_logs AS ( - - SELECT - id, - currency_id, - currency_name, - currency_short_code, - currency_symbol, - created_by, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM currency_logs_join_currencies - -) - -SELECT * FROM final__stg_exchange__currency_logs \ No newline at end of file diff --git a/models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql b/models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql deleted file mode 100644 index e4b0748..0000000 --- a/models/exchange/staging/current_currency_rates/stg_exchange__current_currency_rates.sql +++ /dev/null @@ -1,79 +0,0 @@ --- IMPORTS -WITH currency_rates AS ( - SELECT * FROM {{ ref('base_exchange__currency_rates') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} - WHERE category = 'DEFAULT' -), - -service_types AS ( - SELECT * FROM {{ ref('base_exchange__service_types') }} -), - --- LOGIC -currency_rates_join_currencies_payment_methods_service_types AS ( - - SELECT - currency_rates.id, - - currency_rates.quote_currency_id, - currencies.name AS quote_currency_name, - currencies.short_code AS quote_currency_short_code, - currencies.symbol AS quote_currency_symbol, - - currency_rates.base_to_quote_currency_exchange_rate, - - currency_rates.payment_method, - payment_methods.name AS payment_method_name, - - currency_rates.service_type, - service_types.name AS service_type_name, - - currency_rates.deleted_at, - currency_rates.created_at, - currency_rates.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM currency_rates - - LEFT JOIN currencies - ON (currency_rates.quote_currency_id = currencies.id) - - LEFT JOIN payment_methods - ON (currency_rates.payment_method = payment_methods.id) - - LEFT JOIN service_types - ON (currency_rates.service_type = service_types.id) -), - - --- FINAL -final__stg_exchange__current_currency_rates AS ( - - SELECT - id, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - payment_method, - payment_method_name, - service_type, - service_type_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM currency_rates_join_currencies_payment_methods_service_types - -) - -SELECT * FROM final__stg_exchange__current_currency_rates \ No newline at end of file diff --git a/models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql b/models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql deleted file mode 100644 index 11c8c99..0000000 --- a/models/exchange/staging/current_wallets/stg_exchange__current_wallets.sql +++ /dev/null @@ -1,61 +0,0 @@ --- IMPORTS -WITH wallets AS ( - SELECT * FROM {{ ref('base_exchange__wallets') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - - --- LOGIC -wallets_join_currencies AS ( - - SELECT - wallets.id, - wallets.owner_type, - wallets.owner_id, - wallets.code, - - wallets.currency_id, - currencies.name AS currency_name, - currencies.short_code AS currency_short_code, - currencies.symbol AS currency_symbol, - - wallets.amount, - wallets.deleted_at, - wallets.created_at, - wallets.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM wallets - - LEFT JOIN currencies - ON (wallets.currency_id = currencies.id) - -), - - --- FINAL -final__stg_exchange__current_wallets AS ( - - SELECT - id, - owner_type, - owner_id, - code, - currency_id, - currency_name, - currency_short_code, - currency_symbol, - amount, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM wallets_join_currencies - -) - -SELECT * FROM final__stg_exchange__current_wallets \ No newline at end of file diff --git a/models/exchange/staging/districts/stg_exchange__districts.sql b/models/exchange/staging/districts/stg_exchange__districts.sql deleted file mode 100644 index bca54b5..0000000 --- a/models/exchange/staging/districts/stg_exchange__districts.sql +++ /dev/null @@ -1,39 +0,0 @@ --- IMPORTS -WITH districts AS ( - SELECT * FROM {{ ref('base_exchange__districts') }} -), - - --- LOGIC -districts_list AS ( - - SELECT - districts.id, - districts.country_id, - districts.state_id, - districts.name, - districts.postcode, - districts.status, - current_timestamp() AS _dbt_ran_at - - FROM districts -), - - --- FINAL -final__stg_exchange__districts AS ( - - SELECT - id, - country_id, - state_id, - name, - postcode, - status, - _dbt_ran_at - - FROM districts_list - -) - -SELECT * FROM districts \ No newline at end of file diff --git a/models/exchange/staging/documents/stg_exchange__documents.sql b/models/exchange/staging/documents/stg_exchange__documents.sql deleted file mode 100644 index b209b22..0000000 --- a/models/exchange/staging/documents/stg_exchange__documents.sql +++ /dev/null @@ -1,66 +0,0 @@ --- IMPORTS -WITH documents AS ( - SELECT * FROM {{ ref('base_exchange__documents') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -documents_join_status AS ( - - SELECT - documents.id, - documents.owner_type, - documents.owner_id, - documents.document_type, - documents.reference, - - documents.status, - status.name AS status_name, - - documents.issued_at, - documents.expired_at, - documents.approved_by, - documents.approved_at, - documents.deleted_at, - documents.created_at, - documents.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM documents - - LEFT JOIN status - ON (documents.status = status.id) - -), - - --- FINAL -final__stg_exchange__documents AS ( - - SELECT - id, - owner_type, - owner_id, - document_type, - reference, - status, - status_name, - issued_at, - expired_at, - approved_by, - approved_at, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM documents_join_status - -) - -SELECT * FROM final__stg_exchange__documents \ No newline at end of file diff --git a/models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql b/models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql deleted file mode 100644 index 1b57653..0000000 --- a/models/exchange/staging/failed_jobs/stg_exchange__failed_jobs.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH failed_jobs AS ( - SELECT * FROM {{ ref('base_exchange__failed_jobs') }} -), - - --- LOGIC -failed_jobs_rename AS ( - - SELECT - failed_jobs.id, - failed_jobs.connection, - failed_jobs.queue, - failed_jobs.payload, - failed_jobs.exception, - failed_jobs.failed_at, - current_timestamp() AS _dbt_ran_at - - FROM failed_jobs - -), - - --- FINAL -final__stg_exchange__failed_jobs AS ( - - SELECT - id, - connection, - queue, - payload, - exception, - failed_at, - _dbt_ran_at - - FROM failed_jobs_rename - -) - -SELECT * FROM final__stg_exchange__failed_jobs \ No newline at end of file diff --git a/models/exchange/staging/files/stg_exchange__files.sql b/models/exchange/staging/files/stg_exchange__files.sql deleted file mode 100644 index 3cf5a46..0000000 --- a/models/exchange/staging/files/stg_exchange__files.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH files AS ( - SELECT * FROM {{ ref('base_exchange__files') }} -), - - --- LOGIC -files_list AS ( - - SELECT - files.id, - files.document_id, - files.file, - files.file_type, - files.deleted_at, - files.created_at, - files.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM files - -), - - --- FINAL -final__stg_exchange__files AS ( - - SELECT - id, - document_id, - file, - file_type, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM files_list - -) - -SELECT * FROM final__stg_exchange__files \ No newline at end of file diff --git a/models/exchange/staging/group_costs/stg_exchange__group_costs.sql b/models/exchange/staging/group_costs/stg_exchange__group_costs.sql deleted file mode 100644 index e9dc262..0000000 --- a/models/exchange/staging/group_costs/stg_exchange__group_costs.sql +++ /dev/null @@ -1,93 +0,0 @@ --- IMPORTS -WITH groups AS ( - SELECT * FROM {{ ref('base_exchange__groups') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -groups_join_currencies_status AS ( - - SELECT - groups.id, - - groups.marking_id, - groups.issued_by, - groups.received_by, - groups.base_amount, - groups.quote_amount, - - groups.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - groups.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - groups.base_to_quote_currency_exchange_rate, - groups.base_tax, - groups.base_service_charge, - - groups.status, - status.name AS status_name, - - groups.created_at, - groups.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM groups - - LEFT JOIN currencies AS base_currencies - ON (groups.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (groups.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (groups.status = status.id) -), - - --- FINAL -final__stg_exchange__groups AS ( - - SELECT - id, - marking_id, - issued_by, - received_by, - base_amount, - quote_amount, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - status, - status_name, - created_at, - updated_at, - _dbt_ran_at - - FROM groups_join_currencies_status - -) - -SELECT * FROM final__stg_exchange__groups \ No newline at end of file diff --git a/models/exchange/staging/jobs/stg_exchange__jobs.sql b/models/exchange/staging/jobs/stg_exchange__jobs.sql deleted file mode 100644 index 3ef745d..0000000 --- a/models/exchange/staging/jobs/stg_exchange__jobs.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH jobs AS ( - SELECT * FROM {{ ref('base_exchange__jobs') }} -), - - --- LOGIC -jobs_list AS ( - - SELECT - jobs.id, - jobs.queue, - jobs.payload, - jobs.attempts, - reserved_at, - available_at, - created_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM jobs - -), - - --- FINAL -final__stg_exchange__jobs AS ( - - SELECT - id, - queue, - payload, - attempts, - reserved_at, - available_at, - created_at, - _dbt_ran_at - - FROM jobs_list - -) - -SELECT * FROM final__stg_exchange__jobs \ No newline at end of file diff --git a/models/exchange/staging/migrations/stg_exchange__migrations.sql b/models/exchange/staging/migrations/stg_exchange__migrations.sql deleted file mode 100644 index 94aaee1..0000000 --- a/models/exchange/staging/migrations/stg_exchange__migrations.sql +++ /dev/null @@ -1,34 +0,0 @@ --- IMPORTS -WITH migrations AS ( - SELECT * FROM {{ ref('base_exchange__migrations') }} -), - - --- LOGIC -migrations_lsit AS ( - - SELECT - migrations.id, - migrations.migration, - migrations.batch, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM migrations - -), - - --- FINAL -final__stg_exchange__migrations AS ( - - SELECT - id, - migration, - batch, - _dbt_ran_at - - FROM migrations_lsit - -) - -SELECT * FROM final__stg_exchange__migrations \ No newline at end of file diff --git a/models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql b/models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql deleted file mode 100644 index 7f03429..0000000 --- a/models/exchange/staging/model_has_roles/stg_exchange__model_has_roles.sql +++ /dev/null @@ -1,43 +0,0 @@ --- IMPORTS -WITH model_has_roles AS ( - SELECT * FROM {{ ref('base_exchange__model_has_roles') }} -), - -roles AS ( - SELECT * FROM {{ ref('base_exchange__roles') }} -), - --- LOGIC -migrations_join_roles AS ( - - SELECT - model_has_roles.role_id, - roles.name AS role_name, - - model_has_roles.model_type, - model_has_roles.model_id, - current_timestamp() AS _dbt_ran_at - - FROM model_has_roles - - LEFT JOIN roles - ON (model_has_roles.role_id = roles.id) - -), - - --- FINAL -final__stg_exchange__model_has_roles AS ( - - SELECT - role_id, - role_name, - model_type, - model_id, - _dbt_ran_at - - FROM migrations_join_roles - -) - -SELECT * FROM final__stg_exchange__model_has_roles \ No newline at end of file diff --git a/models/exchange/staging/notifications/stg_exchange__notifications.sql b/models/exchange/staging/notifications/stg_exchange__notifications.sql deleted file mode 100644 index 20009d6..0000000 --- a/models/exchange/staging/notifications/stg_exchange__notifications.sql +++ /dev/null @@ -1,66 +0,0 @@ --- ASk edmond to talk about this - --- IMPORTS -WITH notifications AS ( - SELECT * FROM {{ ref('base_exchange__notifications') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - --- LOGIC -notifications_join_status AS ( - - SELECT - notifications.id, - notifications.title, - notifications.description, - notifications.subject_type, - notifications.subject_id, - notifications.target_type, - notifications.target_id, - notifications.causer_type, - notifications.causer_id, - - notifications.status, - status.name AS status_name, - - notifications.deleted_at, - notifications.created_at, - notifications.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM notifications - - LEFT JOIN status - ON (notifications.status = status.id) -), - - --- FINAL -final__stg_exchange__notifications AS ( - - SELECT - id, - title, - description, - subject_type, - subject_id, - target_type, - target_id, - causer_type, - causer_id, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM notifications_join_status - -) - -SELECT * FROM final__stg_exchange__notifications \ No newline at end of file diff --git a/models/exchange/staging/password_resets/stg_exchange__password_resets.sql b/models/exchange/staging/password_resets/stg_exchange__password_resets.sql deleted file mode 100644 index 65997b6..0000000 --- a/models/exchange/staging/password_resets/stg_exchange__password_resets.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH password_resets AS ( - SELECT * FROM {{ ref('base_exchange__password_resets') }} -), - - --- LOGIC -password_resets_list AS ( - - SELECT - password_resets.id, - password_resets.user_id, - password_resets.token, - password_resets.is_expired, - password_resets.is_complete, - password_resets.created_at, - password_resets.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM password_resets - -), - - --- FINAL -final__stg_exchange__password_resets AS ( - - SELECT - id, - user_id, - token, - is_expired, - is_complete, - created_at, - updated_at, - _dbt_ran_at - - FROM password_resets_list - -) - -SELECT * FROM final__stg_exchange__password_resets \ No newline at end of file diff --git a/models/exchange/staging/permissions/stg_exchange__permissions.sql b/models/exchange/staging/permissions/stg_exchange__permissions.sql deleted file mode 100644 index 5a01d85..0000000 --- a/models/exchange/staging/permissions/stg_exchange__permissions.sql +++ /dev/null @@ -1,38 +0,0 @@ --- IMPORTS -WITH permissions AS ( - SELECT * FROM {{ ref('base_exchange__permissions') }} -), - - --- LOGIC -permissions_list AS ( - - SELECT - permissions.id, - permissions.name, - permissions.guard_name, - permissions.created_at, - permissions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM permissions - -), - - --- FINAL -final__stg_exchange__permissions AS ( - - SELECT - id, - name, - guard_name, - created_at, - updated_at, - _dbt_ran_at - - FROM permissions_list - -) - -SELECT * FROM final__stg_exchange__permissions \ No newline at end of file diff --git a/models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql b/models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql deleted file mode 100644 index 49975ff..0000000 --- a/models/exchange/staging/role_has_permissions/stg_exchange__role_has_permissions.sql +++ /dev/null @@ -1,32 +0,0 @@ --- IMPORTS -WITH role_has_permissions AS ( - SELECT * FROM {{ ref('base_exchange__role_has_permissions') }} -), - - --- LOGIC -role_has_permissions_list AS ( - - SELECT - role_has_permissions.permission_id, - role_has_permissions.role_id, - current_timestamp() AS _dbt_ran_at - - FROM role_has_permissions - -), - - --- FINAL -final__stg_exchange__role_has_permissions AS ( - - SELECT - permission_id, - role_id, - _dbt_ran_at - - FROM role_has_permissions_list - -) - -SELECT * FROM final__stg_exchange__role_has_permissions \ No newline at end of file diff --git a/models/exchange/staging/roles/stg_exchange__roles.sql b/models/exchange/staging/roles/stg_exchange__roles.sql deleted file mode 100644 index bd4d609..0000000 --- a/models/exchange/staging/roles/stg_exchange__roles.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH roles AS ( - SELECT * FROM {{ ref('base_exchange__roles') }} -), - - --- LOGIC -roles_list AS ( - - SELECT - roles.id, - roles.name, - roles.guard_name, - roles.type, - roles.created_at, - roles.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM roles - -), - - --- FINAL -final__stg_exchange__roles AS ( - - SELECT - id, - name, - guard_name, - type, - created_at, - updated_at, - _dbt_ran_at - - FROM roles_list - -) - -SELECT * FROM final__stg_exchange__roles \ No newline at end of file diff --git a/models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql b/models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql deleted file mode 100644 index 5393fd5..0000000 --- a/models/exchange/staging/segment_constants/stg_exchange__segment_constants.sql +++ /dev/null @@ -1,55 +0,0 @@ --- IMPORTS -WITH segment_constants AS ( - SELECT * FROM {{ ref('base_exchange__segment_constants') }} -), - -segments AS ( - SELECT * FROM {{ ref('base_exchange__segments') }} -), - --- LOGIC -segment_constants_join_segments AS ( - - SELECT - segment_constants.id, - - segment_constants.segment_id, - segments.name AS segment_name, - segments.type AS segment_type, - - segment_constants.name, - segment_constants.marking_id, - segment_constants.detail, - segment_constants.deleted_at, - segment_constants.created_at, - segment_constants.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM segment_constants - - LEFT JOIN segments - ON (segment_constants.segment_id = segments.id) -), - - --- FINAL -final__stg_exchange__segment_constants AS ( - - SELECT - id, - segment_id, - segment_name, - segment_type, - name, - marking_id, - detail, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM segment_constants_join_segments - -) - -SELECT * FROM final__stg_exchange__segment_constants \ No newline at end of file diff --git a/models/exchange/staging/segments/stg_exchange__segments.sql b/models/exchange/staging/segments/stg_exchange__segments.sql deleted file mode 100644 index a3ad8d9..0000000 --- a/models/exchange/staging/segments/stg_exchange__segments.sql +++ /dev/null @@ -1,53 +0,0 @@ --- IMPORTS -WITH segments AS ( - SELECT * FROM {{ ref('base_exchange__segments') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -segments_join_status AS ( - - SELECT - segments.id, - UPPER(segments.name) AS name, - segments.type, - - segments.status, - status.name AS status_name, - - segments.deleted_at, - segments.created_at, - segments.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM segments - - LEFT JOIN status - ON (segments.status = status.id) -), - - --- FINAL -final__stg_exchange__segments AS ( - - SELECT - id, - name, - type, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM segments_join_status - -) - -SELECT * FROM final__stg_exchange__segments \ No newline at end of file diff --git a/models/exchange/staging/service_types/stg_exchange__service_types.sql b/models/exchange/staging/service_types/stg_exchange__service_types.sql deleted file mode 100644 index 12e82f8..0000000 --- a/models/exchange/staging/service_types/stg_exchange__service_types.sql +++ /dev/null @@ -1,51 +0,0 @@ --- IMPORTS -WITH service_types AS ( - SELECT * FROM {{ ref('base_exchange__service_types') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -service_types_join_status AS ( - - SELECT - service_types.id, - service_types.name, - - service_types.status, - status.name AS status_name, - - service_types.deleted_at, - service_types.created_at, - service_types.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM service_types - - LEFT JOIN status - ON (service_types.status = status.id) -), - - --- FINAL -final__stg_exchange__service_types AS ( - - SELECT - id, - name, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM service_types_join_status - -) - -SELECT * FROM final__stg_exchange__service_types \ No newline at end of file diff --git a/models/exchange/staging/states/stg_exchange__states.sql b/models/exchange/staging/states/stg_exchange__states.sql deleted file mode 100644 index 5b041f9..0000000 --- a/models/exchange/staging/states/stg_exchange__states.sql +++ /dev/null @@ -1,66 +0,0 @@ --- IMPORTS -WITH states AS ( - SELECT * FROM {{ ref('base_exchange__states') }} -), - -countries AS ( - SELECT * FROM {{ ref('base_exchange__countries') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -states_join_countries_status AS ( - - SELECT - states.id, - - states.country_id, - countries.name AS country_name, - countries.short_code AS country_short_code, - - states.name, - - states.status, - status.name AS status_name, - - states.deleted_at, - states.created_at, - states.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM states - - LEFT JOIN countries - ON (states.country_id = countries.id) - - LEFT JOIN status - ON (states.status = status.id) -), - - --- FINAL -final__stg_exchange___states AS ( - - SELECT - id, - country_id, - country_name, - country_short_code, - name, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM states_join_countries_status - -) - -SELECT * FROM final__stg_exchange___states \ No newline at end of file diff --git a/models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql b/models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql deleted file mode 100644 index 0f95a3a..0000000 --- a/models/exchange/staging/transaction_costs/stg_exchange__transaction_costs.sql +++ /dev/null @@ -1,139 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ ref('base_exchange__transactions') }} -), - -transaction_types AS ( - SELECT * FROM {{ ref('seed_exchange__transaction_types') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -transactions_join_transaction_types_payment_methods_currencies_status AS ( - - SELECT - transactions.id, - transactions.owner_type, - transactions.owner_id AS transaction_order_id, - - transactions.transaction_type, - transaction_types.name AS transaction_type_name, - - transactions.issued_by AS supplier_company_id, - transactions.received_by AS buyer_company_id, - - transactions.recipient_bank_account_id AS bank_id, - - transactions.payment_method, - payment_methods.name AS payment_method_name, - - transactions.payment_reference, - transactions.bill_no, - - transactions.base_value, - transactions.quote_value, - - transactions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - transactions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - transactions.base_to_quote_currency_exchange_rate, - transactions.base_tax, - transactions.base_service_charge, - transactions.expired_at, - - transactions.status, - status.name AS status_name, - - transactions.deleted_at, - transactions.created_at, - transactions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transactions - - LEFT JOIN transaction_types - ON (transactions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transactions.payment_method = payment_methods.id) - - LEFT JOIN currencies AS base_currencies - ON (transactions.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (transactions.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (transactions.status = status.id) - - - WHERE - owner_type = 'App\\Models\\Transaction' - AND - transaction_type = '3' - -), - - --- FINAL -final__stg_exchange__transaction_costs AS ( - - SELECT - id, - owner_type, - transaction_order_id, - transaction_type, - transaction_type_name, - supplier_company_id, - buyer_company_id, - bank_id, - payment_method, - payment_method_name, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions_join_transaction_types_payment_methods_currencies_status - -) - -SELECT * FROM final__stg_exchange__transaction_costs \ No newline at end of file diff --git a/models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql b/models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql deleted file mode 100644 index fef1193..0000000 --- a/models/exchange/staging/transaction_costs_and_group_costs_relations/stg_exchange__transaction_costs_and_group_costs_relations.sql +++ /dev/null @@ -1,34 +0,0 @@ --- IMPORTS -WITH group_transactions AS ( - SELECT * FROM {{ ref('base_exchange__group_transactions') }} -), - - --- LOGIC -group_transactions_list AS ( - - SELECT - group_transactions.id, - group_transactions.group_id, - group_transactions.transaction_id, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM group_transactions - -), - - --- FINAL -final__stg_exchange__group_transactions AS ( - - SELECT - id, - group_id, - transaction_id, - _dbt_ran_at - - FROM group_transactions_list - -) - -SELECT * FROM final__stg_exchange__group_transactions \ No newline at end of file diff --git a/models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql b/models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql deleted file mode 100644 index 2db4783..0000000 --- a/models/exchange/staging/transaction_log_costs/stg_exchange__transaction_log_costs.sql +++ /dev/null @@ -1,183 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ ref('base_exchange__transactions') }} -), - -transaction_logs AS ( - SELECT * FROM {{ ref('base_exchange__transaction_logs') }} -), - -transaction_types AS ( - SELECT * FROM {{ ref('seed_exchange__transaction_types') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -transactions_add_hash_ids AS ( - SELECT - MD5_NUMBER_LOWER64(CONCAT(id, created_at)) AS id, - transactions.id AS transaction_cost_id, - owner_type, - owner_id, - transaction_type, - issued_by, - received_by, - recipient_bank_account_id, - payment_method, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions -), - - -transaction_logs_unions AS ( - SELECT * FROM transactions_add_hash_ids - UNION - SELECT * FROM transaction_logs -), - - -transaction_logs_join_transaction_types_payment_methods_currencies_status AS ( - - SELECT - transaction_logs_unions.id, - transaction_logs_unions.transaction_cost_id, - transaction_logs_unions.owner_type, - transaction_logs_unions.owner_id AS transaction_order_id, - - transaction_logs_unions.transaction_type, - transaction_types.name AS transaction_type_name, - - transaction_logs_unions.issued_by AS supplier_company_id, - transaction_logs_unions.received_by AS buyer_company_id, - - transaction_logs_unions.recipient_bank_account_id AS bank_id, - - transaction_logs_unions.payment_method, - payment_methods.name AS payment_method_name, - - transaction_logs_unions.payment_reference, - transaction_logs_unions.bill_no, - - transaction_logs_unions.base_value, - transaction_logs_unions.quote_value, - - transaction_logs_unions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - transaction_logs_unions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - transaction_logs_unions.base_to_quote_currency_exchange_rate, - transaction_logs_unions.base_tax, - transaction_logs_unions.base_service_charge, - transaction_logs_unions.expired_at, - - transaction_logs_unions.status, - status.name AS status_name, - - transaction_logs_unions.deleted_at, - transaction_logs_unions.created_at, - transaction_logs_unions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transaction_logs_unions - - LEFT JOIN transaction_types - ON (transaction_logs_unions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transaction_logs_unions.payment_method = payment_methods.id) - - LEFT JOIN currencies AS base_currencies - ON (transaction_logs_unions.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (transaction_logs_unions.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (transaction_logs_unions.status = status.id) - - - WHERE - owner_type = 'App\\Models\\Transaction' - AND - transaction_type = '3' - -), - - --- FINAL -final__stg_exchange__transaction_log_costs AS ( - - SELECT - id, - transaction_cost_id, - owner_type, - transaction_order_id, - transaction_type, - transaction_type_name, - supplier_company_id, - buyer_company_id, - bank_id, - payment_method, - payment_method_name, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transaction_logs_join_transaction_types_payment_methods_currencies_status - -) - -SELECT * FROM final__stg_exchange__transaction_log_costs \ No newline at end of file diff --git a/models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql b/models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql deleted file mode 100644 index c9d9e89..0000000 --- a/models/exchange/staging/transaction_log_orders/stg_exchange__transaction_log_orders.sql +++ /dev/null @@ -1,183 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ ref('base_exchange__transactions') }} -), - -transaction_logs AS ( - SELECT * FROM {{ ref('base_exchange__transaction_logs') }} -), - -transaction_types AS ( - SELECT * FROM {{ ref('seed_exchange__transaction_types') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -transactions_add_hash_ids AS ( - SELECT - MD5_NUMBER_LOWER64(CONCAT(id, created_at)) AS id, - transactions.id AS transaction_order_id, - owner_type, - owner_id, - transaction_type, - issued_by, - received_by, - recipient_bank_account_id, - payment_method, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions -), - - -transaction_logs_unions AS ( - SELECT * FROM transactions_add_hash_ids - UNION - SELECT * FROM transaction_logs -), - - -transaction_logs_join_transaction_types_payment_methods_currencies_status AS ( - - SELECT - transaction_logs_unions.id, - transaction_logs_unions.transaction_order_id, - transaction_logs_unions.owner_type, - transaction_logs_unions.owner_id AS booking_id, - - transaction_logs_unions.transaction_type, - transaction_types.name AS transaction_type_name, - - transaction_logs_unions.issued_by AS seller_company_id, - transaction_logs_unions.received_by AS company_id, - - transaction_logs_unions.recipient_bank_account_id AS bank_id, - - transaction_logs_unions.payment_method, - payment_methods.name AS payment_method_name, - - transaction_logs_unions.payment_reference, - transaction_logs_unions.bill_no, - - transaction_logs_unions.base_value, - transaction_logs_unions.quote_value, - - transaction_logs_unions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - transaction_logs_unions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - transaction_logs_unions.base_to_quote_currency_exchange_rate, - transaction_logs_unions.base_tax, - transaction_logs_unions.base_service_charge, - transaction_logs_unions.expired_at, - - transaction_logs_unions.status, - status.name AS status_name, - - transaction_logs_unions.deleted_at, - transaction_logs_unions.created_at, - transaction_logs_unions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transaction_logs_unions - - LEFT JOIN transaction_types - ON (transaction_logs_unions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transaction_logs_unions.payment_method = payment_methods.id) - - LEFT JOIN currencies AS base_currencies - ON (transaction_logs_unions.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (transaction_logs_unions.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (transaction_logs_unions.status = status.id) - - - WHERE - transaction_logs_unions.owner_type = 'App\\Models\\Booking' - AND - transaction_logs_unions.transaction_type = '1' - -), - - --- FINAL -final__stg_exchange__transaction_log_orders AS ( - - SELECT - id, - transaction_order_id, - owner_type, - booking_id, - transaction_type, - transaction_type_name, - seller_company_id, - company_id, - bank_id, - payment_method, - payment_method_name, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transaction_logs_join_transaction_types_payment_methods_currencies_status - -) - -SELECT * FROM final__stg_exchange__transaction_log_orders \ No newline at end of file diff --git a/models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql b/models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql deleted file mode 100644 index 4ac337b..0000000 --- a/models/exchange/staging/transaction_order_details/stg_exchange__transaction_order_details.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH transaction_details AS ( - SELECT * FROM {{ ref('base_exchange__transaction_details') }} -), - - --- LOGIC -transaction_details_list AS ( - - SELECT - transaction_details.id, - transaction_details.transaction_id, - transaction_details.product_code, - transaction_details.product_name, - transaction_details.quantity, - transaction_details.price, - transaction_details.amount, - transaction_details.deleted_at, - transaction_details.created_at, - transaction_details.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at - - FROM transaction_details - -), - - --- FINAL -final__stg_exchange__transaction_details AS ( - - SELECT - id, - transaction_id, - product_code, - product_name, - quantity, - price, - amount, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transaction_details_list - -) - -SELECT * FROM final__stg_exchange__transaction_details \ No newline at end of file diff --git a/models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql b/models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql deleted file mode 100644 index 87a116c..0000000 --- a/models/exchange/staging/transaction_orders/stg_exchange__transaction_orders.sql +++ /dev/null @@ -1,139 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ ref('base_exchange__transactions') }} -), - -transaction_types AS ( - SELECT * FROM {{ ref('seed_exchange__transaction_types') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -transactions_join_transaction_types_payment_methods_currencies_status AS ( - - SELECT - transactions.id, - transactions.owner_type, - transactions.owner_id AS booking_id, - - transactions.transaction_type, - transaction_types.name AS transaction_type_name, - - transactions.issued_by AS seller_company_id, - transactions.received_by AS company_id, - - transactions.recipient_bank_account_id AS bank_id, - - transactions.payment_method, - payment_methods.name AS payment_method_name, - - transactions.payment_reference, - transactions.bill_no, - - transactions.base_value, - transactions.quote_value, - - transactions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - transactions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - transactions.base_to_quote_currency_exchange_rate, - transactions.base_tax, - transactions.base_service_charge, - transactions.expired_at, - - transactions.status, - status.name AS status_name, - - transactions.deleted_at, - transactions.created_at, - transactions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transactions - - LEFT JOIN transaction_types - ON (transactions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transactions.payment_method = payment_methods.id) - - LEFT JOIN currencies AS base_currencies - ON (transactions.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (transactions.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (transactions.status = status.id) - - - WHERE - transactions.owner_type = 'App\\Models\\Booking' - AND - transactions.transaction_type = '1' - -), - - --- FINAL -final__stg_exchange__transaction_orders AS ( - - SELECT - id, - owner_type, - booking_id, - transaction_type, - transaction_type_name, - seller_company_id, - company_id, - bank_id, - payment_method, - payment_method_name, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions_join_transaction_types_payment_methods_currencies_status - -) - -SELECT * FROM final__stg_exchange__transaction_orders \ No newline at end of file diff --git a/models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql b/models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql deleted file mode 100644 index de2b23e..0000000 --- a/models/exchange/staging/transaction_wallets/stg_exchange__transaction_wallets.sql +++ /dev/null @@ -1,137 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ ref('base_exchange__transactions') }} -), - -transaction_types AS ( - SELECT * FROM {{ ref('seed_exchange__transaction_types') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -transactions_join_transaction_types_payment_methods_currencies_status AS ( - - SELECT - transactions.id, - transactions.owner_type, - transactions.owner_id AS booking_id, - - transactions.transaction_type, - transaction_types.name AS transaction_type_name, - - transactions.issued_by AS seller_company_id, - transactions.received_by AS company_id, - - transactions.recipient_bank_account_id, - - transactions.payment_method, - payment_methods.name AS payment_method_name, - - transactions.payment_reference, - transactions.bill_no, - - transactions.base_value, - transactions.quote_value, - - transactions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - transactions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - transactions.base_to_quote_currency_exchange_rate, - transactions.base_tax, - transactions.base_service_charge, - transactions.expired_at, - - transactions.status, - status.name AS status_name, - - transactions.deleted_at, - transactions.created_at, - transactions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transactions - - LEFT JOIN transaction_types - ON (transactions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transactions.payment_method = payment_methods.id) - - LEFT JOIN currencies AS base_currencies - ON (transactions.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (transactions.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (transactions.status = status.id) - - - WHERE - owner_type = 'App\\Models\\Wallet' - -), - - --- FINAL -final__stg_exchange__transactions AS ( - - SELECT - id, - owner_type, - booking_id, - transaction_type, - transaction_type_name, - seller_company_id, - company_id, - recipient_bank_account_id, - payment_method, - payment_method_name, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions_join_transaction_types_payment_methods_currencies_status - -) - -SELECT * FROM final__stg_exchange__transactions \ No newline at end of file diff --git a/models/exchange/staging/transactions/stg_exchange__transactions.sql b/models/exchange/staging/transactions/stg_exchange__transactions.sql deleted file mode 100644 index 6494389..0000000 --- a/models/exchange/staging/transactions/stg_exchange__transactions.sql +++ /dev/null @@ -1,132 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ ref('base_exchange__transactions') }} -), - -transaction_types AS ( - SELECT * FROM {{ ref('seed_exchange__transaction_types') }} -), - -payment_methods AS ( - SELECT * FROM {{ ref('seed_exchange__payment_methods') }} -), - -currencies AS ( - SELECT * FROM {{ ref('base_exchange__currencies') }} -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -transactions_join_transaction_types_payment_methods_currencies_status AS ( - - SELECT - transactions.id, - transactions.owner_type, - transactions.owner_id, - - transactions.transaction_type, - transaction_types.name AS transaction_type_name, - - transactions.issued_by, - transactions.received_by, - - transactions.recipient_bank_account_id, - - transactions.payment_method, - payment_methods.name AS payment_method_name, - - transactions.payment_reference, - transactions.bill_no, - - transactions.base_value, - transactions.quote_value, - - transactions.base_currency_id, - base_currencies.name AS base_currency_name, - base_currencies.short_code AS base_currency_short_code, - base_currencies.symbol AS base_currency_symbol, - - transactions.quote_currency_id, - quote_currencies.name AS quote_currency_name, - quote_currencies.short_code AS quote_currency_short_code, - quote_currencies.symbol AS quote_currency_symbol, - - transactions.base_to_quote_currency_exchange_rate, - transactions.base_tax, - transactions.base_service_charge, - transactions.expired_at, - - transactions.status, - status.name AS status_name, - - transactions.deleted_at, - transactions.created_at, - transactions.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM transactions - - LEFT JOIN transaction_types - ON (transactions.transaction_type = transaction_types.id) - - LEFT JOIN payment_methods - ON (transactions.payment_method = payment_methods.id) - - LEFT JOIN currencies AS base_currencies - ON (transactions.base_currency_id = base_currencies.id) - - LEFT JOIN currencies AS quote_currencies - ON (transactions.quote_currency_id = quote_currencies.id) - - LEFT JOIN status - ON (transactions.status = status.id) -), - - --- FINAL -final__stg_exchange__transactions AS ( - - SELECT - id, - owner_type, - owner_id, - transaction_type, - transaction_type_name, - issued_by, - received_by, - recipient_bank_account_id, - payment_method, - payment_method_name, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - base_currency_name, - base_currency_short_code, - base_currency_symbol, - quote_currency_id, - quote_currency_name, - quote_currency_short_code, - quote_currency_symbol, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_at, - status, - status_name, - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM transactions_join_transaction_types_payment_methods_currencies_status - -) - -SELECT * FROM final__stg_exchange__transactions \ No newline at end of file diff --git a/models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql b/models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql deleted file mode 100644 index 8d319ef..0000000 --- a/models/exchange/staging/user_email_verifications/stg_exchange__user_email_verifications.sql +++ /dev/null @@ -1,53 +0,0 @@ --- IMPORTS -WITH user_email_verifications AS ( - SELECT * FROM {{ ref('base_exchange__user_email_verifications') }} -), - -users AS ( - SELECT * FROM {{ ref('base_exchange__users') }} -), - --- LOGIC -user_email_verifications_join_users AS ( - - SELECT - user_email_verifications.id, - - users.id AS user_id, - - user_email_verifications.email, - user_email_verifications.token, - user_email_verifications.is_completed, - user_email_verifications.is_activated, - user_email_verifications.is_sent, - user_email_verifications.created_at, - user_email_verifications.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM user_email_verifications - - LEFT JOIN users - ON (user_email_verifications.email = users.email) -), - - --- FINAL -final__base_exchange__user_email_verifications AS ( - - SELECT - id, - user_id, - email, - token, - is_completed, - is_activated, - is_sent, - created_at, - updated_at, - _dbt_ran_at - - FROM user_email_verifications_join_users - -) - -SELECT * FROM final__base_exchange__user_email_verifications \ No newline at end of file diff --git a/models/exchange/staging/users/stg_exchange__users.sql b/models/exchange/staging/users/stg_exchange__users.sql deleted file mode 100644 index 15c6fb8..0000000 --- a/models/exchange/staging/users/stg_exchange__users.sql +++ /dev/null @@ -1,80 +0,0 @@ --- IMPORTS -WITH users AS ( - SELECT * FROM {{ ref('base_exchange__users') }} -), - -employees AS ( - SELECT * FROM {{ ref('base_exchange__employees') }} -), - -user_types AS ( - SELECT * FROM {{ ref('seed_exchange__user_types') }} - WHERE category = 'DEFAULT' -), - -status AS ( - SELECT * FROM {{ ref('seed_exchange__status') }} - WHERE category = 'DEFAULT' -), - - --- LOGIC -users_join_user_types_status AS ( - - SELECT - users.id, - users.name, - users.email, - - users.user_type, - user_types.name AS user_type_name, - - users.status, - status.name AS status_name, - - employees.company_id AS company_id, - - users.remember_token, - users.active_at, - users.deleted_at, - users.created_at, - users.updated_at, - current_timestamp() AS _dbt_ran_at - - FROM users - - LEFT JOIN employees - ON (users.id = employees.user_id) - - LEFT JOIN user_types - ON (users.user_type = user_types.id) - - LEFT JOIN status - ON (users.status = status.id) -), - - --- FINAL -final__stg_exchange__users AS ( - - SELECT - id, - name, - email, - user_type, - user_type_name, - status, - status_name, - company_id, - remember_token, - active_at - deleted_at, - created_at, - updated_at, - _dbt_ran_at - - FROM users_join_user_types_status - -) - -SELECT * FROM final__stg_exchange__users \ No newline at end of file diff --git a/models/crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql b/models/intermediate/int_crisp__messages_generate_dialog_ids.sql similarity index 58% rename from models/crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql rename to models/intermediate/int_crisp__messages_generate_dialog_ids.sql index 36b61a5..3aaa56e 100644 --- a/models/crisp/intermediate/int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by.sql +++ b/models/intermediate/int_crisp__messages_generate_dialog_ids.sql @@ -1,7 +1,7 @@ --IMPORT WITH messages AS ( SELECT * FROM {{ ref('stg_crisp__messages') }} - ORDER BY conversation_id, message_created_datetime, id + ORDER BY conversation_id, created_datetime, message_id ), conversations AS ( @@ -12,7 +12,7 @@ conversations AS ( mark_first_dialog_message AS ( SELECT CASE - WHEN LAG(event_type) OVER (ORDER BY conversation_id, message_created_datetime, id) = 'state:resolved' + WHEN LAG(event_type) OVER (ORDER BY conversation_id, created_datetime, message_id) = 'state:resolved' THEN 1 ELSE 0 END AS first_dialog_message_flag, @@ -23,18 +23,18 @@ mark_first_dialog_message AS ( message_with_dialog_id AS ( SELECT - (SUM(first_dialog_message_flag) OVER (ORDER BY conversation_id, message_created_datetime, id ROWS UNBOUNDED PRECEDING))+1 AS dialog_id, + (SUM(first_dialog_message_flag) OVER (ORDER BY conversation_id, created_datetime, message_id ROWS UNBOUNDED PRECEDING))+1 AS dialog_id, * FROM mark_first_dialog_message - ORDER BY conversation_id, message_created_datetime + ORDER BY conversation_id, created_datetime ), --FINAL -final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_type AS ( +message_with_dialog_id_get_user_id AS ( SELECT message_with_dialog_id.dialog_id, - message_with_dialog_id.id, + message_with_dialog_id.message_id, message_with_dialog_id.conversation_id, @@ -53,10 +53,10 @@ final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_ty WHEN message_with_dialog_id.message_from_side = 'operator' AND message_with_dialog_id.user_id IS NOT NULL THEN 'operator' WHEN message_with_dialog_id.message_from_side = 'operator' AND message_with_dialog_id.user_id IS NULL THEN 'bot' ELSE 'ERROR please contact DATA team' - END AS message_generate_by, + END AS message_generated_by, - message_with_dialog_id.message_generate_platform, - message_with_dialog_id.message_deliverd_platform, + message_with_dialog_id.message_generated_platform, + message_with_dialog_id.message_delivered_platform, message_with_dialog_id.message_read_platform, message_with_dialog_id.message_format, message_with_dialog_id.event_type, @@ -66,12 +66,49 @@ final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_ty message_with_dialog_id.attachment_type, message_with_dialog_id.attachment_name, message_with_dialog_id.attachment_url, - message_with_dialog_id.message_created_datetime + message_with_dialog_id.created_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM message_with_dialog_id LEFT JOIN conversations - ON (message_with_dialog_id.user_id = conversations.id) --The user_id will become conversation_id if message_from_side is user + ON (message_with_dialog_id.user_id = conversations.conversation_id) --The user_id will become conversation_id if message_from_side is user +), + +-- FINAL +final__int_crisp__messages_generate_dialog_ids AS ( + SELECT + -- ids + dialog_id, + message_id, + conversation_id, + user_id, + + -- dimensions + user_name, + message_from_side, + message_generated_by, + message_generated_platform, + message_delivered_platform, + message_read_platform, + message_format, + event_type, + event_message, + message, + is_edited_message, + attachment_type, + attachment_name, + attachment_url, + + -- measures + + -- date/times + created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM message_with_dialog_id_get_user_id ) -SELECT * FROM final_int_crisp__conversation_messages_get_dialog_id_user_id_classify_message_type \ No newline at end of file +SELECT * FROM final__int_crisp__messages_generate_dialog_ids \ No newline at end of file diff --git a/models/exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql b/models/intermediate/int_exchange__bookings_get_estimate_booking_values.sql similarity index 58% rename from models/exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql rename to models/intermediate/int_exchange__bookings_get_estimate_booking_values.sql index 3360294..c7ee3ce 100644 --- a/models/exchange/intermediate/bookings_get_estimate_rates/int_exchange__bookings_get_estimate_rates.sql +++ b/models/intermediate/int_exchange__bookings_get_estimate_booking_values.sql @@ -12,32 +12,24 @@ currency_rate_logs AS ( bookings_calculate_value_join_currency_rate_logs AS ( SELECT - bookings.id, + bookings.booking_id, bookings.company_id, bookings.user_id, - bookings.marking_id, + bookings.booking_marking_id, bookings.service_type, - bookings.service_type_name, bookings.bank_id, bookings.fix_value, bookings.fix_currency_id, - bookings.fix_currency_name, - bookings.fix_currency_short_code, - currency_rate_logs.payment_method AS estimate_payment_method, - currency_rate_logs.payment_method_name AS estimate_payment_method_name, - currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate, + currency_rate_logs.payment_method AS estimate_payment_method, + currency_rate_logs.base_to_quote_currency_exchange_rate AS estimate_base_to_quote_currency_exchange_rate, bookings.quote_currency_id, - bookings.quote_currency_name, - bookings.quote_currency_short_code, IFF(bookings.quote_currency_id = bookings.fix_currency_id, fix_value, ROUND(fix_value * estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_quote_value, bookings.base_currency_id, - bookings.base_currency_name, - bookings.base_currency_short_code, IFF(bookings.base_currency_id = bookings.fix_currency_id, fix_value, ROUND(fix_value / estimate_base_to_quote_currency_exchange_rate,2)) AS estimate_base_value, @@ -53,30 +45,57 @@ bookings_calculate_value_join_currency_rate_logs AS ( END AS estimate_value_rm, bookings.status, - bookings.status_name, - bookings.booking_deleted_datetime, - bookings.booking_created_datetime, - bookings.booking_updated_datetime, - current_timestamp() AS _dbt_ran_at + bookings.deleted_datetime, + bookings.created_datetime, + bookings.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM bookings LEFT JOIN currency_rate_logs - ON (bookings.booking_created_datetime >= currency_rate_logs.created_date_from) - AND (bookings.booking_created_datetime < currency_rate_logs.created_date_to) + ON (bookings.created_datetime >= currency_rate_logs.from_datetime) + AND (bookings.created_datetime < currency_rate_logs.to_datetime) AND (bookings.service_type = currency_rate_logs.service_type) AND (bookings.quote_currency_id = currency_rate_logs.quote_currency_id) - AND (currency_rate_logs.payment_method = '1') + AND (currency_rate_logs.payment_method = 'CASH') ), -- FINAL -final__int_bookings_get_estimate_rates__exchange AS ( +final__bookings_get_estimate_booking_values AS ( SELECT - * + -- ids + booking_id, + booking_marking_id, + company_id, + user_id, + bank_id, + fix_currency_id, + quote_currency_id, + base_currency_id, + + -- dimensions + service_type, + estimate_payment_method, + status, + + -- measures + fix_value, + estimate_base_to_quote_currency_exchange_rate, + estimate_quote_value, + estimate_base_value, + estimate_value_rm, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime FROM bookings_calculate_value_join_currency_rate_logs ) -SELECT * FROM final__int_bookings_get_estimate_rates__exchange \ No newline at end of file +SELECT * FROM final__bookings_get_estimate_booking_values diff --git a/models/intermediate/int_exchange__companies_join_segments.sql b/models/intermediate/int_exchange__companies_join_segments.sql new file mode 100644 index 0000000..e5cf7b3 --- /dev/null +++ b/models/intermediate/int_exchange__companies_join_segments.sql @@ -0,0 +1,95 @@ +-- Import +WITH companies AS ( + SELECT * FROM {{ ref('stg_exchange__companies') }} +), + +segments AS ( + SELECT * FROM {{ ref('stg_exchange__segments') }} +), + +companies_brg_segments AS ( + SELECT * FROM {{ ref('stg_exchange__companies_brg_segments') }} +), + + +-- LOGIC +get_is_migrated_companies AS ( + SELECT + segment_id, + company_id + + FROM companies_brg_segments + + WHERE + segment_id = 5 +), + + +-- Because we do not have timestamp to get maximum segment_id +-- Will act like a id from in asc order, and add in the companies_brg_segments table +get_segment_row_created_sequences AS ( + SELECT + *, + ROW_NUMBER() OVER (ORDER BY NULL) AS row_created_sequence + + FROM companies_brg_segments + + WHERE + segment_id IN (1,2,3,4) + AND + dbt_valid_to_datetime IS NULL +), + +-- use the asc order id in the get_segment_row_created_sequences, and find out the company latest segment +get_company_latest_segments AS ( + SELECT + *, + row_number() OVER + (PARTITION BY company_id + ORDER BY row_created_sequence DESC) AS is_latest_segment + + FROM get_segment_row_created_sequences + + QUALIFY + is_latest_segment = 1 +), + + +-- FINAL +final__int_exchange__companies_join_segments AS ( + SELECT + -- ids + companies.company_id, + companies.company_marking_id, + companies.autocount_id, + + -- dimensions + companies.name, + companies.company_type, + companies.business_type, + companies.status, + segments.name AS exchange_rate_segment, + IFF(get_is_migrated_companies.company_id IS NOT NULL, 1, 0) AS is_migrated_company, + + -- measures + + -- date/times + companies.created_datetime, + companies.updated_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies + + LEFT JOIN get_company_latest_segments + ON (companies.company_id = get_company_latest_segments.company_id) + + LEFT JOIN segments + ON (get_company_latest_segments.segment_id = segments.segment_id) + + LEFT JOIN get_is_migrated_companies + ON (companies.company_id = get_is_migrated_companies.company_id) +) + +SELECT * FROM final__int_exchange__companies_join_segments \ No newline at end of file diff --git a/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql b/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql new file mode 100644 index 0000000..8e99ad5 --- /dev/null +++ b/models/intermediate/int_exchange__transaction_orders_get_user_ids.sql @@ -0,0 +1,61 @@ +-- IMPORTS +WITH transaction_orders AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_orders') }} +), + +bookings AS ( + SELECT * FROM {{ ref('stg_exchange__bookings') }} +), + +-- LOGICS +transaction_orders_join_bookings AS ( + SELECT + transaction_orders.*, + bookings.user_id + + FROM transaction_orders + + LEFT JOIN bookings + ON (transaction_orders.booking_id = bookings.booking_id) +), + + +-- FINAL +final__int_exchange__transaction_orders_get_user_ids AS ( + SELECT + -- ids + transaction_order_id, + booking_id, + company_id, + user_id, + bank_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_orders_join_bookings +) + +SELECT * FROM final__int_exchange__transaction_orders_get_user_ids \ No newline at end of file diff --git a/models/exchange/marts/warehouse/test_exchange__bookings.yml b/models/marts/reporting/_reporting_models.yml similarity index 100% rename from models/exchange/marts/warehouse/test_exchange__bookings.yml rename to models/marts/reporting/_reporting_models.yml diff --git a/models/exchange/marts/warehouse/test_exchange__companies.yml b/models/marts/reporting/havent_do_metric_comparisons.sql similarity index 100% rename from models/exchange/marts/warehouse/test_exchange__companies.yml rename to models/marts/reporting/havent_do_metric_comparisons.sql diff --git a/analyses/exchange/company_conversion_v2.sql b/models/marts/reporting/rep_exchange__company_conversion_funnels.sql similarity index 55% rename from analyses/exchange/company_conversion_v2.sql rename to models/marts/reporting/rep_exchange__company_conversion_funnels.sql index af2afe3..1b28f75 100644 --- a/analyses/exchange/company_conversion_v2.sql +++ b/models/marts/reporting/rep_exchange__company_conversion_funnels.sql @@ -1,5 +1,5 @@ -- MARCRO for count the cummulative_activity_number -{% set funnel_activity = ['register','email_verified','identity_documents_uploaded','identity_documents_verified','first_time_booking','first_time_order','first_time_order_completed','repeat_completed_order_after_30days','repeat_completed_order_after_60days']%} +{% set funnel_activity = ['register','email_verified','identity_document_uploaded','identity_document_verified','first_time_booking','first_time_order','first_time_order_completed','repeat_completed_order_after_30days','repeat_completed_order_after_60days']%} -- IMPORT WITH companies AS ( @@ -15,7 +15,7 @@ company_identity_documents AS ( ), email_verifications AS ( - SELECT * FROM {{ ref('fct_exchange__email_verifications') }} + SELECT * FROM {{ ref('fct_exchange__user_email_verifications') }} ), bookings AS ( @@ -29,7 +29,7 @@ remove_supplier_company AS ( * FROM companies - WHERE business_type NOT IN ('1','3') + WHERE business_type NOT IN ('CURRENCY_VENDOR', 'FREIGHT_FORWARDER') ), clean_company_transaction_orders AS ( @@ -40,7 +40,7 @@ clean_company_transaction_orders AS ( -- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders INNER JOIN remove_supplier_company - ON (transaction_orders.company_id = remove_supplier_company.id) + ON (transaction_orders.company_id = remove_supplier_company.company_id) ), clean_company_bookings AS ( @@ -51,15 +51,15 @@ clean_company_bookings AS ( -- Remove supplier (If any) 16/3/2023 Currently do not have any supplier in transaction orders INNER JOIN remove_supplier_company - ON (bookings.company_id = remove_supplier_company.id) + ON (bookings.company_id = remove_supplier_company.company_id) ), company_first_time_order_date AS ( SELECT - id, + transaction_order_id, company_id, - created_at + order_created_datetime FROM clean_company_transaction_orders @@ -69,13 +69,14 @@ company_first_time_order_date AS ( ---start defind funnel +--start defind funnel activity repeat_completed_order_after_60days_activities AS ( SELECT clean_company_transaction_orders.company_id, 0 AS is_data_cleansing_generate_row, 'repeat_completed_order_after_60days' AS activity, - MIN(clean_company_transaction_orders.created_at) AS activity_datetime + 9 AS activtity_rank, + MIN(clean_company_transaction_orders.order_created_datetime) AS activity_datetime FROM clean_company_transaction_orders @@ -83,7 +84,7 @@ repeat_completed_order_after_60days_activities AS ( ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id) WHERE - DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 60 + DATEDIFF(day, company_first_time_order_date.order_created_datetime, clean_company_transaction_orders.order_created_datetime) >= 60 GROUP BY clean_company_transaction_orders.company_id @@ -94,7 +95,8 @@ repeat_completed_order_after_30days_activities AS ( clean_company_transaction_orders.company_id, 0 AS is_data_cleansing_generate_row, 'repeat_completed_order_after_30days' AS activity, - MIN(clean_company_transaction_orders.created_at) AS activity_datetime + 8 AS activtity_rank, + MIN(clean_company_transaction_orders.order_created_datetime) AS activity_datetime FROM clean_company_transaction_orders @@ -102,7 +104,7 @@ repeat_completed_order_after_30days_activities AS ( ON (clean_company_transaction_orders.company_id = company_first_time_order_date.company_id) WHERE - DATEDIFF(day, company_first_time_order_date.created_at, clean_company_transaction_orders.created_at) >= 30 + DATEDIFF(day, company_first_time_order_date.order_created_datetime, clean_company_transaction_orders.order_created_datetime) >= 30 GROUP BY clean_company_transaction_orders.company_id @@ -113,7 +115,8 @@ first_time_order_completed_activities AS ( clean_company_transaction_orders.company_id, 0 AS is_data_cleansing_generate_row, 'first_time_order_completed' AS activity, - MIN(clean_company_transaction_orders.created_at) AS activity_datetime + 7 AS activtity_rank, + MIN(clean_company_transaction_orders.order_created_datetime) AS activity_datetime FROM clean_company_transaction_orders @@ -129,7 +132,8 @@ first_time_order_activities AS ( clean_company_transaction_orders.company_id, 0 AS is_data_cleansing_generate_row, 'first_time_order' AS activity, - MIN(clean_company_transaction_orders.created_at) AS activity_datetime + 6 AS activtity_rank, + MIN(clean_company_transaction_orders.order_created_datetime) AS activity_datetime FROM clean_company_transaction_orders @@ -145,6 +149,7 @@ first_time_booking_activities AS ( IFNULL( unclean_first_time_booking_activities.company_id , first_time_order_activities.company_id ) AS company_id, IFF( unclean_first_time_booking_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, 'first_time_booking' AS activity, + 5 AS activtity_rank, IFNULL( unclean_first_time_booking_activities.activity_datetime , first_time_order_activities.activity_datetime ) AS activity_datetime FROM @@ -168,19 +173,20 @@ first_time_booking_activities AS ( ), -identity_documents_verified_activities AS ( +identity_document_verified_activities AS ( SELECT - IFNULL( unclean_identity_documents_verified_activities.company_id , first_time_booking_activities.company_id ) AS company_id, - IFF( unclean_identity_documents_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, - 'identity_documents_verified' AS activity, - IFNULL( unclean_identity_documents_verified_activities.activity_datetime , first_time_booking_activities.activity_datetime ) AS activity_datetime + IFNULL( unclean_identity_document_verified_activities.company_id , first_time_booking_activities.company_id ) AS company_id, + IFF( unclean_identity_document_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'identity_document_verified' AS activity, + 4 AS activtity_rank, + IFNULL( unclean_identity_document_verified_activities.activity_datetime , first_time_booking_activities.activity_datetime ) AS activity_datetime FROM ( SELECT company_id, - 'identity_documents_verified' AS activity, + 'identity_document_verified' AS activity, MIN(identity_created_datetime) AS activity_datetime FROM company_identity_documents @@ -191,27 +197,28 @@ identity_documents_verified_activities AS ( GROUP BY company_id - ) AS unclean_identity_documents_verified_activities + ) AS unclean_identity_document_verified_activities FULL OUTER JOIN first_time_booking_activities - ON (unclean_identity_documents_verified_activities.company_id = first_time_booking_activities.company_id) + ON (unclean_identity_document_verified_activities.company_id = first_time_booking_activities.company_id) ), -identity_documents_uploaded_activities AS ( +identity_document_uploaded_activities AS ( SELECT - IFNULL( unclean_identity_documents_uploaded_activities.company_id , identity_documents_verified_activities.company_id ) AS company_id, - IFF( unclean_identity_documents_uploaded_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, - 'identity_documents_uploaded' AS activity, - IFNULL( unclean_identity_documents_uploaded_activities.activity_datetime , identity_documents_verified_activities.activity_datetime ) AS activity_datetime + IFNULL( unclean_identity_document_uploaded_activities.company_id , identity_document_verified_activities.company_id ) AS company_id, + IFF( unclean_identity_document_uploaded_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, + 'identity_document_uploaded' AS activity, + 3 AS activtity_rank, + IFNULL( unclean_identity_document_uploaded_activities.activity_datetime , identity_document_verified_activities.activity_datetime ) AS activity_datetime FROM ( SELECT company_id, - 'identity_documents_uploaded' AS activity, + 'identity_document_uploaded' AS activity, MIN(identity_created_datetime) AS activity_datetime FROM company_identity_documents @@ -219,26 +226,27 @@ identity_documents_uploaded_activities AS ( GROUP BY company_id - ) AS unclean_identity_documents_uploaded_activities + ) AS unclean_identity_document_uploaded_activities - FULL OUTER JOIN identity_documents_verified_activities - ON (unclean_identity_documents_uploaded_activities.company_id = identity_documents_verified_activities.company_id) + FULL OUTER JOIN identity_document_verified_activities + ON (unclean_identity_document_uploaded_activities.company_id = identity_document_verified_activities.company_id) ), email_verified_activities AS ( SELECT - IFNULL( unclean_email_verified_activities.company_id , identity_documents_uploaded_activities.company_id ) AS company_id, + IFNULL( unclean_email_verified_activities.company_id , identity_document_uploaded_activities.company_id ) AS company_id, IFF( unclean_email_verified_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, 'email_verified' AS activity, - IFNULL( unclean_email_verified_activities.activity_datetime , identity_documents_uploaded_activities.activity_datetime ) AS activity_datetime + 2 AS activtity_rank, + IFNULL( unclean_email_verified_activities.activity_datetime , identity_document_uploaded_activities.activity_datetime ) AS activity_datetime FROM ( SELECT company_id, 'email_verified' AS activity, - MIN(verification_email_created_datetime) AS activity_datetime + MIN(user_email_verification_created_datetime) AS activity_datetime FROM email_verifications @@ -249,8 +257,8 @@ email_verified_activities AS ( company_id ) AS unclean_email_verified_activities - FULL OUTER JOIN identity_documents_uploaded_activities - ON (unclean_email_verified_activities.company_id = identity_documents_uploaded_activities.company_id) + FULL OUTER JOIN identity_document_uploaded_activities + ON (unclean_email_verified_activities.company_id = identity_document_uploaded_activities.company_id) ), @@ -259,19 +267,20 @@ register_activities AS ( IFNULL( unclean_register_activities.company_id , email_verified_activities.company_id ) AS company_id, IFF( unclean_register_activities.company_id IS NULL, 1, 0) AS is_data_cleansing_generate_row, 'register' AS activity, + 1 AS activtity_rank, IFNULL( unclean_register_activities.activity_datetime , email_verified_activities.activity_datetime ) AS activity_datetime FROM ( SELECT - id AS company_id, + company_id, 'register' AS activity, - MIN(created_at) AS activity_datetime + MIN(company_created_datetime) AS activity_datetime FROM remove_supplier_company GROUP BY - id + company_id ) AS unclean_register_activities FULL OUTER JOIN email_verified_activities @@ -303,39 +312,83 @@ union_all_activities_flag_migrated_company AS ( FROM union_all_activities LEFT JOIN remove_supplier_company - ON (union_all_activities.company_id = remove_supplier_company.id) + ON (union_all_activities.company_id = remove_supplier_company.company_id) ), - ---FINAL - -sem_exchange__company_funnels AS ( +get_number_of_conversions AS ( SELECT company_id, is_data_cleansing_generate_row, is_migrated_company, + activity, + activtity_rank, + IFF( ROW_NUMBER() OVER (PARTITION BY company_id ORDER BY activtity_rank DESC) = 1, + 1, + 0) AS is_company_latest_activity, + activity_datetime, - {% for activity in funnel_activity %} + {% for activity_type in funnel_activity %} - count(iff(activity = '{{activity}}', company_id, NULL)) - OVER (ORDER BY activity_datetime) AS cumulative_{{activity}}_activity, + count(iff(activity = '{{activity_type}}' AND is_migrated_company = 0, company_id, NULL)) + OVER (ORDER BY activity_datetime) AS cumulative_{{activity_type}}_activity, {% endfor %} - {% for activity in funnel_activity %} + {% for activity_type in funnel_activity %} - count(iff(activity = '{{activity}}' AND is_migrated_company = 0, company_id, NULL)) - OVER (ORDER BY activity_datetime) AS exclude_migrated_cumulative_{{activity}}_activity, + count(iff(activity = '{{activity_type}}', company_id, NULL)) + OVER (ORDER BY activity_datetime) AS include_migrated_cumulative_{{activity_type}}_activity, {% endfor %} - '__macro_end' AS __macro_end + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM union_all_activities_flag_migrated_company ORDER BY activity_datetime +), + +-- FINAL +final__rep_exchange__company_conversion_funnels AS ( + SELECT + -- ids + company_id, + + -- dimensions + activity, + activtity_rank, + is_company_latest_activity, + is_data_cleansing_generate_row, + is_migrated_company, + + -- measures + cumulative_register_activity, + cumulative_email_verified_activity, + cumulative_identity_document_uploaded_activity, + cumulative_identity_document_verified_activity, + cumulative_first_time_booking_activity, + cumulative_first_time_order_activity, + cumulative_first_time_order_completed_activity, + cumulative_repeat_completed_order_after_30days_activity, + cumulative_repeat_completed_order_after_60days_activity, + include_migrated_cumulative_register_activity, + include_migrated_cumulative_email_verified_activity, + include_migrated_cumulative_identity_document_uploaded_activity, + include_migrated_cumulative_first_time_booking_activity, + include_migrated_cumulative_first_time_order_activity, + include_migrated_cumulative_first_time_order_completed_activity, + include_migrated_cumulative_repeat_completed_order_after_30days_activity, + include_migrated_cumulative_repeat_completed_order_after_60days_activity, + + -- date/times + activity_datetime, + + -- metadata + _dbt_ran_datetime + + FROM get_number_of_conversions ) -SELECT * FROM sem_exchange__company_funnels \ No newline at end of file +SELECT * FROM final__rep_exchange__company_conversion_funnels \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__company_engagement_behaviors.sql b/models/marts/reporting/rep_exchange__company_engagement_behaviors.sql new file mode 100644 index 0000000..312bba9 --- /dev/null +++ b/models/marts/reporting/rep_exchange__company_engagement_behaviors.sql @@ -0,0 +1,216 @@ +-- AVAILALE FILTER VALUE +{% set day_use_to_churn = 61 %} + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + + +-- LOGIC +remove_supplier_company AS ( + SELECT + * + FROM companies + + WHERE business_type = 'IMPORTER' +), + + +order_date_row_rank AS ( + SELECT + *, + row_number() OVER + (PARTITION BY transaction_orders.company_id + ORDER BY transaction_orders.order_created_datetime) AS row_rank_index + + FROM + transaction_orders +), + +get_next_and_previous_order_date_orders AS ( + SELECT + order_date_row_rank.*, + companies.is_migrated_company, + next_order_date_row_rank_clone.order_created_datetime AS next_order_created_datetime, + previous_order_date_row_rank_clone.order_created_datetime AS previous_order_created_datetime, + companies.company_marking_id, + companies.company_created_datetime + + FROM order_date_row_rank + + LEFT JOIN companies + ON (order_date_row_rank.company_id = companies.company_id) + + LEFT JOIN order_date_row_rank AS next_order_date_row_rank_clone + ON order_date_row_rank.row_rank_index = next_order_date_row_rank_clone.row_rank_index - 1 + AND order_date_row_rank.company_id = next_order_date_row_rank_clone.company_id + + LEFT JOIN order_date_row_rank AS previous_order_date_row_rank_clone + ON order_date_row_rank.row_rank_index = previous_order_date_row_rank_clone.row_rank_index + 1 + AND order_date_row_rank.company_id = previous_order_date_row_rank_clone.company_id +), + +-- Register / Churn / Register; Churn +company_register_events AS ( + SELECT + remove_supplier_company.company_id AS company_id, + remove_supplier_company.is_migrated_company, + IFF(MAX(transaction_orders.company_id) IS NULL, 1, 0) AS is_never_order_company, + remove_supplier_company.company_marking_id, + 'Register' AS register_event, + IFF( + DATEDIFF( + day, + remove_supplier_company.company_created_datetime, + COALESCE(MIN(transaction_orders.order_created_datetime), CURRENT_DATE()) + ) + >= {{day_use_to_churn}}, + 'Churn', + NULL) AS churn_event, + + RTRIM(CONCAT( + CASE WHEN register_event IS NOT NULL THEN 'Register, ' ELSE '' END, + CASE WHEN churn_event IS NOT NULL THEN 'Churn, ' ELSE '' END ),', ' ) AS concat_event, + + remove_supplier_company.company_created_datetime AS event_datetime + + FROM remove_supplier_company + + LEFT JOIN transaction_orders + ON (remove_supplier_company.company_id = transaction_orders.company_id) + + GROUP BY + remove_supplier_company.company_id, + remove_supplier_company.company_marking_id, + remove_supplier_company.is_migrated_company, + remove_supplier_company.company_created_datetime +), + + +-- FirstTimeOrder / Repeat / Churn / Reactive +firsttime_repeat_churn_reactive_order_events AS ( + SELECT + company_id, + company_marking_id, + is_migrated_company, + 0 AS is_never_order_company, + transaction_order_id, + order_status, + payment_method, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + value_rm, + tax_rm, + service_charge_rm, + IFF(is_first_time_company = 1, 'First Time Order', NULL) AS first_time_order_event, + IFF(is_first_time_company = 0, 'Repeat', NULL) AS repeat_event, + IFF(DATEDIFF(day, order_created_datetime, COALESCE(next_order_created_datetime, CURRENT_DATE())) >= {{day_use_to_churn}}, 'Churn', NULL ) AS churn_event, + IFF(DATEDIFF(day, COALESCE(previous_order_created_datetime, company_created_datetime), order_created_datetime) >= {{day_use_to_churn}}, 'Reactive', NULL) AS reactive_event, + + RTRIM(CONCAT( + CASE WHEN first_time_order_event IS NOT NULL THEN 'First Time Order, ' ELSE '' END, + CASE WHEN repeat_event IS NOT NULL THEN 'Repeat, ' ELSE '' END, + CASE WHEN reactive_event IS NOT NULL THEN 'Reactive, ' ELSE '' END, + CASE WHEN churn_event IS NOT NULL THEN 'Churn, ' ELSE '' END ),', ' ) AS concat_event, + + order_created_datetime AS event_datetime + + FROM get_next_and_previous_order_date_orders +), + +union_event_tables AS ( + SELECT + company_id, + company_marking_id, + is_migrated_company, + is_never_order_company, + NULL AS transaction_order_id, + NULL AS order_status, + NULL AS payment_method, + NULL AS base_currency_id, + NULL AS quote_currency_id, + NULL AS base_to_quote_currency_exchange_rate, + NULL AS value_rm, + NULL AS tax_rm, + NULL AS service_charge_rm, + register_event, + NULL AS first_time_order_event, + NULL AS repeat_event, + churn_event, + NULL AS reactive_event, + concat_event, + event_datetime + + FROM company_register_events + + UNION + + SELECT + company_id, + company_marking_id, + is_migrated_company, + is_never_order_company, + transaction_order_id, + order_status, + payment_method, + base_currency_id, + quote_currency_id, + base_to_quote_currency_exchange_rate, + value_rm, + tax_rm, + service_charge_rm, + NULL AS register_event, + first_time_order_event, + repeat_event, + churn_event, + reactive_event, + concat_event, + event_datetime + + FROM firsttime_repeat_churn_reactive_order_events +), + +-- FINAL +final__rep_exchange__company_engagement_behaviors AS ( + SELECT + -- ids + company_id, + company_marking_id, + transaction_order_id, + base_currency_id, + quote_currency_id, + + -- dimensions + is_migrated_company, + is_never_order_company, + order_status, + payment_method, + register_event, + first_time_order_event, + repeat_event, + churn_event, + reactive_event, + concat_event, + + -- measures + base_to_quote_currency_exchange_rate, + value_rm, + tax_rm, + service_charge_rm, + + -- date/times + event_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM union_event_tables +) + +SELECT * FROM final__rep_exchange__company_engagement_behaviors \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__company_lists.sql b/models/marts/reporting/rep_exchange__company_lists.sql new file mode 100644 index 0000000..af2131c --- /dev/null +++ b/models/marts/reporting/rep_exchange__company_lists.sql @@ -0,0 +1,78 @@ +-- AVAILALE FILTER VALUE +-- Source:preset_custom_filter Column:day_use_to_churn +{% set day_use_to_churn = 61 %} + +-- IMPORT +WITH companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + +-- LOGIC + +company_agg AS ( + + SELECT + companies.company_id, + companies.is_migrated_company, + companies.company_marking_id, + companies.company_created_datetime, + COUNT(transaction_orders.transaction_order_id) AS count_order, + COUNT(IFF(transaction_orders.order_status = 'completed', transaction_orders.transaction_order_id, null)) AS count_completed_order, + MAX(transaction_orders.order_created_datetime) AS last_order_datetime, + COALESCE(last_order_datetime, company_created_datetime) AS last_activity_datetime, + IFF(companies.company_created_datetime < '2021-07-01', 1, 0) AS is_wrong_created_datetime_company, + IFF(MAX(transaction_orders.order_created_datetime) IS NULL, 1, 0) AS is_never_order_company, + IFF(DATEDIFF(day, last_activity_datetime, CURRENT_DATE()) >= {{day_use_to_churn}}, 1, 0 ) AS is_churn_company, + IFF( is_churn_company = 1, DATEDIFF(day, company_created_datetime, last_activity_datetime) + {{day_use_to_churn}}, NULL) AS create_to_churn_day, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM + companies + + LEFT JOIN transaction_orders + ON (companies.company_id = transaction_orders.company_id) + + GROUP BY + companies.company_id, + companies.is_migrated_company, + companies.company_marking_id, + companies.company_created_datetime +), + +-- FINAL +final__rep_exchange__company_lists AS ( + SELECT + -- ids + company_id, + company_marking_id, + + -- dimensions + is_migrated_company, + is_wrong_created_datetime_company, + is_never_order_company, + is_churn_company, + + -- measures + count_order, + count_completed_order, + create_to_churn_day, + + + -- date/times + company_created_datetime, + last_order_datetime, + last_activity_datetime, + + -- metadata + _dbt_ran_datetime + + FROM company_agg +) + + +SELECT * FROM final__rep_exchange__company_lists \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__daily_orders.sql b/models/marts/reporting/rep_exchange__daily_orders.sql new file mode 100644 index 0000000..b1d41d7 --- /dev/null +++ b/models/marts/reporting/rep_exchange__daily_orders.sql @@ -0,0 +1,257 @@ +-- IMPORT +WITH transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + +bookings AS ( + SELECT * FROM {{ ref('fct_exchange__bookings') }} +), + +transaction_costs AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_costs') }} +), + +companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + +currencies AS ( + SELECT * FROM {{ ref('dim_exchange__currencies') }} +), + +dates AS ( + SELECT * FROM {{ ref('dim__dates') }} +), + + +--LOGIC +transaction_orders_get_next_order_created_datetime AS ( + SELECT + *, + LEAD(order_created_datetime) OVER (PARTITION BY company_id ORDER BY order_created_datetime) AS next_order_created_datetime + + FROM + transaction_orders +), + +fct_and_dim_joins AS ( + SELECT + bookings.booking_id, + bookings.booking_marking_id, + bookings.company_id, + + companies.company_marking_id AS company_marking_id, + companies.autocount_id AS company_autocount_id, + companies.company_type AS company_type, + companies.business_type AS company_business_type, + companies.exchange_rate_segment AS company_exchange_rate_segment, + companies.is_migrated_company AS is_migrated_company, + companies.country_name AS company_country_name, + companies.state_name AS company_state_name, + companies.district_name AS company_district_name, + companies.postcode AS company_postcode, + companies.has_wallet AS company_has_wallet, + + bookings.user_id, + bookings.bank_id, + + fix_currencies.currency_name AS fix_currency_name, + + quote_currencies.currency_name AS quote_currency_name, + + base_currencies.currency_name AS base_currency_name, + + bookings.service_type, + bookings.status AS booking_status, + bookings.is_first_time_company AS is_first_time_booking_company, + bookings.is_first_time_company_completed AS is_first_time_booking_company_completed, + bookings.is_first_time_user AS is_first_time_booking_user, + bookings.is_first_time_user_completed AS is_first_time_booking_user_completed, + bookings.estimate_base_to_quote_currency_exchange_rate AS estimate_booking_base_to_quote_currency_exchange_rate, + bookings.estimate_quote_value AS estimate_booking_quote_value, + bookings.estimate_base_value AS estimate_booking_base_value, + bookings.estimate_value_rm AS estimate_booking_value_rm, + bookings.booking_created_datetime, + bookings.booking_approved_datetime, + bookings.booking_completed_datetime, + bookings.booking_suspended_datetime, + + transaction_orders_get_next_order_created_datetime.transaction_order_id AS order_id, + transaction_orders_get_next_order_created_datetime.payment_method AS order_payment_method, + transaction_orders_get_next_order_created_datetime.order_status, + transaction_orders_get_next_order_created_datetime.payment_reference AS order_payment_reference, + transaction_orders_get_next_order_created_datetime.bill_number AS order_bill_number, + transaction_orders_get_next_order_created_datetime.is_first_time_company AS is_first_time_order_company, + transaction_orders_get_next_order_created_datetime.is_first_time_company_completed AS is_first_time_order_company_completed, + transaction_orders_get_next_order_created_datetime.is_first_time_user AS is_first_time_order_user, + transaction_orders_get_next_order_created_datetime.is_first_time_user_completed AS is_first_time_order_user_completed, + transaction_orders_get_next_order_created_datetime.base_to_quote_currency_exchange_rate AS order_base_to_quote_currency_exchange_rate , + transaction_orders_get_next_order_created_datetime.base_value AS order_base_value , + transaction_orders_get_next_order_created_datetime.base_service_charge AS order_base_service_charge , + transaction_orders_get_next_order_created_datetime.base_tax AS order_base_tax , + transaction_orders_get_next_order_created_datetime.quote_value AS order_quote_value , + transaction_orders_get_next_order_created_datetime.value_rm AS order_value_rm , + transaction_orders_get_next_order_created_datetime.service_charge_rm AS order_service_charge_rm , + transaction_orders_get_next_order_created_datetime.tax_rm AS order_tax_rm , + transaction_orders_get_next_order_created_datetime.total_value_rm AS total_order_value_rm, + + transaction_orders_get_next_order_created_datetime.order_created_datetime, + order_created_dates.first_day_of_week AS order_created_week, + + transaction_orders_get_next_order_created_datetime.order_pending_submission_datetime, + transaction_orders_get_next_order_created_datetime.order_pending_verification_datetime, + transaction_orders_get_next_order_created_datetime.order_approved_datetime, + transaction_orders_get_next_order_created_datetime.order_completed_datetime, + transaction_orders_get_next_order_created_datetime.order_rejected_datetime, + transaction_orders_get_next_order_created_datetime.order_suspended_datetime, + transaction_orders_get_next_order_created_datetime.next_order_created_datetime, + + transaction_costs.transaction_cost_id AS cost_id, + transaction_costs.bank_id AS currency_vendor_bank_id, + transaction_costs.transaction_type AS cost_transaction_type, + transaction_costs.payment_method AS cost_payment_method, + transaction_costs.status AS cost_status, + transaction_costs.payment_reference AS cost_payment_reference, + transaction_costs.bill_number AS cost_bill_number, + transaction_costs.base_value AS cost_base_value, + transaction_costs.quote_value AS cost_quote_value, + transaction_costs.base_to_quote_currency_exchange_rate AS cost_base_to_quote_currency_exchange_rate, + transaction_costs.base_tax AS cost_base_tax, + transaction_costs.base_service_charge AS cost_base_service_charge, + transaction_costs.value_rm AS cost_value_rm, + transaction_costs.tax_rm AS cost_tax_rm, + transaction_costs.service_charge_rm AS cost_service_charge_rm, + transaction_costs.total_value_rm AS total_cost_value_rm, + transaction_costs.cost_created_datetime, + transaction_costs.cost_pending_submission_datetime, + transaction_costs.cost_pending_verification_datetime, + transaction_costs.cost_approved_datetime, + transaction_costs.cost_completed_datetime, + transaction_costs.cost_rejected_datetime, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM bookings + + LEFT JOIN transaction_orders_get_next_order_created_datetime + ON (bookings.booking_id = transaction_orders_get_next_order_created_datetime.booking_id) + + LEFT JOIN transaction_costs + ON (transaction_orders_get_next_order_created_datetime.transaction_order_id = transaction_costs.transaction_order_id) + + LEFT JOIN companies + ON (bookings.company_id = companies.company_id) + + LEFT JOIN dates AS order_created_dates + ON (transaction_orders_get_next_order_created_datetime.order_created_datetime = order_created_dates.date_day) + + LEFT JOIN currencies AS fix_currencies + ON (bookings.fix_currency_id = fix_currencies.currency_id) + + LEFT JOIN currencies AS quote_currencies + ON (bookings.quote_currency_id = quote_currencies.currency_id) + + LEFT JOIN currencies AS base_currencies + ON (bookings.base_currency_id = base_currencies.currency_id) +), + +--FINAL +final__rep_exchange__daily_orders AS ( + SELECT + -- ids + order_id, + bank_id, + company_id, + company_marking_id, + company_autocount_id, + user_id, + cost_id, + currency_vendor_bank_id, + booking_id, + booking_marking_id, + + -- dimensions + fix_currency_name, + quote_currency_name, + base_currency_name, + service_type, + order_payment_method, + order_status, + order_payment_reference, + order_bill_number, + is_first_time_order_company, + is_first_time_order_company_completed, + is_first_time_order_user, + is_first_time_order_user_completed, + company_exchange_rate_segment, + company_type, + company_business_type, + is_migrated_company, + company_country_name, + company_state_name, + company_district_name, + company_postcode, + company_has_wallet, + cost_transaction_type, + cost_payment_method, + cost_status, + cost_payment_reference, + cost_bill_number, + booking_status, + is_first_time_booking_company, + is_first_time_booking_company_completed, + is_first_time_booking_user, + is_first_time_booking_user_completed, + + -- measures + order_base_to_quote_currency_exchange_rate, + order_base_value, + order_base_service_charge, + order_base_tax, + order_quote_value, + order_value_rm, + order_service_charge_rm, + order_tax_rm, + total_order_value_rm, + cost_base_value, + cost_quote_value, + cost_base_to_quote_currency_exchange_rate, + cost_base_tax, + cost_base_service_charge, + cost_value_rm, + cost_tax_rm, + cost_service_charge_rm, + total_cost_value_rm, + estimate_booking_base_to_quote_currency_exchange_rate, + estimate_booking_quote_value, + estimate_booking_base_value, + estimate_booking_value_rm, + + -- date/times + order_created_datetime, + order_created_week, + order_pending_submission_datetime, + order_pending_verification_datetime, + order_approved_datetime, + order_completed_datetime, + order_rejected_datetime, + order_suspended_datetime, + next_order_created_datetime, + cost_created_datetime, + cost_pending_submission_datetime, + cost_pending_verification_datetime, + cost_approved_datetime, + cost_completed_datetime, + cost_rejected_datetime, + booking_created_datetime, + booking_approved_datetime, + booking_completed_datetime, + booking_suspended_datetime, + + -- metadata + _dbt_ran_datetime + + FROM fct_and_dim_joins +) + +SELECT * FROM final__rep_exchange__daily_orders \ No newline at end of file diff --git a/models/marts/reporting/rep_exchange__rfm.sql b/models/marts/reporting/rep_exchange__rfm.sql new file mode 100644 index 0000000..b40cf87 --- /dev/null +++ b/models/marts/reporting/rep_exchange__rfm.sql @@ -0,0 +1,290 @@ +-- AVAILALE FILTER VALUE +{% set custom_one_year_from_date = modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")).date() - modules.datetime.timedelta(365) %} +{% set custom_one_year_to_date = modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")).date() %} + +{% set recency_list_lifetime = [60,120,180,240] %} +{% set frequency_list_lifetime = [1,3,9,27] %} +{% set monetary_list_lifetime = [2000,7000,25000,82000] %} + +{% set recency_list_one_year = [60,120,180,240] %} +{% set frequency_list_one_year = [1,2,6,21] %} +{% set monetary_list_one_year = [2000,7000,25000,77000] %} + +-- IMPORT +WITH transaction_orders AS ( + SELECT * FROM {{ ref('fct_exchange__transaction_orders') }} +), + + +companies AS ( + SELECT * FROM {{ ref('dim_exchange__companies') }} +), + + +-- LOGIC +transaction_order_one_years AS ( + SELECT + company_id, + MAX(transaction_orders.order_created_datetime) AS last_approved_or_completed_order_created_datetime_one_year, + DATEDIFF(day, last_approved_or_completed_order_created_datetime_one_year, CURRENT_DATE() ) AS day_after_approved_or_completed_order_one_year, + COUNT(transaction_orders.transaction_order_id) AS number_of_approved_or_completed_order_one_year, + ROUND(SUM(transaction_orders.value_rm + transaction_orders.tax_rm + transaction_orders.service_charge_rm),2) AS total_value_of_approved_or_completed_order_one_year + + FROM + transaction_orders + + WHERE + order_status IN ('APPROVED','COMPLETED') + AND + order_created_datetime >= '{{ custom_one_year_from_date }}' + AND + order_created_datetime < '{{ custom_one_year_to_date }}' + + GROUP BY + company_id +), + +company_rfm AS ( + SELECT + transaction_orders.company_id, + companies.company_marking_id AS company_marking_id, + MAX(transaction_orders.order_created_datetime) AS last_approved_or_completed_order_created_datetime, + + DATEDIFF(day, last_approved_or_completed_order_created_datetime, CURRENT_DATE() ) AS day_after_approved_or_completed_order_lifetime, + MAX(transaction_order_one_years.day_after_approved_or_completed_order_one_year) AS day_after_approved_or_completed_order_one_year, + + COUNT(transaction_orders.transaction_order_id) AS number_of_approved_or_completed_order_lifetime, + MAX(transaction_order_one_years.number_of_approved_or_completed_order_one_year) AS number_of_approved_or_completed_order_one_year, + + ROUND(SUM(transaction_orders.value_rm + transaction_orders.tax_rm + transaction_orders.service_charge_rm),2) AS total_value_of_approved_or_completed_order_lifetime, + MAX(transaction_order_one_years.total_value_of_approved_or_completed_order_one_year) AS total_value_of_approved_or_completed_order_one_year + + FROM transaction_orders + + LEFT JOIN transaction_order_one_years + ON (transaction_orders.company_id = transaction_order_one_years.company_id) + + LEFT JOIN companies + ON (transaction_orders.company_id = companies.company_id) + + + WHERE + transaction_orders.order_status IN ('APPROVED','COMPLETED') + + GROUP BY + transaction_orders.company_id, + companies.company_marking_id +), + +calculate_rfm AS ( + SELECT + *, + + + CASE + WHEN day_after_approved_or_completed_order_lifetime <= {{recency_list_lifetime[0]}} THEN 5 + WHEN (day_after_approved_or_completed_order_lifetime > {{recency_list_lifetime[0]}} AND day_after_approved_or_completed_order_lifetime <= {{recency_list_lifetime[1]}}) THEN 4 + WHEN (day_after_approved_or_completed_order_lifetime > {{recency_list_lifetime[1]}} AND day_after_approved_or_completed_order_lifetime <= {{recency_list_lifetime[2]}}) THEN 3 + WHEN (day_after_approved_or_completed_order_lifetime > {{recency_list_lifetime[2]}} AND day_after_approved_or_completed_order_lifetime <= {{recency_list_lifetime[3]}}) THEN 2 + WHEN day_after_approved_or_completed_order_lifetime > {{recency_list_lifetime[3]}} THEN 1 + ELSE -999 + END AS r_score_lifetime, + CASE + WHEN number_of_approved_or_completed_order_lifetime <= {{frequency_list_lifetime[0]}} THEN 1 + WHEN (number_of_approved_or_completed_order_lifetime > {{frequency_list_lifetime[0]}} AND number_of_approved_or_completed_order_lifetime <= {{frequency_list_lifetime[1]}}) THEN 2 + WHEN (number_of_approved_or_completed_order_lifetime > {{frequency_list_lifetime[1]}} AND number_of_approved_or_completed_order_lifetime <= {{frequency_list_lifetime[2]}}) THEN 3 + WHEN (number_of_approved_or_completed_order_lifetime > {{frequency_list_lifetime[2]}} AND number_of_approved_or_completed_order_lifetime <= {{frequency_list_lifetime[3]}}) THEN 4 + WHEN number_of_approved_or_completed_order_lifetime > {{frequency_list_lifetime[3]}} THEN 5 + ELSE -999 + END AS f_score_lifetime, + CASE + WHEN total_value_of_approved_or_completed_order_lifetime <= {{monetary_list_lifetime[0]}} THEN 1 + WHEN (total_value_of_approved_or_completed_order_lifetime > {{monetary_list_lifetime[0]}} AND total_value_of_approved_or_completed_order_lifetime <= {{monetary_list_lifetime[1]}}) THEN 2 + WHEN (total_value_of_approved_or_completed_order_lifetime > {{monetary_list_lifetime[1]}} AND total_value_of_approved_or_completed_order_lifetime <= {{monetary_list_lifetime[2]}}) THEN 3 + WHEN (total_value_of_approved_or_completed_order_lifetime > {{monetary_list_lifetime[2]}} AND total_value_of_approved_or_completed_order_lifetime <= {{monetary_list_lifetime[3]}}) THEN 4 + WHEN total_value_of_approved_or_completed_order_lifetime > {{monetary_list_lifetime[3]}} THEN 5 + ELSE -999 + END AS m_score_lifetime, + + CASE + WHEN (r_score_lifetime = 5) AND (f_score_lifetime = 5) AND (m_score_lifetime = 5) + THEN '1_King' + + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime = 1) AND (m_score_lifetime = 1) + THEN '10_At Cheap Risk' + + WHEN (r_score_lifetime = 1) AND (f_score_lifetime = 1) AND (m_score_lifetime = 1) + THEN '13_Cheap Lost' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 1 AND 2) AND (m_score_lifetime BETWEEN 4 AND 5) + THEN '4_Whale' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 4 AND 5) AND (m_score_lifetime BETWEEN 1 AND 2) + THEN '3_Sardine' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 3 AND 5) AND (m_score_lifetime BETWEEN 3 AND 5) + THEN '2_Loyal Customer' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 2 AND 5) AND (m_score_lifetime BETWEEN 2 AND 5) + THEN '5_Potention Loyal' + + WHEN (r_score_lifetime = 5) AND (f_score_lifetime = 1) AND (m_score_lifetime BETWEEN 1 AND 2) + THEN '7_New Customer' + + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime BETWEEN 4 AND 5) AND (m_score_lifetime BETWEEN 4 AND 5) + THEN '8_At High Risk' + + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime BETWEEN 1 AND 5) AND (m_score_lifetime BETWEEN 1 AND 5) + THEN '9_At Risk' + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime = 1) AND (m_score_lifetime BETWEEN 1 AND 2) + THEN '9_At Risk' + WHEN (r_score_lifetime BETWEEN 2 AND 3) AND (f_score_lifetime BETWEEN 1 AND 2) AND (m_score_lifetime = 1) + THEN '9_At Risk' + + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 4 AND 5) AND (m_score_lifetime BETWEEN 1 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 1 AND 5) AND (m_score_lifetime BETWEEN 4 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 3 AND 5) AND (m_score_lifetime BETWEEN 3 AND 5) + THEN '11_Cant Lose Them' + + WHEN (r_score_lifetime = 1) AND (f_score_lifetime BETWEEN 1 AND 3) AND (m_score_lifetime BETWEEN 1 AND 3) + THEN '12_Lost' + + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime = 1) AND (m_score_lifetime BETWEEN 1 AND 3) + THEN '6_Normal Customer' + WHEN (r_score_lifetime BETWEEN 4 AND 5) AND (f_score_lifetime BETWEEN 1 AND 3) AND (m_score_lifetime = 1) + THEN '6_Normal Customer' + + ELSE 'ERROR Please Contact DATA Team' + END AS segment_lifetime, + + + CASE + WHEN day_after_approved_or_completed_order_one_year IS NULL THEN NULL + WHEN day_after_approved_or_completed_order_one_year <= {{recency_list_one_year[0]}} THEN 5 + WHEN (day_after_approved_or_completed_order_one_year > {{recency_list_one_year[0]}} AND day_after_approved_or_completed_order_one_year <= {{recency_list_one_year[1]}}) THEN 4 + WHEN (day_after_approved_or_completed_order_one_year > {{recency_list_one_year[1]}} AND day_after_approved_or_completed_order_one_year <= {{recency_list_one_year[2]}}) THEN 3 + WHEN (day_after_approved_or_completed_order_one_year > {{recency_list_one_year[2]}} AND day_after_approved_or_completed_order_one_year <= {{recency_list_one_year[3]}}) THEN 2 + WHEN day_after_approved_or_completed_order_one_year > {{recency_list_one_year[3]}} THEN 1 + ELSE -999 + END AS r_score_one_year, + CASE + WHEN number_of_approved_or_completed_order_one_year IS NULL THEN NULL + WHEN number_of_approved_or_completed_order_one_year <= {{frequency_list_one_year[0]}} THEN 1 + WHEN (number_of_approved_or_completed_order_one_year > {{frequency_list_one_year[0]}} AND number_of_approved_or_completed_order_one_year <= {{frequency_list_one_year[1]}}) THEN 2 + WHEN (number_of_approved_or_completed_order_one_year > {{frequency_list_one_year[1]}} AND number_of_approved_or_completed_order_one_year <= {{frequency_list_one_year[2]}}) THEN 3 + WHEN (number_of_approved_or_completed_order_one_year > {{frequency_list_one_year[2]}} AND number_of_approved_or_completed_order_one_year <= {{frequency_list_one_year[3]}}) THEN 4 + WHEN number_of_approved_or_completed_order_one_year > {{frequency_list_one_year[3]}} THEN 5 + ELSE -999 + END AS f_score_one_year, + CASE + WHEN total_value_of_approved_or_completed_order_one_year IS NULL THEN NULL + WHEN total_value_of_approved_or_completed_order_one_year <= {{monetary_list_one_year[0]}} THEN 1 + WHEN (total_value_of_approved_or_completed_order_one_year > {{monetary_list_one_year[0]}} AND total_value_of_approved_or_completed_order_one_year <= {{monetary_list_one_year[1]}}) THEN 2 + WHEN (total_value_of_approved_or_completed_order_one_year > {{monetary_list_one_year[1]}} AND total_value_of_approved_or_completed_order_one_year <= {{monetary_list_one_year[2]}}) THEN 3 + WHEN (total_value_of_approved_or_completed_order_one_year > {{monetary_list_one_year[2]}} AND total_value_of_approved_or_completed_order_one_year <= {{monetary_list_one_year[3]}}) THEN 4 + WHEN total_value_of_approved_or_completed_order_one_year > {{monetary_list_one_year[3]}} THEN 5 + ELSE -999 + END AS m_score_one_year, + + CASE + WHEN (r_score_one_year IS NULL) AND (f_score_one_year IS NULL) AND (m_score_one_year IS NULL) + THEN '14_Not Active Between 1 year' + + WHEN (r_score_one_year = 5) AND (f_score_one_year = 5) AND (m_score_one_year = 5) + THEN '1_King' + + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year = 1) AND (m_score_one_year = 1) + THEN '10_At Cheap Risk' + + WHEN (r_score_one_year = 1) AND (f_score_one_year = 1) AND (m_score_one_year = 1) + THEN '13_Cheap Lost' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 1 AND 2) AND (m_score_one_year BETWEEN 4 AND 5) + THEN '4_Whale' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 4 AND 5) AND (m_score_one_year BETWEEN 1 AND 2) + THEN '3_Sardine' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 3 AND 5) AND (m_score_one_year BETWEEN 3 AND 5) + THEN '2_Loyal Customer' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 2 AND 5) AND (m_score_one_year BETWEEN 2 AND 5) + THEN '5_Potention Loyal' + + WHEN (r_score_one_year = 5) AND (f_score_one_year = 1) AND (m_score_one_year BETWEEN 1 AND 2) + THEN '7_New Customer' + + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year BETWEEN 4 AND 5) AND (m_score_one_year BETWEEN 4 AND 5) + THEN '8_At High Risk' + + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year BETWEEN 1 AND 5) AND (m_score_one_year BETWEEN 1 AND 5) + THEN '9_At Risk' + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year = 1) AND (m_score_one_year BETWEEN 1 AND 2) + THEN '9_At Risk' + WHEN (r_score_one_year BETWEEN 2 AND 3) AND (f_score_one_year BETWEEN 1 AND 2) AND (m_score_one_year = 1) + THEN '9_At Risk' + + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 4 AND 5) AND (m_score_one_year BETWEEN 1 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 1 AND 5) AND (m_score_one_year BETWEEN 4 AND 5) + THEN '11_Cant Lose Them' + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 3 AND 5) AND (m_score_one_year BETWEEN 3 AND 5) + THEN '11_Cant Lose Them' + + WHEN (r_score_one_year = 1) AND (f_score_one_year BETWEEN 1 AND 3) AND (m_score_one_year BETWEEN 1 AND 3) + THEN '12_Lost' + + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year = 1) AND (m_score_one_year BETWEEN 1 AND 3) + THEN '6_Normal Customer' + WHEN (r_score_one_year BETWEEN 4 AND 5) AND (f_score_one_year BETWEEN 1 AND 3) AND (m_score_one_year = 1) + THEN '6_Normal Customer' + + ELSE 'ERROR Please Contact DATA Team' + END AS segment_one_year, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + + FROM + company_rfm + +), + +-- FINAL +final__rep_exchange__rfm AS ( + SELECT + -- ids + company_id, + company_marking_id, + + -- dimensions + segment_lifetime, + segment_one_year, + + -- measures + day_after_approved_or_completed_order_lifetime, + number_of_approved_or_completed_order_lifetime, + total_value_of_approved_or_completed_order_lifetime, + r_score_lifetime, + f_score_lifetime, + m_score_lifetime, + day_after_approved_or_completed_order_one_year, + number_of_approved_or_completed_order_one_year, + total_value_of_approved_or_completed_order_one_year, + r_score_one_year, + f_score_one_year, + m_score_one_year, + + -- date/times + last_approved_or_completed_order_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM calculate_rfm +) +SELECT * FROM final__rep_exchange__rfm + + diff --git a/models/marts/warehouse/_warehouse_models.yml b/models/marts/warehouse/_warehouse_models.yml new file mode 100644 index 0000000..69662cd --- /dev/null +++ b/models/marts/warehouse/_warehouse_models.yml @@ -0,0 +1,47 @@ +version: 2 + +models: + - name: dim__dates + description: date dimension + columns: + - name: date_day + description: Primary key for 'dim_dates' + tests: + - unique + - not_null + + - name: dim_exchange__companies + description: exchange companies + columns: + - name: company_id + description: Primary key for 'dim_exchange__companies' + tests: + - unique + - not_null + + - name: fct_exchange__bookings + description: exchange bookings + columns: + - name: booking_id + description: Primary key for 'fct_exchange__bookings' + tests: + - unique + - not_null + + - name: fct_exchange__costs + description: exchange transaction_cost + columns: + - name: transaction_cost_id + description: Primary key for 'fct_exchange__transaction_costs' + tests: + - unique + - not_null + + - name: fct_exchange__transaction_orders + description: exchange transaction_orders + columns: + - name: transaction_order_id + description: Primary key for 'fct_exchange__transaction_orders' + tests: + - unique + - not_null diff --git a/models/marts/warehouse/dim__dates.sql b/models/marts/warehouse/dim__dates.sql new file mode 100644 index 0000000..7e071ca --- /dev/null +++ b/models/marts/warehouse/dim__dates.sql @@ -0,0 +1,201 @@ +-- Code and forum link refer: https://gitlab.com/gitlab-data/analytics/-/blob/master/transform/snowflake-dbt/models/sources/date/date_details_source.sql https://stackoverflow.com/questions/71016585/dbt-use-dbt-modeling-to-insert-rows-in-a-table-like-date-dimension-table-in-az +-- Import +WITH holiday_and_non_working_day_lists AS ( + SELECT * FROM {{ ref('seed_exchange__holiday_and_non_working_day_lists') }} +), + +-- LOGIC +date_spine AS ( + + {{ dbt_utils.date_spine( + start_date="to_date('01/01/2015', 'mm/dd/yyyy')", + datepart="day", + end_date="dateadd(year, 40, current_date)" + ) + }} + +), + +calculated as ( + + SELECT + date_day, + date_day AS date_actual, + + DAYNAME(date_day) AS day_name, + + DATE_PART('month', date_day) AS month_actual, + DATE_PART('year', date_day) AS year_actual, + + DATE_PART(quarter, date_day) AS quarter_actual, + + CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7 + ELSE DATE_PART(dayofweek, date_day) END AS day_of_week, + + CASE WHEN day_name = 'Mon' THEN date_day + ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week, + + CASE WHEN day_name = 'Mon' THEN WEEK(date_day) + ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column + + CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1' + THEN '1' + ELSE week_of_year_temp END AS week_of_year, + + DATE_PART('day', date_day) AS day_of_month, + + ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter, + ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year, + + -- @TODO @GERSON ,done + -- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023) + CASE WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual + when DATE_PART('month', DATEADD( month,-1, date_day) ) =12 THEN year_actual-1 + + ELSE (year_actual+1) END AS fiscal_year, + + -- @TODO @GERSON , done + -- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4) + CASE WHEN month_actual in (2,3,4) THEN '1' + WHEN month_actual in (5,6,7) THEN '2' + WHEN month_actual in (8,9,10) THEN '3' + WHEN month_actual in (11,12) THEN '4' + WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4' + -- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4' + ELSE '4' END AS fiscal_quarter, + + -- @TODO @GERSON ,done + ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter, + ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year, + + TO_CHAR(date_day, 'MMMM') AS month_name, + + TRUNC(date_day, 'Month') AS first_day_of_month, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month, + + FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year, + + FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter, + + -- @TODO @GERSON ,done + -- Change this 2 too + FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter, + LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter, + + -- @TODO @GERSON,done + FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year, + LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year, + + -- @TODO @GERSON ,done + DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year, + + -- @TODO @GERSON ,done + CASE WHEN EXTRACT('month', date_day) = 1 THEN 12 + ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year, + + LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week, + + (year_actual || '-Q' || fiscal_quarter) AS quarter_name, + + -- @TODO @GERSON,done + (fiscal_year || '-' || DECODE(fiscal_quarter, + 1, 'Q1', + 2, 'Q2', + 3, 'Q3', + 4, 'Q4')) AS fiscal_quarter_name, + ('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy, + DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute, + --here check + fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name, + ('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy, + + -- @TODO @GERSON + -- Use the Seed, we will not doing the hard code in SQL, we use CSV to refer the holiday and desc + holiday_and_non_working_day_lists.holiday_desc, + holiday_and_non_working_day_lists.is_holiday, + holiday_and_non_working_day_lists.holiday_race, + holiday_and_non_working_day_lists.holiday_country, + holiday_and_non_working_day_lists.holiday_state, + holiday_and_non_working_day_lists.is_non_working_day, + -- @TODO @GERSON + DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter, + IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter, + DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year, + IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year, + + DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa, + DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM date_spine + + LEFT JOIN holiday_and_non_working_day_lists + ON date_spine.date_day = to_date(holiday_and_non_working_day_lists.date, 'dd/mm/yyyy') + + QUALIFY date_actual >= '2016-01-01' + + ORDER BY date_day + + +), + +final AS ( + + SELECT + date_day, + date_actual, + day_name, + month_actual, + year_actual, + quarter_actual, + day_of_week, + first_day_of_week, + week_of_year, + day_of_month, + day_of_quarter, + day_of_year, + fiscal_year, + fiscal_quarter, + day_of_fiscal_quarter, + day_of_fiscal_year, + month_name, + first_day_of_month, + last_day_of_month, + first_day_of_year, + last_day_of_year, + first_day_of_quarter, + last_day_of_quarter, + first_day_of_fiscal_quarter, + last_day_of_fiscal_quarter, + first_day_of_fiscal_year, + last_day_of_fiscal_year, + week_of_fiscal_year, + month_of_fiscal_year, + last_day_of_week, + quarter_name, + fiscal_quarter_name, + fiscal_quarter_name_fy, + fiscal_quarter_number_absolute, + fiscal_month_name, + fiscal_month_name_fy, + holiday_desc, + is_holiday, + holiday_race, + holiday_country, + holiday_state, + is_non_working_day, + last_month_of_fiscal_quarter, + is_first_day_of_last_month_of_fiscal_quarter, + last_month_of_fiscal_year, + is_first_day_of_last_month_of_fiscal_year, + snapshot_date_fpa, + snapshot_date_billings, + _dbt_ran_datetime + FROM calculated + +) + +SELECT * FROM final \ No newline at end of file diff --git a/models/marts/warehouse/dim_crisp__users.sql b/models/marts/warehouse/dim_crisp__users.sql new file mode 100644 index 0000000..1ba9d33 --- /dev/null +++ b/models/marts/warehouse/dim_crisp__users.sql @@ -0,0 +1,5 @@ +WITH users AS ( + SELECT * FROM {{ ref('stg_crisp__users') }} +) + +SELECT * FROM users \ No newline at end of file diff --git a/models/marts/warehouse/dim_exchange__companies.sql b/models/marts/warehouse/dim_exchange__companies.sql new file mode 100644 index 0000000..1875a19 --- /dev/null +++ b/models/marts/warehouse/dim_exchange__companies.sql @@ -0,0 +1,197 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ ref('int_exchange__companies_join_segments') }} +), + +company_addresses AS ( + SELECT * FROM {{ ref('stg_exchange__company_addresses') }} +), + +company_wallets AS ( + SELECT * FROM {{ ref('stg_exchange__company_wallets') }} +), + +companies_brg_users AS ( + SELECT * FROM {{ ref('stg_exchange__companies_brg_users') }} +), + +company_contacts AS ( + SELECT * FROM {{ ref('stg_exchange__company_contacts') }} +), + +company_documents AS ( + SELECT * FROM {{ ref('stg_exchange__company_documents') }} +), + +currencies AS ( + SELECT * FROM {{ ref('stg_exchange__currencies') }} +), + +-- -- LOGICS +count_company_user AS ( + SELECT + company_id, + COUNT(user_id) AS count_user + + FROM companies_brg_users + + GROUP BY company_id +), + +company_billing_addresses AS ( + SELECT + * + + FROM company_addresses + + WHERE + is_billing_address = '1' +), + +latest_company_upload_identity_documents AS ( + SELECT + *, + row_number() OVER + (PARTITION BY company_id + ORDER BY created_datetime DESC) AS latest_row_number + + FROM company_documents + + WHERE + document_type IN ('IDENTITY_CARD','SSM_REGISTRATION') + AND + status = 'APPROVED' + + QUALIFY + latest_row_number = '1' + +), + +companies_rename_and_join_table AS ( + SELECT + companies.company_id, + companies.name, + companies.company_marking_id, + companies.autocount_id, + companies.company_type, + companies.business_type, + companies.exchange_rate_segment, + companies.is_migrated_company, + companies.status, + companies.created_datetime AS company_created_datetime, + + company_contacts.contact_id, + company_contacts.contact_name, + company_contacts.phone AS contact_phone, + company_contacts.email AS contact_email, + company_contacts.wechat AS contact_wechat, + + company_billing_addresses.company_address_id, + company_billing_addresses.country_id, + company_billing_addresses.country_name, + company_billing_addresses.country_short_code, + company_billing_addresses.state_id, + company_billing_addresses.state_name, + company_billing_addresses.district_id, + company_billing_addresses.district_name, + company_billing_addresses.postcode, + company_billing_addresses.address_line_one, + company_billing_addresses.address_line_two, + + company_wallets.wallet_id, + IFF(company_wallets.wallet_id IS NOT NULL, 1, 0) AS has_wallet, + company_wallets.amount AS wallet_balance, + company_wallets.currency_id AS wallet_currency_id, + currencies.name AS wallet_currency_name, + company_wallets.created_datetime AS wallet_created_datetime, + company_wallets.updated_datetime AS wallet_updated_datetime, + + latest_company_upload_identity_documents.document_id AS identity_document_id, + latest_company_upload_identity_documents.document_type AS identity_document_type, + latest_company_upload_identity_documents.reference AS identity_reference, + latest_company_upload_identity_documents.status AS identity_status, + latest_company_upload_identity_documents.approved_datetime AS identity_approved_datetime, + latest_company_upload_identity_documents.created_datetime AS identity_created_datetime, + + count_company_user.count_user AS number_of_user, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies + + LEFT JOIN company_billing_addresses + ON (companies.company_id = company_billing_addresses.company_id) + + LEFT JOIN company_wallets + ON (companies.company_id = company_wallets.company_id) + + LEFT JOIN count_company_user + ON (companies.company_id = count_company_user.company_id) + + LEFT JOIN company_contacts + ON (companies.company_id = company_contacts.company_id) + + LEFT JOIN latest_company_upload_identity_documents + ON (companies.company_id = latest_company_upload_identity_documents.company_id) + + LEFT JOIN currencies + ON (company_wallets.currency_id = currencies.currency_id) +), + +-- FINAL +final__dim_exchange__companies AS ( + SELECT + -- ids + company_id, + company_marking_id, + autocount_id, + contact_id, + company_address_id, + country_id, + state_id, + district_id, + wallet_id, + wallet_currency_id, + identity_document_id, + + -- dimensions + name, + company_type, + business_type, + exchange_rate_segment, + is_migrated_company, + status, + contact_name, + contact_phone, + contact_email, + contact_wechat, + country_name, + country_short_code, + state_name, + district_name, + postcode, + address_line_one, + address_line_two, + has_wallet, + wallet_currency_name, + identity_document_type, + identity_reference, + identity_status, + + -- measures + wallet_balance, + number_of_user, + + -- date/times + company_created_datetime, + wallet_created_datetime, + wallet_updated_datetime, + identity_approved_datetime, + identity_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM companies_rename_and_join_table +) + +SELECT * FROM final__dim_exchange__companies diff --git a/models/marts/warehouse/dim_exchange__currencies.sql b/models/marts/warehouse/dim_exchange__currencies.sql new file mode 100644 index 0000000..a25bdbf --- /dev/null +++ b/models/marts/warehouse/dim_exchange__currencies.sql @@ -0,0 +1,43 @@ +-- IMPORT +WITH currencies AS ( + SELECT * FROM {{ ref('stg_exchange__currencies') }} +), + +-- LOGIC +currencies_rename AS ( + SELECT + currencies.currency_id, + currencies.name AS currency_name, + currencies.short_code AS currency_short_code, + currencies.symbol AS currency_symbol, + currencies.country_name AS currency_country_name, + currencies.created_datetime AS currency_created_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM currencies +), + +-- FINAL +final__dim_exchange__currencies AS ( + SELECT + -- ids + currency_id, + + -- dimensions + currency_name, + currency_short_code, + currency_symbol, + currency_country_name, + + -- measures + + -- date/times + currency_created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM currencies_rename +) + +SELECT * FROM final__dim_exchange__currencies \ No newline at end of file diff --git a/models/crisp/marts/warehouse/fct_crisp__dialogs.sql b/models/marts/warehouse/fct_crisp__dialogs.sql similarity index 57% rename from models/crisp/marts/warehouse/fct_crisp__dialogs.sql rename to models/marts/warehouse/fct_crisp__dialogs.sql index e844829..efeb636 100644 --- a/models/crisp/marts/warehouse/fct_crisp__dialogs.sql +++ b/models/marts/warehouse/fct_crisp__dialogs.sql @@ -1,46 +1,44 @@ --IMPORT WITH messages AS ( - SELECT * FROM {{ ref('int_crisp__conversation_messages_get_dialog_id_user_id_message_generate_by') }} - ORDER BY conversation_id, message_created_datetime, id + SELECT * FROM {{ ref('int_crisp__messages_generate_dialog_ids') }} + ORDER BY conversation_id, created_datetime, message_id ), --LOGIC messages_with_ftu_id AS ( SELECT *, - FIRST_VALUE(CASE WHEN messages.message_generate_by = 'user' THEN messages.user_id END) + FIRST_VALUE(CASE WHEN messages.message_generated_by = 'user' THEN messages.user_id END) IGNORE NULLS - OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS first_user_id, - LAST_VALUE(CASE WHEN messages.message_generate_by = 'user' THEN messages.user_id END) + OVER (PARTITION BY messages.dialog_id ORDER BY messages.created_datetime ASC) AS first_user_id, + LAST_VALUE(CASE WHEN messages.message_generated_by = 'user' THEN messages.user_id END) IGNORE NULLS - OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS last_user_id, + OVER (PARTITION BY messages.dialog_id ORDER BY messages.created_datetime ASC) AS last_user_id, - FIRST_VALUE(CASE WHEN messages.message_generate_by = 'operator' THEN messages.user_id END) + FIRST_VALUE(CASE WHEN messages.message_generated_by = 'operator' THEN messages.user_id END) IGNORE NULLS - OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS first_operator_id, - LAST_VALUE(CASE WHEN messages.message_generate_by = 'operator' THEN messages.user_id END) + OVER (PARTITION BY messages.dialog_id ORDER BY messages.created_datetime ASC) AS first_operator_id, + LAST_VALUE(CASE WHEN messages.message_generated_by = 'operator' THEN messages.user_id END) IGNORE NULLS - OVER (PARTITION BY messages.dialog_id ORDER BY messages.message_created_datetime ASC) AS last_operator_id + OVER (PARTITION BY messages.dialog_id ORDER BY messages.created_datetime ASC) AS last_operator_id from messages ), dialog_message_last_row AS ( SELECT - RANK() OVER (PARTITION BY dialog_id ORDER BY message_created_datetime DESC) AS last_row_desc_rank, + RANK() OVER (PARTITION BY dialog_id ORDER BY created_datetime DESC) AS last_row_desc_rank, * FROM messages QUALIFY last_row_desc_rank = 1 ), ---FINAL - -final_fct_crisp__dialogs AS ( +transform_to_dialogs AS ( SELECT messages_with_ftu_id.dialog_id AS dialog_id, max(messages_with_ftu_id.conversation_id) AS conversation_id, - array_agg(messages_with_ftu_id.id) AS message_ids, + array_agg(messages_with_ftu_id.message_id) AS message_ids, CASE WHEN max(dialog_message_last_row.event_type) = 'state:resolved' THEN 'close' @@ -48,44 +46,44 @@ final_fct_crisp__dialogs AS ( ELSE 'ERROR, Please contact DATA team' END AS dialog_status, - min(messages_with_ftu_id.message_created_datetime) AS dialog_created_datetime, + min(messages_with_ftu_id.created_datetime) AS dialog_created_datetime, - min(IFF(messages_with_ftu_id.message_generate_by = 'user', - messages_with_ftu_id.message_created_datetime, + min(IFF(messages_with_ftu_id.message_generated_by = 'user', + messages_with_ftu_id.created_datetime, null)) AS first_user_reply_datetime, - min(IFF(messages_with_ftu_id.message_generate_by = 'operator', - messages_with_ftu_id.message_created_datetime, + min(IFF(messages_with_ftu_id.message_generated_by = 'operator', + messages_with_ftu_id.created_datetime, null)) AS first_operator_reply_datetime, - min(IFF(messages_with_ftu_id.message_generate_by = 'bot', - messages_with_ftu_id.message_created_datetime, + min(IFF(messages_with_ftu_id.message_generated_by = 'bot', + messages_with_ftu_id.created_datetime, null)) AS first_bot_reply_datetime, - max(IFF(messages_with_ftu_id.message_generate_by = 'user', - messages_with_ftu_id.message_created_datetime, + max(IFF(messages_with_ftu_id.message_generated_by = 'user', + messages_with_ftu_id.created_datetime, null)) AS last_user_reply_datetime, - max(IFF(messages_with_ftu_id.message_generate_by = 'operator', - messages_with_ftu_id.message_created_datetime, + max(IFF(messages_with_ftu_id.message_generated_by = 'operator', + messages_with_ftu_id.created_datetime, null)) AS last_operator_reply_datetime, - max(IFF(messages_with_ftu_id.message_generate_by = 'bot', - messages_with_ftu_id.message_created_datetime, + max(IFF(messages_with_ftu_id.message_generated_by = 'bot', + messages_with_ftu_id.created_datetime, null)) AS last_bot_reply_datetime, max(IFF((messages_with_ftu_id.event_type = 'state:resolved'), - messages_with_ftu_id.message_created_datetime, + messages_with_ftu_id.created_datetime, null)) AS dialog_solved_datetime, array_agg(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'user' , + IFF(messages_with_ftu_id.message_generated_by = 'user' , messages_with_ftu_id.user_id, NULL))) AS user_ids, count(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'user', + IFF(messages_with_ftu_id.message_generated_by = 'user', messages_with_ftu_id.user_id, null))) AS number_of_user_involved, @@ -94,12 +92,12 @@ final_fct_crisp__dialogs AS ( array_agg(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'operator' , + IFF(messages_with_ftu_id.message_generated_by = 'operator' , messages_with_ftu_id.user_id, NULL))) AS operator_ids, count(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'operator', + IFF(messages_with_ftu_id.message_generated_by = 'operator', messages_with_ftu_id.user_id, null))) AS number_of_operator_involved, @@ -108,19 +106,21 @@ final_fct_crisp__dialogs AS ( count(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'user', - messages_with_ftu_id.id, + IFF(messages_with_ftu_id.message_generated_by = 'user', + messages_with_ftu_id.message_id, null))) AS number_of_user_chat, count(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'operator', - messages_with_ftu_id.id, + IFF(messages_with_ftu_id.message_generated_by = 'operator', + messages_with_ftu_id.message_id, null))) AS number_of_operator_chat, count(distinct( - IFF(messages_with_ftu_id.message_generate_by = 'bot', - messages_with_ftu_id.id, - null))) AS number_of_bot_chat + IFF(messages_with_ftu_id.message_generated_by = 'bot', + messages_with_ftu_id.message_id, + null))) AS number_of_bot_chat, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM messages_with_ftu_id @@ -130,8 +130,47 @@ final_fct_crisp__dialogs AS ( GROUP BY messages_with_ftu_id.dialog_id +), + +-- FINAL +final_fct_crisp__dialogs AS ( + SELECT + -- ids + dialog_id, + conversation_id, + message_ids, + user_ids, + operator_ids, + first_user_id, + last_user_id, + first_operator_id, + last_operator_id, + + -- dimensions + dialog_status, + + -- measures + number_of_user_involved, + number_of_operator_involved, + number_of_user_chat, + number_of_operator_chat, + number_of_bot_chat, + + -- date/times + dialog_created_datetime, + dialog_solved_datetime, + first_user_reply_datetime, + first_operator_reply_datetime, + first_bot_reply_datetime, + last_user_reply_datetime, + last_operator_reply_datetime, + last_bot_reply_datetime, + + -- metadata + _dbt_ran_datetime + + + FROM transform_to_dialogs ) -SELECT * FROM final_fct_crisp__dialogs - ---Check 'K Huan Huan' why result number_of_user_id = 2 session_id = 'session_75a3a65d-390b-4ed9-90a7-f98a8a39b768'; \ No newline at end of file +SELECT * FROM final_fct_crisp__dialogs \ No newline at end of file diff --git a/models/marts/warehouse/fct_exchange__bookings.sql b/models/marts/warehouse/fct_exchange__bookings.sql new file mode 100644 index 0000000..dbf8a39 --- /dev/null +++ b/models/marts/warehouse/fct_exchange__bookings.sql @@ -0,0 +1,107 @@ +-- IMPORTS +{% set booking_status = dbt_utils.get_column_values( + table=ref('stg_exchange__booking_logs'), + column='status') %} + +WITH bookings AS ( + SELECT * FROM {{ ref('int_exchange__bookings_get_estimate_booking_values') }} +), + +booking_logs AS ( + SELECT * FROM {{ ref('stg_exchange__booking_logs') }} +), + +-- LOGICS +booking_status_datetime AS ( + + SELECT + booking_id, + MIN(created_datetime) AS booking_created_datetime, + {%- for status in booking_status %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS booking_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM booking_logs + + GROUP BY + booking_id +), + +bookings_lists AS ( + SELECT + bookings.*, + + {% for column_name in ["company", "user"] %} + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY bookings.{{column_name}}_id ORDER BY bookings.created_datetime) = 1 THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}, + + CASE + WHEN bookings.status = 'COMPLETED' + AND + MIN(IFF(bookings.status = 'COMPLETED', bookings.created_datetime, NULL)) + OVER (PARTITION BY bookings.{{column_name}}_id) + = bookings.created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}_completed, + {% endfor %} + + booking_status_datetime.booking_created_datetime, + {%- for status in booking_status %} + booking_status_datetime.booking_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM + bookings + + LEFT JOIN booking_status_datetime + ON (bookings.booking_id = booking_status_datetime.booking_id) +), + +-- FINAL +final_fct_exchange__bookings AS ( + SELECT + -- ids + booking_id, + booking_marking_id, + company_id, + user_id, + bank_id, + fix_currency_id, + quote_currency_id, + base_currency_id, + + -- dimensions + service_type, + estimate_payment_method, + status, + is_first_time_company, + is_first_time_company_completed, + is_first_time_user, + is_first_time_user_completed, + + -- measures + estimate_base_to_quote_currency_exchange_rate, + estimate_quote_value, + estimate_base_value, + estimate_value_rm, + + -- date/times + booking_created_datetime, + booking_approved_datetime, + booking_completed_datetime, + booking_suspended_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM bookings_lists +) + +SELECT * FROM final_fct_exchange__bookings + diff --git a/models/exchange/marts/warehouse/fct_exchange__company_identity_documents.sql b/models/marts/warehouse/fct_exchange__company_identity_documents.sql similarity index 56% rename from models/exchange/marts/warehouse/fct_exchange__company_identity_documents.sql rename to models/marts/warehouse/fct_exchange__company_identity_documents.sql index 526c1af..f1482ab 100644 --- a/models/exchange/marts/warehouse/fct_exchange__company_identity_documents.sql +++ b/models/marts/warehouse/fct_exchange__company_identity_documents.sql @@ -1,6 +1,6 @@ --IMPORT With documents AS ( - SELECT * FROM {{ ref('stg_exchange__documents') }} + SELECT * FROM {{ ref('stg_exchange__company_documents') }} ), --LOGIC @@ -10,9 +10,7 @@ company_identity_documents AS ( FROM documents - WHERE - owner_type = 'App\\Models\\Company' - AND + WHERE document_type IN ('IDENTITY_CARD','SSM_REGISTRATION') ), @@ -20,17 +18,15 @@ company_identity_documents AS ( --FINAL final__fct_exchange__company_identity_documents AS ( SELECT - id, - owner_type, - owner_id AS company_id, + document_id, + company_id, document_type AS identity_document_type_name, reference AS identity_number, status AS identity_status, - status_name AS identity_status_name, - approved_by AS identity_approved_by, - deleted_at AS identity_deleted_datetine, - approved_at AS identity_approved_datetine, - created_at AS identity_created_datetime + approver_user_id AS identity_approver_user_id, + deleted_datetime AS identity_deleted_datetine, + approved_datetime AS identity_approved_datetine, + created_datetime AS identity_created_datetime FROM company_identity_documents diff --git a/models/marts/warehouse/fct_exchange__transaction_costs.sql b/models/marts/warehouse/fct_exchange__transaction_costs.sql new file mode 100644 index 0000000..62d85cc --- /dev/null +++ b/models/marts/warehouse/fct_exchange__transaction_costs.sql @@ -0,0 +1,134 @@ +-- IMPORTS +{% set cost_status = dbt_utils.get_column_values( + table=ref('stg_exchange__transaction_order_logs'), + column='status') %} + +WITH transaction_costs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_costs') }} +), + +transaction_cost_logs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_cost_logs') }} +), + +-- LOGICS +transaction_cost_status_datetime AS ( + + SELECT + transaction_cost_id, + MIN(created_datetime) AS cost_created_datetime, + + {%- for status in cost_status %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS cost_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM transaction_cost_logs + + GROUP BY + transaction_cost_id +), + +transaction_costs_lists AS ( + SELECT + transaction_costs.transaction_cost_id, + transaction_costs.transaction_order_id, + transaction_costs.supplier_company_id, + transaction_costs.bank_id, + transaction_costs.base_currency_id, + transaction_costs.quote_currency_id, + transaction_costs.transaction_type, + transaction_costs.payment_method, + transaction_costs.status, + transaction_costs.payment_reference, + transaction_costs.bill_number, + transaction_costs.base_value, + transaction_costs.quote_value, + transaction_costs.base_to_quote_currency_exchange_rate, + transaction_costs.base_tax, + transaction_costs.base_service_charge, + + -- use for future if base currency id other from '1' + IFF(transaction_costs.base_currency_id = '1', + transaction_costs.base_value, + -9999999999) AS value_rm, + + -- use for future if base currency id other from '1' + IFF(transaction_costs.base_currency_id = '1', + transaction_costs.base_tax, + -9999999999) AS tax_rm, + + -- use for future if base currency id other from '1' + IFF(transaction_costs.base_currency_id = '1', + transaction_costs.base_service_charge, + -9999999999) AS service_charge_rm, + + (value_rm + service_charge_rm + tax_rm) AS total_value_rm, + + transaction_costs.expired_datetime, + transaction_costs.deleted_datetime, + transaction_costs.created_datetime, + transaction_costs.updated_datetime, + + transaction_cost_status_datetime.cost_created_datetime, + {%- for status in cost_status %} + transaction_cost_status_datetime.cost_{{status.lower()}}_datetime, + {% endfor %} + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_costs + + LEFT JOIN transaction_cost_status_datetime + ON (transaction_costs.transaction_cost_id = transaction_cost_status_datetime.transaction_cost_id) +), + + +-- FINAL +final__fct_exchange__transaction_costs AS ( + SELECT + -- ids + transaction_cost_id, + transaction_order_id, + supplier_company_id, + bank_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + value_rm, + tax_rm, + service_charge_rm, + total_value_rm, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + cost_created_datetime, + cost_pending_submission_datetime, + cost_pending_verification_datetime, + cost_approved_datetime, + cost_completed_datetime, + cost_rejected_datetime, + cost_suspended_datetime + + -- metadata + _dbt_ran_datetime + + FROM transaction_costs_lists +) + +SELECT * FROM final__fct_exchange__transaction_costs \ No newline at end of file diff --git a/models/marts/warehouse/fct_exchange__transaction_orders.sql b/models/marts/warehouse/fct_exchange__transaction_orders.sql new file mode 100644 index 0000000..4866782 --- /dev/null +++ b/models/marts/warehouse/fct_exchange__transaction_orders.sql @@ -0,0 +1,158 @@ +-- IMPORTS +{% set order_status = dbt_utils.get_column_values( + table=ref('stg_exchange__transaction_order_logs'), + column='status') %} + +WITH transaction_orders AS ( + SELECT * FROM {{ ref('int_exchange__transaction_orders_get_user_ids') }} +), + +transaction_order_logs AS ( + SELECT * FROM {{ ref('stg_exchange__transaction_order_logs') }} +), + +-- LOGICS +transaction_order_order_status_datetime AS ( + + SELECT + transaction_order_id, + MIN(created_datetime) AS order_created_datetime, + + {%- for status in order_status %} + MAX(CASE WHEN status = '{{status}}' THEN updated_datetime END) AS order_{{status.lower()}}_datetime {%- if not loop.last %},{% endif -%} + {% endfor %} + + FROM transaction_order_logs + + GROUP BY + transaction_order_id +), + + +transaction_orders_lists AS ( + SELECT + transaction_orders.transaction_order_id, + transaction_orders.booking_id, + transaction_orders.bank_id, + transaction_orders.company_id, + transaction_orders.user_id, + transaction_orders.status AS order_status, + transaction_orders.transaction_type, + transaction_orders.payment_method, + transaction_orders.base_currency_id, + transaction_orders.quote_currency_id, + transaction_orders.base_to_quote_currency_exchange_rate, + transaction_orders.payment_reference, + transaction_orders.bill_number, + transaction_orders.expired_datetime AS order_expired_datetime, + + transaction_orders.base_value, + transaction_orders.quote_value, + -- use for future if base currency id other from '1' + IFF(transaction_orders.base_currency_id = '1', + transaction_orders.base_value, + -9999999999) AS value_rm, + + transaction_orders.base_tax AS base_tax, + -- use for future if base currency id other from '1' + IFF(transaction_orders.base_currency_id = '1', + transaction_orders.base_tax, + -9999999999) AS tax_rm, + + transaction_orders.base_service_charge, + -- use for future if base currency id other from '1' + IFF(transaction_orders.base_currency_id = '1', + transaction_orders.base_service_charge, + -9999999999) AS service_charge_rm, + + (value_rm + service_charge_rm + tax_rm) AS total_value_rm, + + transaction_orders.deleted_datetime AS order_deleted_datetime, + transaction_orders.updated_datetime AS order_updated_datetime, + + {% for column_name in ["company", "user"] %} + CASE + WHEN ROW_NUMBER() OVER (PARTITION BY transaction_orders.{{column_name}}_id ORDER BY transaction_orders.created_datetime) = 1 THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}, + + CASE + WHEN transaction_orders.status = 'COMPLETED' + AND + MIN(IFF(transaction_orders.status = 'COMPLETED', transaction_orders.created_datetime, NULL)) + OVER (PARTITION BY transaction_orders.{{column_name}}_id) + = transaction_orders.created_datetime + THEN 1 + ELSE 0 + END AS is_first_time_{{column_name}}_completed, + {% endfor %} + + transaction_order_order_status_datetime.order_created_datetime, + + {%- for status in order_status %} + transaction_order_order_status_datetime.order_{{status.lower()}}_datetime, + {% endfor %} + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_orders + + LEFT JOIN transaction_order_order_status_datetime + ON (transaction_orders.transaction_order_id = transaction_order_order_status_datetime.transaction_order_id) +), + +-- FINAL +final__fct_exchange__transaction_orders AS ( + SELECT + -- ids + transaction_order_id, + booking_id, + company_id, + user_id, + bank_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + order_status, + payment_reference, + bill_number, + is_first_time_company, + is_first_time_company_completed, + is_first_time_user, + is_first_time_user_completed, + + -- measures + base_to_quote_currency_exchange_rate, + base_value, + base_service_charge, + base_tax, + + quote_value, + + value_rm, + service_charge_rm, + tax_rm, + total_value_rm, + + -- date/times + order_expired_datetime, + order_deleted_datetime, + order_updated_datetime, + order_created_datetime, + order_pending_submission_datetime, + order_pending_verification_datetime, + order_approved_datetime, + order_completed_datetime, + order_rejected_datetime, + order_suspended_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transaction_orders_lists +) + +SELECT * FROM final__fct_exchange__transaction_orders \ No newline at end of file diff --git a/models/marts/warehouse/fct_exchange__user_email_verifications.sql b/models/marts/warehouse/fct_exchange__user_email_verifications.sql new file mode 100644 index 0000000..d5355d2 --- /dev/null +++ b/models/marts/warehouse/fct_exchange__user_email_verifications.sql @@ -0,0 +1,57 @@ +--IMPORT +WITH user_email_verifications AS ( + SELECT * FROM {{ ref('stg_exchange__user_email_verifications') }} +), + +companies_brg_users AS ( + SELECT * FROM {{ ref('stg_exchange__companies_brg_users') }} +), + + +--LOGIC +user_email_verifications_get_company_id AS ( + SELECT + user_email_verifications.user_email_verification_id, + user_email_verifications.user_id, + companies_brg_users.company_id, + user_email_verifications.email, + user_email_verifications.is_sent, + user_email_verifications.is_completed, + user_email_verifications.created_datetime AS user_email_verification_created_datetime, + user_email_verifications.updated_datetime AS user_email_verification_updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM user_email_verifications + + LEFT JOIN companies_brg_users + ON (user_email_verifications.user_id = companies_brg_users.user_id) +), + + +--FINAL +final__fct_exchange__company_email_verifications AS ( + SELECT + -- ids + user_email_verification_id, + user_id, + company_id, + + -- dimensions + email, + is_sent, + is_completed, + + -- measures + + -- date/times + user_email_verification_created_datetime, + user_email_verification_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM user_email_verifications_get_company_id +) + + +SELECT * FROM final__fct_exchange__company_email_verifications \ No newline at end of file diff --git a/models/shipping/base/addresses/base_shipping__addresses.sql b/models/shipping/base/addresses/base_shipping__addresses.sql deleted file mode 100644 index fc756a3..0000000 --- a/models/shipping/base/addresses/base_shipping__addresses.sql +++ /dev/null @@ -1,56 +0,0 @@ --- IMPORTS -WITH addresses AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'addresses') }} -), - --- LOGIC -addresses_rename AS ( - - SELECT - addresses.id, - addresses.owner_type, - addresses.owner_id, - addresses.reference, - addresses.street_one AS address_line_one, - addresses.street_two AS address_line_two, - addresses.district_id, - addresses.state_id, - addresses.country_id, - addresses.postcode, - addresses.type, - addresses.default AS is_default, - addresses.status, - addresses.deleted_at AS deleted_datetime, - addresses.created_at AS created_datetime, - addresses.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM addresses -), - --- FINAL -final__base_shipping__addresses AS ( - - SELECT - id, - owner_type, - owner_id, - reference, - address_line_one, - address_line_two, - district_id, - state_id, - country_id, - postcode, - type, - is_default, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM addresses_rename -) - -SELECT * FROM final__base_shipping__addresses \ No newline at end of file diff --git a/models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql b/models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql deleted file mode 100644 index 251794f..0000000 --- a/models/shipping/base/announcement_segments/base_shipping__announcement_segments.sql +++ /dev/null @@ -1,34 +0,0 @@ --- IMPORTS -WITH announcement_segments AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'announcement_segment') }} -), - - --- LOGIC -announcement_segments_rename AS ( - - SELECT - announcement_segments.announcement_id, - announcement_segments.segment_id, - announcement_segments.created_at AS created_datetime, - announcement_segments.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM announcement_segments -), - - --- FINAL -final__base_shipping__announcement_segments AS ( - - SELECT - announcement_id, - segment_id, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM announcement_segments_rename -) - -SELECT * FROM final__base_shipping__announcement_segments \ No newline at end of file diff --git a/models/shipping/base/announcements/base_shipping__announcements.sql b/models/shipping/base/announcements/base_shipping__announcements.sql deleted file mode 100644 index 02e6dc4..0000000 --- a/models/shipping/base/announcements/base_shipping__announcements.sql +++ /dev/null @@ -1,50 +0,0 @@ --- IMPORTS -WITH announcements AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'announcements') }} -), - - --- LOGIC -announcements_rename AS ( - - SELECT - id, - title, - description, - starting_on AS started_datetime, - ending_on AS ended_datetime, - is_all_day, - duration, - is_recurring, - recurrence_pattern, - deleted_at AS deleted_datetime, - created_at AS created_datetime, - updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM announcements -), - - --- FINAL -final__base_shipping__announcements AS ( - - SELECT - id, - title, - description, - started_datetime, - ended_datetime, - is_all_day, - duration, - is_recurring, - recurrence_pattern, - deleted_datetime, - created_datetime, - updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM announcements_rename -) - -SELECT * FROM final__base_shipping__announcements \ No newline at end of file diff --git a/models/shipping/base/companies/base_shipping__companies.sql b/models/shipping/base/companies/base_shipping__companies.sql deleted file mode 100644 index 47ac082..0000000 --- a/models/shipping/base/companies/base_shipping__companies.sql +++ /dev/null @@ -1,45 +0,0 @@ --- IMPORTS -WITH companies AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'companies') }} -), - - --- LOGIC -companies_rename AS ( - - SELECT - companies.id, - companies.name, - companies.reference, --Link with company_module - companies.debtor AS account_software_id, - companies.type AS company_type, - companies.status, - companies.deleted_at AS deleted_datetime, - companies.created_at AS created_datetime, - companies.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM companies -), - - --- FINAL -final__base_shipping__companies AS ( - - SELECT - id, - name, - reference, - account_software_id, - company_type, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM companies_rename - -) - -SELECT * FROM final__base_shipping__companies \ No newline at end of file diff --git a/models/shipping/base/company_connections/base_shipping__company_connections.sql b/models/shipping/base/company_connections/base_shipping__company_connections.sql deleted file mode 100644 index 7f14b2c..0000000 --- a/models/shipping/base/company_connections/base_shipping__company_connections.sql +++ /dev/null @@ -1,45 +0,0 @@ --- IMPORTS -WITH company_connections AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'company_connections') }} -), - - --- LOGIC -company_connections_rename AS ( - - SELECT - company_connections.id, - company_connections.inviter_id, - company_connections.invitee_id, - company_connections.inviter_reference, - company_connections.invitee_reference, - company_connections.status, - company_connections.deleted_at AS deleted_datetime, - company_connections.created_at AS created_datetime, - company_connections.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM company_connections -), - - --- FINAL -final__base_shipping__company_connections AS ( - - SELECT - id, - inviter_id, - invitee_id, - inviter_reference, - invitee_reference, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM company_connections_rename - -) - -SELECT * FROM final__base_shipping__company_connections \ No newline at end of file diff --git a/models/shipping/base/company_employees/base_shipping__company_employees.sql b/models/shipping/base/company_employees/base_shipping__company_employees.sql deleted file mode 100644 index ffeb4b9..0000000 --- a/models/shipping/base/company_employees/base_shipping__company_employees.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH company_employees AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'company_employees') }} -), - - --- LOGIC -company_employees_rename AS ( - - SELECT - company_employees.company_module_id, - company_employees.user_id, - company_employees.role_id, - company_employees.status, - company_employees.deleted_at AS deleted_datetime, - company_employees.created_at AS created_datetime, - company_employees.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM company_employees -), - - --- FINAL -final__base_shipping__company_employees AS ( - - SELECT - company_module_id, - user_id, - role_id, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM company_employees_rename -) - -SELECT * FROM final__base_shipping__company_employees \ No newline at end of file diff --git a/models/shipping/base/company_modules/base_shipping__company_modules.sql b/models/shipping/base/company_modules/base_shipping__company_modules.sql deleted file mode 100644 index c7f1d27..0000000 --- a/models/shipping/base/company_modules/base_shipping__company_modules.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH company_modules AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'company_modules') }} -), - - --- LOGIC -company_modules_rename AS ( - - SELECT - company_modules.id, - company_modules.company_id, - company_modules.name, - company_modules.reference AS marking_id, - company_modules.type, - company_modules.status, - company_modules.unity_hash_id, - company_modules.unity_signature, - company_modules.deleted_at AS deleted_datetime, - company_modules.created_at AS created_datetime, - company_modules.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM company_modules -), - - --- FINAL -final__base_shipping__company_modules AS ( - - SELECT - id, - company_id, - name, - marking_id, - type, - status, - unity_hash_id, - unity_signature, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM company_modules_rename -) - -SELECT * FROM final__base_shipping__company_modules \ No newline at end of file diff --git a/models/shipping/base/connection_segments/base_shipping__connection_segments.sql b/models/shipping/base/connection_segments/base_shipping__connection_segments.sql deleted file mode 100644 index 7f6bd84..0000000 --- a/models/shipping/base/connection_segments/base_shipping__connection_segments.sql +++ /dev/null @@ -1,38 +0,0 @@ --- IMPORTS -WITH connection_segments AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'connection_segments') }} -), - - --- LOGIC -connection_segments_rename AS ( - - SELECT - connection_segments.id, - connection_segments.company_connection_id, - connection_segments.segment_id, - connection_segments.deleted_at AS deleted_datetime, - connection_segments.created_at AS created_datetime, - connection_segments.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM connection_segments -), - - --- FINAL -final__base_shipping__connection_segments AS ( - - SELECT - id, - company_connection_id, - segment_id, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM connection_segments_rename -) - -SELECT * FROM final__base_shipping__connection_segments \ No newline at end of file diff --git a/models/shipping/base/contacts/base_shipping__contacts.sql b/models/shipping/base/contacts/base_shipping__contacts.sql deleted file mode 100644 index bd18ac0..0000000 --- a/models/shipping/base/contacts/base_shipping__contacts.sql +++ /dev/null @@ -1,50 +0,0 @@ --- IMPORTS -WITH contacts AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'contacts') }} -), - - --- LOGIC -contacts_rename AS ( - - SELECT - contacts.id, - contacts.owner_type, - contacts.owner_id, - contacts.reference, - contacts.phone, - contacts.email, - contacts.wechat_id, - contacts.default AS is_default, - contacts.deleted_at AS deleted_datetime, - contacts.created_at AS created_datetime, - contacts.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM contacts - -), - - --- FINAL -final__base_shipping__contacts AS ( - - SELECT - id, - owner_type, - owner_id, - reference, - phone, - email, - wechat_id, - is_default, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM contacts_rename - -) - -SELECT * FROM final__base_shipping__contacts \ No newline at end of file diff --git a/models/shipping/base/containers/base_shipping__containers.sql b/models/shipping/base/containers/base_shipping__containers.sql deleted file mode 100644 index 402d9ee..0000000 --- a/models/shipping/base/containers/base_shipping__containers.sql +++ /dev/null @@ -1,52 +0,0 @@ --- IMPORTS -WITH containers AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'containers') }} -), - - --- LOGIC -containers_rename AS ( - - SELECT - containers.id, - containers.owner_type, - containers.owner_id, - containers.reference, - containers.container_number, - containers.container_type, - containers.seal_reference, - containers.loading_date, - containers.status, - containers.deleted_at AS deleted_datetime, - containers.created_at AS created_datetime, - containers.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM containers - -), - - --- FINAL -final__base_shipping__containers AS ( - - SELECT - id, - owner_type, - owner_id, - reference, - container_number, - container_type, - seal_reference, - loading_date, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM containers_rename - -) - -SELECT * FROM final__base_shipping__containers \ No newline at end of file diff --git a/models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql b/models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql deleted file mode 100644 index c1d6ba8..0000000 --- a/models/shipping/base/containers_and_packing_lists_relations/base_shipping__container_packing_lists.sql +++ /dev/null @@ -1,40 +0,0 @@ --- IMPORTS -WITH container_packing_lists AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'container_packing_lists') }} -), - - --- LOGIC -container_packing_lists_rename AS ( - - SELECT - container_packing_lists.id, - container_packing_lists.container_id, - container_packing_lists.packing_list_id, - container_packing_lists.deleted_at AS deleted_datetime, - container_packing_lists.created_at AS created_datetime, - container_packing_lists.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM container_packing_lists - -), - - --- FINAL -final__base_shipping__container_packing_lists AS ( - - SELECT - id, - container_id, - packing_list_id, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM container_packing_lists_rename - -) - -SELECT * FROM final__base_shipping__container_packing_lists \ No newline at end of file diff --git a/models/shipping/base/documents/base_shipping__documents.sql b/models/shipping/base/documents/base_shipping__documents.sql deleted file mode 100644 index f419fcf..0000000 --- a/models/shipping/base/documents/base_shipping__documents.sql +++ /dev/null @@ -1,54 +0,0 @@ --- IMPORTS -WITH documents AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'documents') }} -), - - --- LOGIC -documents_rename AS ( - - SELECT - documents.id, - documents.owner_type, - documents.owner_id, - documents.document_type, - documents.reference, - documents.status, - documents.issued_date, - documents.expired_date, - documents.approver, - documents.approval_date, - documents.deleted_at AS deleted_datetime, - documents.created_at AS created_datetime, - documents.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM documents - -), - - --- FINAL -final__base_shipping__documents AS ( - - SELECT - id, - owner_type, - owner_id, - document_type, - reference, - status, - issued_date, - expired_date, - approver, - approval_date, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM documents_rename - -) - -SELECT * FROM final__base_shipping__documents \ No newline at end of file diff --git a/models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql b/models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql deleted file mode 100644 index bc9e2fa..0000000 --- a/models/shipping/base/failed_jobs/base_shipping__failed_jobs.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH failed_jobs AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'failed_jobs') }} -), - - --- LOGIC -failed_jobs_rename AS ( - - SELECT - failed_jobs.id, - failed_jobs.uuid, - failed_jobs.connection AS is_connection, - failed_jobs.queue, - failed_jobs.payload, - failed_jobs.exception AS is_exception, - failed_jobs.failed_at AS failed_datetime, - current_timestamp() AS _dbt_ran_at - - FROM failed_jobs - -), - - --- FINAL -final__base_shipping__failed_jobs AS ( - - SELECT - id, - uuid, - is_connection, - queue, - payload, - is_exception, - failed_datetime, - _dbt_ran_at - - FROM failed_jobs_rename - -) - -SELECT * FROM final__base_shipping__failed_jobs \ No newline at end of file diff --git a/models/shipping/base/files/base_shipping__files.sql b/models/shipping/base/files/base_shipping__files.sql deleted file mode 100644 index 15013ef..0000000 --- a/models/shipping/base/files/base_shipping__files.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH files AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'files') }} -), - - --- LOGIC -files_rename AS ( - - SELECT - files.id, - files.document_id, - files.file AS is_file, - files.file_type, - files.deleted_at AS deleted_datetime, - files.created_at AS created_datetime, - files.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM files - -), - - --- FINAL -final__base_shipping__files AS ( - - SELECT - id, - document_id, - is_file, - file_type, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM files_rename - -) - -SELECT * FROM final__base_shipping__files \ No newline at end of file diff --git a/models/shipping/base/order_roles/base_shipping__order_roles.sql b/models/shipping/base/order_roles/base_shipping__order_roles.sql deleted file mode 100644 index 67a827c..0000000 --- a/models/shipping/base/order_roles/base_shipping__order_roles.sql +++ /dev/null @@ -1,46 +0,0 @@ --- IMPORTS -WITH order_roles AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'order_roles') }} -), - - --- LOGIC -order_roles_rename AS ( - - SELECT - order_roles.id, - order_roles.order_id, - order_roles.company_module_id, - order_roles.role_id, - order_roles.entity_hash_id, - order_roles.entity_signature, - order_roles.deleted_at AS deleted_datetime, - order_roles.created_at AS created_datetime, - order_roles.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM order_roles - -), - - --- FINAL -final__base_shipping__order_roles AS ( - - SELECT - id, - order_id, - company_module_id, - role_id, - entity_hash_id, - entity_signature, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM order_roles_rename - -) - -SELECT * FROM final__base_shipping__order_roles \ No newline at end of file diff --git a/models/shipping/base/packages/base_shipping__packages.sql b/models/shipping/base/packages/base_shipping__packages.sql deleted file mode 100644 index 0bcf81e..0000000 --- a/models/shipping/base/packages/base_shipping__packages.sql +++ /dev/null @@ -1,56 +0,0 @@ --- IMPORTS -WITH packages AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'packages') }} -), - - --- LOGIC -packages_rename AS ( - - SELECT - packages.id, - packages.packing_list_id, - packages.type, - packages.reference, - packages.description, - packages.width, - packages.height, - packages.length, - packages.weight, - packages.quantity, - packages.status, - packages.deleted_at AS deleted_datetime, - packages.created_at AS created_datetime, - packages.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM packages - -), - - --- FINAL -final__base_shipping__packages AS ( - - SELECT - id, - packing_list_id, - type, - reference, - description, - width, - height, - length, - weight, - quantity, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM packages_rename - -) - -SELECT * FROM final__base_shipping__packages \ No newline at end of file diff --git a/models/shipping/base/packing_lists/base_shipping__packing_lists.sql b/models/shipping/base/packing_lists/base_shipping__packing_lists.sql deleted file mode 100644 index 32962e7..0000000 --- a/models/shipping/base/packing_lists/base_shipping__packing_lists.sql +++ /dev/null @@ -1,50 +0,0 @@ --- IMPORTS -WITH packing_lists AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'packing_lists') }} -), - - --- LOGIC -packing_lists_rename AS ( - - SELECT - packing_lists.id, - packing_lists.owner_type, - packing_lists.owner_id, - packing_lists.claimant_id, - packing_lists.reference_contract, - packing_lists.reference, - packing_lists.type, - packing_lists.status, - packing_lists.deleted_at AS deleted_datetime, - packing_lists.created_at AS created_datetime, - packing_lists.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM packing_lists - -), - - --- FINAL -final__base_shipping__packing_lists AS ( - - SELECT - id, - owner_type, - owner_id, - claimant_id, - reference_contract, - reference, - type, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM packing_lists_rename - -) - -SELECT * FROM final__base_shipping__packing_lists \ No newline at end of file diff --git a/models/shipping/base/password_resets/base_shipping__password_resets.sql b/models/shipping/base/password_resets/base_shipping__password_resets.sql deleted file mode 100644 index e9193f2..0000000 --- a/models/shipping/base/password_resets/base_shipping__password_resets.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH password_resets AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'password_resets') }} -), - - --- LOGIC -password_resets_rename AS ( - - SELECT - password_resets.id, - password_resets.user_id, - password_resets.token, - password_resets.is_expired, - password_resets.is_complete, - password_resets.created_at AS created_datetime, - password_resets.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM password_resets - -), - - --- FINAL -final__base_shipping__password_resets AS ( - - SELECT - id, - user_id, - token, - is_expired, - is_complete, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM password_resets_rename - -) - -SELECT * FROM final__base_shipping__password_resets \ No newline at end of file diff --git a/models/shipping/base/remarks/base_shipping__remarks.sql b/models/shipping/base/remarks/base_shipping__remarks.sql deleted file mode 100644 index bc70514..0000000 --- a/models/shipping/base/remarks/base_shipping__remarks.sql +++ /dev/null @@ -1,44 +0,0 @@ --- IMPORTS -WITH remarks AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'remarks') }} -), - - --- LOGIC -remarks_rename AS ( - - SELECT - remarks.id, - remarks.owner_type, - remarks.owner_id, - remarks.commenter_id, - remarks.content, - remarks.deleted_at AS deleted_datetime, - remarks.created_at AS created_datetime, - remarks.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM remarks - -), - - --- FINAL -final__base_shipping__remarks AS ( - - SELECT - id, - owner_type, - owner_id, - commenter_id, - content, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM remarks_rename - -) - -SELECT * FROM final__base_shipping__remarks \ No newline at end of file diff --git a/models/shipping/base/schedules/base_shipping__schedules.sql b/models/shipping/base/schedules/base_shipping__schedules.sql deleted file mode 100644 index a680672..0000000 --- a/models/shipping/base/schedules/base_shipping__schedules.sql +++ /dev/null @@ -1,46 +0,0 @@ --- IMPORTS -WITH schedules AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'schedules') }} -), - - --- LOGIC -schedules_rename AS ( - - SELECT - schedules.id, - schedules.owner_type, - schedules.owner_id, - schedules.etd, - schedules.eta, - schedules.status, - schedules.deleted_at AS deleted_datetime, - schedules.created_at AS created_datetime, - schedules.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM schedules - -), - - --- FINAL -final__base_shipping__schedules AS ( - - SELECT - id, - owner_type, - owner_id, - etd, - eta, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM schedules_rename - -) - -SELECT * FROM final__base_shipping__schedules \ No newline at end of file diff --git a/models/shipping/base/segment_constants/base_shipping__segment_constants.sql b/models/shipping/base/segment_constants/base_shipping__segment_constants.sql deleted file mode 100644 index aef6f88..0000000 --- a/models/shipping/base/segment_constants/base_shipping__segment_constants.sql +++ /dev/null @@ -1,42 +0,0 @@ --- IMPORTS -WITH segment_constants AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'segment_constants') }} -), - - --- LOGIC -segment_constants_rename AS ( - - SELECT - segment_constants.id, - segment_constants.segment_id, - segment_constants.reference, - segment_constants.value, - segment_constants.deleted_at AS deleted_datetime, - segment_constants.created_at AS created_datetime, - segment_constants.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM segment_constants - -), - - --- FINAL -final__base_shipping__segment_constants AS ( - - SELECT - id, - segment_id, - reference, - value, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM segment_constants_rename - -) - -SELECT * FROM final__base_shipping__segment_constants \ No newline at end of file diff --git a/models/shipping/base/transaction_details/base_shipping__transaction_details.sql b/models/shipping/base/transaction_details/base_shipping__transaction_details.sql deleted file mode 100644 index ec563ed..0000000 --- a/models/shipping/base/transaction_details/base_shipping__transaction_details.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH transaction_details AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'transaction_details') }} -), - - --- LOGIC -transaction_details_rename AS ( - - SELECT - transaction_details.id, - transaction_details.transaction_id, - transaction_details.reference, - transaction_details.name, - transaction_details.quantity, - transaction_details.price, - transaction_details.amount, - transaction_details.deleted_at AS deleted_datetime, - transaction_details.created_at AS created_datetime, - transaction_details.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM transaction_details - -), - - --- FINAL -final__base_shipping__transaction_details AS ( - - SELECT - id, - transaction_id, - reference, - name, - quantity, - price, - amount, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM transaction_details_rename - -) - -SELECT * FROM final__base_shipping__transaction_details \ No newline at end of file diff --git a/models/shipping/base/transactions/base_shipping__transactions.sql b/models/shipping/base/transactions/base_shipping__transactions.sql deleted file mode 100644 index ccc012d..0000000 --- a/models/shipping/base/transactions/base_shipping__transactions.sql +++ /dev/null @@ -1,72 +0,0 @@ --- IMPORTS -WITH transactions AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'transactions') }} -), - - --- LOGIC -transactions_rename AS ( - - SELECT - transactions.id, - transactions.owner_type, - transactions.owner_id, - transactions.type AS transaction_type, - transactions.issuer AS issued_by, - transactions.receiver AS received_by, - transactions.recipient_bank_account_id, - transactions.payment_method, - transactions.payment_reference, - transactions.bill_no, - transactions.amount AS base_value, - transactions.original_amount AS quote_value, - transactions.currency_id AS base_currency_id, - transactions.original_currency_id AS quote_currency_id, - transactions.currency_rate AS base_to_quote_currency_exchange_rate, - transactions.tax AS base_tax, - transactions.service_charge AS base_service_charge, - transactions.expires_on AS expired_datetime, - transactions.status, - transactions.deleted_at AS deleted_datetime, - transactions.created_at AS created_datetime, - transactions.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM transactions - -), - - --- FINAL -final__base_shipping__transactions AS ( - - SELECT - id, - owner_type, - owner_id, - transaction_type, - issued_by, - received_by, - recipient_bank_account_id, - payment_method, - payment_reference, - bill_no, - base_value, - quote_value, - base_currency_id, - quote_currency_id, - base_to_quote_currency_exchange_rate, - base_tax, - base_service_charge, - expired_datetime, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM transactions_rename - -) - -SELECT * FROM final__base_shipping__transactions \ No newline at end of file diff --git a/models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql b/models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql deleted file mode 100644 index c03da36..0000000 --- a/models/shipping/base/transport_arrangements/base_shipping__transport_arrangements.sql +++ /dev/null @@ -1,52 +0,0 @@ --- IMPORTS -WITH transport_arrangements AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'transport_arrangements') }} -), - - --- LOGIC -transport_arrangements_rename AS ( - - SELECT - transport_arrangements.id, - transport_arrangements.owner_type, - transport_arrangements.owner_id, - transport_arrangements.type, - transport_arrangements.courier, - transport_arrangements.tracking_number, - transport_arrangements.dispatch_date AS dispatched_datetime, - transport_arrangements.drop_date AS droped_datetime, - transport_arrangements.status, - transport_arrangements.deleted_at AS deleted_datetime, - transport_arrangements.created_at AS created_datetime, - transport_arrangements.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM transport_arrangements - -), - - --- FINAL -final__base_shipping__transport_arrangements AS ( - - SELECT - id, - owner_type, - owner_id, - type, - courier, - tracking_number, - dispatched_datetime, - droped_datetime, - status, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM transport_arrangements_rename - -) - -SELECT * FROM final__base_shipping__transport_arrangements \ No newline at end of file diff --git a/models/shipping/base/users/base_shipping__users.sql b/models/shipping/base/users/base_shipping__users.sql deleted file mode 100644 index fe47323..0000000 --- a/models/shipping/base/users/base_shipping__users.sql +++ /dev/null @@ -1,48 +0,0 @@ --- IMPORTS -WITH users AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'users') }} -), - - --- LOGIC -users_rename AS ( - - SELECT - users.id, - users.name, - users.email, - users.type AS user_type, - users.status, - users.remember_token, - users.active_at AS activated_datetime, - users.deleted_at AS deleted_datetime, - users.created_at AS created_datetime, - users.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at - - FROM users - -), - - --- FINAL -final__base_shipping__users AS ( - - SELECT - id, - name, - email, - user_type, - status, - remember_token, - activated_datetime, - deleted_datetime, - created_datetime, - updated_datetime, - _dbt_ran_at - - FROM users_rename - -) - -SELECT * FROM final__base_shipping__users \ No newline at end of file diff --git a/models/shipping/source/src_shipping__activity_log.yml b/models/shipping/source/src_shipping__activity_log.yml deleted file mode 100644 index 205a7d0..0000000 --- a/models/shipping/source/src_shipping__activity_log.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: activity_log - description: Raw `activity_log` table from Snowflake - columns: - - name: id - description: Primary key for 'activity_log' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__addresses.yml b/models/shipping/source/src_shipping__addresses.yml deleted file mode 100644 index 2318835..0000000 --- a/models/shipping/source/src_shipping__addresses.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: addresses - description: Raw `addresses` table from Snowflake - columns: - - name: id - description: Primary key for 'addresses' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__announcement_segment.yml b/models/shipping/source/src_shipping__announcement_segment.yml deleted file mode 100644 index 94e24b8..0000000 --- a/models/shipping/source/src_shipping__announcement_segment.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: announcement_segment - description: Raw `announcement_segment` table from Snowflake - columns: - - name: announcement_id - description: Primary key for 'announcement_segment' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__announcements.yml b/models/shipping/source/src_shipping__announcements.yml deleted file mode 100644 index 23a8449..0000000 --- a/models/shipping/source/src_shipping__announcements.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: announcements - description: Raw `announcements` table from Snowflake - columns: - - name: id - description: Primary key for 'announcements' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__companies.yml b/models/shipping/source/src_shipping__companies.yml deleted file mode 100644 index c88b538..0000000 --- a/models/shipping/source/src_shipping__companies.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: companies - description: Raw `companies` table from Snowflake - columns: - - name: id - description: Primary key for 'companies' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__company_connections.yml b/models/shipping/source/src_shipping__company_connections.yml deleted file mode 100644 index 79b37dd..0000000 --- a/models/shipping/source/src_shipping__company_connections.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: company_connections - description: Raw `company_connections` table from Snowflake - columns: - - name: id - description: Primary key for 'company_connections' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__company_employees.yml b/models/shipping/source/src_shipping__company_employees.yml deleted file mode 100644 index 514d5c5..0000000 --- a/models/shipping/source/src_shipping__company_employees.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: company_employees - description: Raw `company_employees` table from Snowflake - columns: - - name: company_module_id - description: Primary key for 'company_employees' - tests: - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__company_modules.yml b/models/shipping/source/src_shipping__company_modules.yml deleted file mode 100644 index b03b0f2..0000000 --- a/models/shipping/source/src_shipping__company_modules.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: company_modules - description: Raw `company_modules` table from Snowflake - columns: - - name: id - description: Primary key for 'company_modules' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__connection_segments.yml b/models/shipping/source/src_shipping__connection_segments.yml deleted file mode 100644 index bbf35e7..0000000 --- a/models/shipping/source/src_shipping__connection_segments.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: connection_segments - description: Raw `connection_segments` table from Snowflake - columns: - - name: id - description: Primary key for 'connection_segments' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__contacts.yml b/models/shipping/source/src_shipping__contacts.yml deleted file mode 100644 index e53ba6e..0000000 --- a/models/shipping/source/src_shipping__contacts.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: contacts - description: Raw `contacts` table from Snowflake - columns: - - name: id - description: Primary key for 'contacts' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__container_packing_lists.yml b/models/shipping/source/src_shipping__container_packing_lists.yml deleted file mode 100644 index 19000c8..0000000 --- a/models/shipping/source/src_shipping__container_packing_lists.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: container_packing_lists - description: Raw `container_packing_lists` table from Snowflake - columns: - - name: id - description: Primary key for 'container_packing_lists' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__containers.yml b/models/shipping/source/src_shipping__containers.yml deleted file mode 100644 index 4a2ae0f..0000000 --- a/models/shipping/source/src_shipping__containers.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: containers - description: Raw `containers` table from Snowflake - columns: - - name: id - description: Primary key for 'containers' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__documents.yml b/models/shipping/source/src_shipping__documents.yml deleted file mode 100644 index 5d685b3..0000000 --- a/models/shipping/source/src_shipping__documents.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: documents - description: Raw `documents` table from Snowflake - columns: - - name: id - description: Primary key for 'documents' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__failed_jobs.yml b/models/shipping/source/src_shipping__failed_jobs.yml deleted file mode 100644 index 731dadb..0000000 --- a/models/shipping/source/src_shipping__failed_jobs.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: failed_jobs - description: Raw `failed_jobs` table from Snowflake - columns: - - name: id - description: Primary key for 'failed_jobs' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__files.yml b/models/shipping/source/src_shipping__files.yml deleted file mode 100644 index db26432..0000000 --- a/models/shipping/source/src_shipping__files.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: files - description: Raw `files` table from Snowflake - columns: - - name: id - description: Primary key for 'files' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__order_roles.yml b/models/shipping/source/src_shipping__order_roles.yml deleted file mode 100644 index de69e2b..0000000 --- a/models/shipping/source/src_shipping__order_roles.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: order_roles - description: Raw `order_roles` table from Snowflake - columns: - - name: id - description: Primary key for 'order_roles' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__orders.yml b/models/shipping/source/src_shipping__orders.yml deleted file mode 100644 index 3e3e5c1..0000000 --- a/models/shipping/source/src_shipping__orders.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: orders - description: Raw `orders` table from Snowflake - columns: - - name: id - description: Primary key for 'orders' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__packages.yml b/models/shipping/source/src_shipping__packages.yml deleted file mode 100644 index 930afb3..0000000 --- a/models/shipping/source/src_shipping__packages.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: packages - description: Raw `packages` table from Snowflake - columns: - - name: id - description: Primary key for 'packages' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__packing_lists.yml b/models/shipping/source/src_shipping__packing_lists.yml deleted file mode 100644 index bc247dd..0000000 --- a/models/shipping/source/src_shipping__packing_lists.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: packing_lists - description: Raw `packing_lists` table from Snowflake - columns: - - name: id - description: Primary key for 'packing_lists' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__password_resets.yml b/models/shipping/source/src_shipping__password_resets.yml deleted file mode 100644 index 5a805d7..0000000 --- a/models/shipping/source/src_shipping__password_resets.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: password_resets - description: Raw `password_resets` table from Snowflake - columns: - - name: id - description: Primary key for 'password_resets' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__remarks.yml b/models/shipping/source/src_shipping__remarks.yml deleted file mode 100644 index 65fa3bd..0000000 --- a/models/shipping/source/src_shipping__remarks.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: remarks - description: Raw `remarks` table from Snowflake - columns: - - name: id - description: Primary key for 'remarks' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__schedules.yml b/models/shipping/source/src_shipping__schedules.yml deleted file mode 100644 index 94300e4..0000000 --- a/models/shipping/source/src_shipping__schedules.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: schedules - description: Raw `schedules` table from Snowflake - columns: - - name: id - description: Primary key for 'schedules' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__segment_constants.yml b/models/shipping/source/src_shipping__segment_constants.yml deleted file mode 100644 index 4bde167..0000000 --- a/models/shipping/source/src_shipping__segment_constants.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: segment_constants - description: Raw `segment_constants` table from Snowflake - columns: - - name: id - description: Primary key for 'segment_constants' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__transaction_details.yml b/models/shipping/source/src_shipping__transaction_details.yml deleted file mode 100644 index a8583a9..0000000 --- a/models/shipping/source/src_shipping__transaction_details.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: transaction_details - description: Raw `transaction_details` table from Snowflake - columns: - - name: id - description: Primary key for 'transaction_details' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__transactions.yml b/models/shipping/source/src_shipping__transactions.yml deleted file mode 100644 index 20952eb..0000000 --- a/models/shipping/source/src_shipping__transactions.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: transactions - description: Raw `transactions` table from Snowflake - columns: - - name: id - description: Primary key for 'transactions' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__transport_arrangements.yml b/models/shipping/source/src_shipping__transport_arrangements.yml deleted file mode 100644 index 4994542..0000000 --- a/models/shipping/source/src_shipping__transport_arrangements.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: transport_arrangements - description: Raw `transport_arrangements` table from Snowflake - columns: - - name: id - description: Primary key for 'transport_arrangements' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/shipping/source/src_shipping__users.yml b/models/shipping/source/src_shipping__users.yml deleted file mode 100644 index b13fbb4..0000000 --- a/models/shipping/source/src_shipping__users.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -sources: - - name: src_shipping_mysql - description: Source data from snowflake 'DEV_CIEF_RAW_DB' - database: | - {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB - {%- else -%} DEV_CIEF_RAW_DB - {%- endif -%} - schema: SHIPPING_MYSQL - - tables: - - name: users - description: Raw `users` table from Snowflake - columns: - - name: id - description: Primary key for 'users' - tests: - - unique - - not_null - loaded_at_field: _source_loaded_at - freshness: - warn_after: {count: 2, period: hour} - error_after: {count: 24, period: hour} \ No newline at end of file diff --git a/models/staging/crisp/_crisp__docs.md b/models/staging/crisp/_crisp__docs.md new file mode 100644 index 0000000..9c03a10 --- /dev/null +++ b/models/staging/crisp/_crisp__docs.md @@ -0,0 +1,45 @@ +{% docs src_crisp_api %} + +Application link: https://crisp.chat/en/ + +Data is extract from [CRISP Api Python Library](https://github.com/crisp-im/python-crisp-api), using custom python code. + +The API Token will refresh everyday, we can know our usage from [CRISP Marketplace](https://marketplace.crisp.chat/plugins/). 1 Token will used when 1 API is call. + +- When the python get the JSON data. It will denormalize it. + +Input: + + "data": [ + { + "people_id": "aaaa-bbbb-cccc", + + "email": "tesing@gmail.com", + + "person": { + "gender": "male" + } + }] + + +Output: + +|people_id|email|person__gender| +|----|----|----| +|aaaa-bbbb-cccc|tesing@gmail.com|male| + +- `api_called_at` will add into the raw data when extract. _It should added an underscore but YAM forget_ + +- Python will call the API in everyday 12:30AM + +- The raw data will use copy into snowflake in 2AM everyday + +- Depend on target.name to determin which snowflake raw schema + +```JINJA +{_%- if target.name == "prod" -%} PROD_CIEF_RAW_DB +{_%- else -%} DEV_CIEF_RAW_DB +{_%- endif -%} +``` + +{% enddocs %} \ No newline at end of file diff --git a/models/staging/crisp/_crisp__sources.yml b/models/staging/crisp/_crisp__sources.yml new file mode 100644 index 0000000..cb2d9f2 --- /dev/null +++ b/models/staging/crisp/_crisp__sources.yml @@ -0,0 +1,85 @@ +version: 2 + +sources: + - name: src_crisp_api + description: '{{ doc("src_crisp_api") }}' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: CRISP_API + loader: custom_python_code + loaded_at_field: api_called_at + freshness: + warn_after: {count: 26, period: hour} + error_after: {count: 48, period: hour} + meta: + owner: "@yam" + model_maturity: prod + tags: + - crisp + - daily + + + tables: + - name: get_message_in_conversations + description: Table that get all the message the send by user/operator + columns: + - name: fingerprint + description: Primary key for 'get_message_in_conversations' + tests: + - unique + - not_null + + - name: delivered + description: the platform which after the message is output from origin, and delivered to the user/operator. When an operator send a message but the user is not online anymore, we send it via email only after 2min during these 2min, delivered is empty + + + - name: get_people_data + description: This is CIEF created custom field, when the user is chat with us using FB/Whatsapp/ExchangeWebsite/ShippingWebsite our crisp will auto mapping user login account & some identity information + columns: + - name: people_id + description: Primary key for 'get_people_data' + tests: + - unique + - not_null + + + - name: get_people_subscription_status + description: SCD-Type1. The CRISP email subscription status. If we mapping the user_created_datetime and email_unsubscripte_datetime we will know when the unsubscripte duration + columns: + - name: people_id + description: Primary key for 'get_people_subscription_status' + tests: + - unique + - not_null + + + - name: list_conversations + description: SCD-Type1. All the chat box in the CRISP. One user might have different chat box. + columns: + - name: session_id + description: Primary key for 'list_conversations' + tests: + - unique + - not_null + + + - name: list_people_profiles + description: SCD-Type1. The user profile, combination of user & company data. + columns: + - name: people_id + description: Primary key for 'list_people_profiles' + tests: + - unique + - not_null + + + - name: list_website_operator + description: SCD-Type1. CIEF Crisp Admin/Agent table + columns: + - name: details__user_id + description: Primary key for 'list_website_operator' + tests: + - unique + - not_null \ No newline at end of file diff --git a/models/staging/crisp/base/base_crisp__user_data.sql b/models/staging/crisp/base/base_crisp__user_data.sql new file mode 100644 index 0000000..4873a8a --- /dev/null +++ b/models/staging/crisp/base/base_crisp__user_data.sql @@ -0,0 +1,48 @@ +-- IMPORT +WITH get_people_data AS ( + SELECT * FROM {{ source('src_crisp_api', 'get_people_data') }} +), + +--LOGIC +get_people_data_rename AS ( + SELECT + people_id AS user_id, + data__company_name AS company_name, + data__wa_number AS whatsapp_number, + data__whatsapp_business_number AS whatsapp_business_number, + data__fb_page AS facebook_page_name, + data__fb_page_id AS facebook_page_id, + data__exchange_marking AS exchange_marking_id, + data__izyim_marking AS izyim_marking_id, + data__lite_marking AS lite_marking_id, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM get_people_data +), + +--FINAL +final_base_crisp__get_people_data AS ( + SELECT + -- ids + user_id, + exchange_marking_id, + izyim_marking_id, + lite_marking_id, + facebook_page_id, + + -- dimensions + company_name, + whatsapp_number, + whatsapp_business_number, + facebook_page_name, + + -- measures + + -- date/times + + -- metadata + _dbt_ran_datetime + + FROM get_people_data_rename +) +SELECT * FROM final_base_crisp__get_people_data \ No newline at end of file diff --git a/models/staging/crisp/base/base_crisp__user_email_subscriptions.sql b/models/staging/crisp/base/base_crisp__user_email_subscriptions.sql new file mode 100644 index 0000000..55e44d4 --- /dev/null +++ b/models/staging/crisp/base/base_crisp__user_email_subscriptions.sql @@ -0,0 +1,41 @@ +-- IMPORT +WITH get_people_subscription_status AS ( + SELECT * FROM {{ ref('snap_crisp__get_people_subscription_status') }} +), + +--LOGIC +get_people_subscription_status_rename AS ( + SELECT + dbt_scd_id AS user_email_subscription_id, + people_id AS user_id, + IFF(email = TRUE, 1, 0) AS is_subscribed, + dbt_valid_from AS from_datetime, + dbt_valid_to AS to_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM get_people_subscription_status +), + +--FINAL +final_base_crisp__user_email_subscriptions AS ( + SELECT + -- ids + user_email_subscription_id, + user_id, + + -- dimensions + is_subscribed, + + -- measures + + -- date/times + from_datetime, + to_datetime, + + -- metaadata + _dbt_ran_datetime + + FROM get_people_subscription_status_rename +) + +SELECT * FROM final_base_crisp__user_email_subscriptions \ No newline at end of file diff --git a/models/crisp/staging/stg_crisp__peoples.sql b/models/staging/crisp/base/base_crisp__users.sql similarity index 56% rename from models/crisp/staging/stg_crisp__peoples.sql rename to models/staging/crisp/base/base_crisp__users.sql index f8cb577..75496a0 100644 --- a/models/crisp/staging/stg_crisp__peoples.sql +++ b/models/staging/crisp/base/base_crisp__users.sql @@ -4,41 +4,39 @@ WITH list_people_profiles AS ( ), --LOGIC - ---FINAL -final_stg_crisp__peoples AS ( +list_people_profiles_rename AS ( SELECT - people_id AS id, + people_id AS user_id, person__nickname AS name, person__phone AS phone, email AS email, person__website AS website, score AS score, - segments AS segment_list, + segments AS segments, person__employment__role AS role, person__employment__seniority AS seniority, person__employment__title AS title, person__gender AS gender, person__timezone AS timezone, - person__geolocation__country AS geo_country, - person__geolocation__city AS geo_city, - person__geolocation__region AS geo_region_code, + person__geolocation__country AS country, + person__geolocation__city AS city, + person__geolocation__region AS region_code, person__address AS address, - person__locales AS locale_list, - person__geolocation__coordinates__latitude AS geo_latitude, - person__geolocation__coordinates__longitude AS geo_longitude, + person__locales AS locales, + person__geolocation__coordinates__latitude AS latitude, + person__geolocation__coordinates__longitude AS longitude, convert_timezone('Asia/Kuala_Lumpur', active__last) AS last_activated_datetime, - convert_timezone('Asia/Kuala_Lumpur', created_at) AS people_created_datetime, - convert_timezone('Asia/Kuala_Lumpur', updated_at) AS people_updated_datetime, + convert_timezone('Asia/Kuala_Lumpur', created_at) AS created_datetime, + convert_timezone('Asia/Kuala_Lumpur', updated_at) AS updated_datetime, company__name AS company_name, company__legal_name AS company_legal_name, company__description AS company_description, company__phones AS company_phone, - company__emails AS company_email_list, + company__emails AS company_emails, company__url AS company_website, company__domain AS company_domain, - company__tags AS company_tag_list, + company__tags AS company_tags, company__metrics__employees AS company_number_of_employees, company__metrics__market_cap AS company_market_cap, company__geolocation__country AS company_country, @@ -46,9 +44,66 @@ final_stg_crisp__peoples AS ( company__geolocation__region AS company_region, company__geolocation__coordinates__latitude AS company_latitude, company__geolocation__coordinates__longitude AS company_logitude, - company__timezone AS company_timezone + company__timezone AS company_timezone, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM list_people_profiles +), + + +--FINAL +final_base_crisp__users AS ( + SELECT + -- ids + user_id, + + -- dimensions + name, + phone, + email, + website, + score, + segments, + role, + seniority, + title, + gender, + timezone, + country, + city, + region_code, + address, + locales, + latitude, + longitude, + company_name, + company_legal_name, + company_description, + company_phone, + company_emails, + company_website, + company_domain, + company_tags, + company_number_of_employees, + company_market_cap, + company_country, + company_city, + company_region, + company_latitude, + company_logitude, + company_timezone, + + -- measures + + -- date/times + last_activated_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM list_people_profiles_rename ) -SELECT * FROM final_stg_crisp__peoples \ No newline at end of file +SELECT * FROM final_base_crisp__users \ No newline at end of file diff --git a/models/staging/crisp/stg_crisp__conversations.sql b/models/staging/crisp/stg_crisp__conversations.sql new file mode 100644 index 0000000..c46dc16 --- /dev/null +++ b/models/staging/crisp/stg_crisp__conversations.sql @@ -0,0 +1,46 @@ +-- IMPORT +WITH list_conversations AS ( + SELECT * FROM {{ source('src_crisp_api', 'list_conversations') }} +), + +--LOGIC +list_conversations_rename AS ( + SELECT + session_id AS conversation_id, + people_id AS user_id, + state AS status, + IFF(is_blocked = TRUE, 1, 0) AS is_blocked_conversation, + assigned__user_id AS operator_id, + convert_timezone('Asia/Kuala_Lumpur', created_at) AS conversation_created_datetime, + convert_timezone('Asia/Kuala_Lumpur', updated_at) AS conversation_updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM list_conversations +), + + +--FINAL +final_stg_crisp__conversations AS ( + SELECT + -- ids + conversation_id, + user_id, + operator_id, + + -- dimensions + status, + is_blocked_conversation, + + -- measures + + -- date/times + conversation_created_datetime, + conversation_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM list_conversations_rename +) + +SELECT * FROM final_stg_crisp__conversations \ No newline at end of file diff --git a/models/staging/crisp/stg_crisp__messages.sql b/models/staging/crisp/stg_crisp__messages.sql new file mode 100644 index 0000000..eb8b9ae --- /dev/null +++ b/models/staging/crisp/stg_crisp__messages.sql @@ -0,0 +1,65 @@ +-- IMPORT +WITH get_message_in_conversations AS ( + SELECT * FROM {{ source('src_crisp_api', 'get_message_in_conversations') }} +), + +-- LOGIC +get_message_in_conversations_rename AS ( + SELECT + fingerprint AS message_id, + session_id AS conversation_id, + user__user_id AS user_id, + user__nickname AS user_name, + from_user AS message_from_side, + origin AS message_generated_platform, + delivered AS message_delivered_platform, + read AS message_read_platform, + type AS message_format, + content__namespace AS event_type, + content__text AS event_message, + content AS message, + edited AS is_edited_message, + content__type AS attachment_type, + content__name AS attachment_name, + content__url AS attachment_url, + convert_timezone('Asia/Kuala_Lumpur', timestamp) AS created_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM get_message_in_conversations +), + +-- FINAL +final_stg_crisp__messages AS ( + SELECT + -- ids + message_id, + conversation_id, + user_id, + + -- dimensions + user_name, + message_from_side, + message_generated_platform, + message_delivered_platform, + message_read_platform, + message_format, + event_type, + event_message, + message, + is_edited_message, + attachment_type, + attachment_name, + attachment_url, + + -- measures + + -- date/times + created_datetime, + + -- metadata + _dbt_ran_datetime + + FROM get_message_in_conversations_rename +) + +SELECT * FROM final_stg_crisp__messages \ No newline at end of file diff --git a/models/staging/crisp/stg_crisp__operators.sql b/models/staging/crisp/stg_crisp__operators.sql new file mode 100644 index 0000000..1db5cf8 --- /dev/null +++ b/models/staging/crisp/stg_crisp__operators.sql @@ -0,0 +1,37 @@ +-- IMPORT +WITH list_website_operators AS ( + SELECT * FROM {{ source('src_crisp_api', 'list_website_operator') }} +), + +--LOGIC +list_website_operators_rename AS ( + SELECT + details__user_id AS operator_user_id, + details__title AS title, + details__first_name || ' ' || details__last_name AS name, + details__email AS email, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM list_website_operators +), + +--FINAL +final_stg_crisp__operators AS ( + SELECT + -- ids + operator_user_id, + + -- dimensions + name, + title, + email, + + -- measures + + -- date/times + _dbt_ran_datetime + + FROM list_website_operators_rename +) + +SELECT * FROM final_stg_crisp__operators \ No newline at end of file diff --git a/models/staging/crisp/stg_crisp__users.sql b/models/staging/crisp/stg_crisp__users.sql new file mode 100644 index 0000000..d6cd5b5 --- /dev/null +++ b/models/staging/crisp/stg_crisp__users.sql @@ -0,0 +1,149 @@ +--IMPORT +WITH users AS ( + SELECT * FROM {{ ref('base_crisp__users') }} +), + +user_data AS ( + SELECT * FROM {{ ref('base_crisp__user_data') }} +), + +user_email_subscriptions AS ( + SELECT * FROM {{ ref('base_crisp__user_email_subscriptions') }} +), + +--LOGIC +latest_user_email_subscriptions AS ( + SELECT + * + + FROM user_email_subscriptions + + WHERE + to_datetime IS NULL +), + +people_get_marking_id_and_email_subscriptions AS ( + SELECT + users.user_id, + users.name, + users.phone, + users.email, + users.website, + users.score, + users.segments, + users.role, + users.seniority, + users.title, + users.gender, + users.timezone, + users.country, + users.city, + users.region_code, + users.address, + users.locales, + users.latitude, + users.longitude, + users.last_activated_datetime, + users.created_datetime, + users.updated_datetime, + users.company_name, + users.company_legal_name, + users.company_description, + users.company_phone, + users.company_emails, + users.company_website, + users.company_domain, + users.company_tags, + users.company_number_of_employees, + users.company_market_cap, + users.company_country, + users.company_city, + users.company_region, + users.company_latitude, + users.company_logitude, + users.company_timezone, + user_data.company_name AS marking_company_name, + user_data.whatsapp_number, + user_data.whatsapp_business_number, + user_data.facebook_page_name, + user_data.facebook_page_id, + user_data.exchange_marking_id, + user_data.izyim_marking_id, + user_data.lite_marking_id, + latest_user_email_subscriptions.is_subscribed, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM users + + LEFT JOIN user_data + ON (users.user_id = user_data.user_id) + + LEFT JOIN latest_user_email_subscriptions + ON (users.user_id = latest_user_email_subscriptions.user_id) +), + +--FINAL +final_base_crisp__user_email_subscriptions AS ( + SELECT + -- ids + user_id, + exchange_marking_id, + izyim_marking_id, + lite_marking_id, + facebook_page_id, + + -- dimensions + name, + phone, + email, + website, + score, + segments, + role, + seniority, + title, + gender, + timezone, + country, + city, + region_code, + address, + locales, + latitude, + longitude, + company_name, + company_legal_name, + company_description, + company_phone, + company_emails, + company_website, + company_domain, + company_tags, + company_number_of_employees, + company_market_cap, + company_country, + company_city, + company_region, + company_latitude, + company_logitude, + company_timezone, + marking_company_name, + whatsapp_number, + whatsapp_business_number, + facebook_page_name, + is_subscribed, + + -- measures + + -- date/times + last_activated_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM people_get_marking_id_and_email_subscriptions +) + +SELECT * FROM final_base_crisp__user_email_subscriptions \ No newline at end of file diff --git a/models/staging/exchange/_exchange__docs.md b/models/staging/exchange/_exchange__docs.md new file mode 100644 index 0000000..b69871b --- /dev/null +++ b/models/staging/exchange/_exchange__docs.md @@ -0,0 +1,21 @@ +{% docs src_exchange_mysql %} + +Application link: https://exchange.cief-malaysia.com/ + +Data is extract from CIEF Selfhost MYSQL `exchange_production`, using custom python code. + +- `_source_loaded_at` will add into the raw data when extract + +- Python will run the SQL in everyday 12:30AM + +- The raw data will use copy into snowflake in 2AM everyday + +- Depend on target.name to determin which snowflake raw schema + +```JINJA +{_%- if target.name == "prod" -%} PROD_CIEF_RAW_DB +{_%- else -%} DEV_CIEF_RAW_DB +{_%- endif -%} +``` + +{% enddocs %} \ No newline at end of file diff --git a/models/staging/exchange/_exchange__models.yml b/models/staging/exchange/_exchange__models.yml new file mode 100644 index 0000000..4477bbb --- /dev/null +++ b/models/staging/exchange/_exchange__models.yml @@ -0,0 +1,26 @@ +version: 2 + +models: + - name: stg_exchange__activity_logs + description: exchange user activity logs + + - name: stg_exchange__banks + description: user input bank information, and be bank transfer / alipay / personol bank information + + - name: stg_exchange__booking_logs + description: Full picture of booking logs with current & archived data + + - name: stg_exchange__bookings + description: SCD-Type1 bookings data + + - name: stg_exchange__companies + description: Company that register in Exchange system, contain client/supplier company + + - name: stg_exchange__companies_brg_segments + description: Bridge between `companies` and `segments` + + - name: stg_exchange__companies_brg_users + description: Bridge between `companies` and `users` + + - name: stg_exchange__company_addresses + description: Company addresses that input when register \ No newline at end of file diff --git a/models/staging/exchange/_exchange__sources.yml b/models/staging/exchange/_exchange__sources.yml new file mode 100644 index 0000000..f4e3550 --- /dev/null +++ b/models/staging/exchange/_exchange__sources.yml @@ -0,0 +1,618 @@ +version: 2 + +sources: + - name: src_exchange_mysql + description: '{{ doc("src_exchange_mysql") }}' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: EXCHANGE_MYSQL + loader: custom_python_code + loaded_at_field: _source_loaded_datetime + freshness: + warn_after: {count: 26, period: hour} + error_after: {count: 48, period: hour} + meta: + owner: "@yam" + model_maturity: prod + tags: + - exchange + - daily + + + tables: + - name: activity_log + description: Contains most table's activity log that CUD (created, updated, deleted) in www.exchange.cief-malaysia.com + columns: + - name: id + description: Primary key for 'activity_log' + tests: + - unique + - not_null + + - name: description + description: system CUD activity list down + tests: + - accepted_values: + values: ['created', 'updated', 'deleted'] + + - name: subject_id + description: The exchange CUD table's id, need to refer subject_type column to know what table is CUD in the row + tests: + - not_null + + - name: subject_type + description: What table that CUD using in the row + tests: + - not_null + + - name: causer_id + description: Need to refer causer_type column to know who is CUD in the row. it is not well maintain there is some NULL + + - name: causer_type + description: Who is CUD the row + + - name: properties + description: Json data format that contain attributes of the activity, it is not well maintain, subject (Country, ServiceType, CurrencyRate. UserEmailVerification, File) have properties only + + - name: created_at + description: Datetime that CUD activity occur + tests: + - not_null + + - name: updated_at + description: Exactly same with created_at, redundant column + + tests: + - dbt_expectations.expect_column_pair_values_to_be_equal: + column_A: updated_at + column_B: created_at + + + - name: addresses + description: Addresses that input when user register + columns: + - name: id + description: Primary key for 'addresses' + tests: + - unique + - not_null + + - name: company_id + description: Foreign keys for 'companies' + tests: + - not_null + + - name: country_id + description: Foreign keys for 'countries' + tests: + - not_null + + - name: state_id + description: Foreign keys for 'states' + tests: + - not_null + + - name: district_id + description: Foreign keys for 'districts' + tests: + - not_null + + - name: postcode + description: The number/string that user input + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: integer + + - name: street_one + description: The string that user input + tests: + - not_null + + - name: street_two + description: The string that user input + + - name: billing + description: If mark 1 means it is billing address, 0 means is not billing address, every company should have 1 billing address + tests: + - dbt_expectations.expect_column_max_to_be_between: #Every company should have a billing address + min_value: 1 + max_value: 1 + group_by: [company_id] + - dbt_expectations.expect_column_sum_to_be_between: #1 company should have 1 billing address only + min_value: 1 + max_value: 1 + group_by: [company_id] + row_condition: "deleted_at is null" + + - name: deleted_at + description: System generate datetime that address was deleted + + - name: created_at + description: System generate datetime that address was created + tests: + - not_null + + - name: updated_at + description: System generate datetime that address was updated + tests: + - not_null + + + - name: announcement_segment + description: Superadmin / Operator guys will segment the announcment, but normally they will leave it default. 2 FK combine to becoem a relation table. + columns: + - name: announcement_id + description: Primary key for 'announcement_segment' & Foreign key for 'announcements' + tests: + - not_null + + - name: segment_id + description: Primary key for 'announcement_segment' & Foreign key for 'segments' + tests: + - not_null + + - name: created_at + description: Abandon column + tests: + - dbt_expectations.expect_column_values_to_be_null + + - name: updated_at + description: Abandon column + tests: + - dbt_expectations.expect_column_values_to_be_null + + + - name: announcements + description: Announcements that show in our website which create by superadmin + columns: + - name: id + description: Primary key for 'announcements' + tests: + - unique + - not_null + + - name: title + description: title of announcement, using string + tests: + - not_null + + - name: description + description: description of announcement, using longtext, have
html for break + tests: + - not_null + + - name: starting_on + description: announcement starting datetime + tests: + - not_null + + - name: ending_on + description: announcement ending datetime + tests: + - not_null + + - name: is_all_day + description: column might not in use, system auto generate + tests: + - not_null + - accepted_values: + values: [0] + + - name: duration + description: column might not in use, system auto generate + tests: + - not_null + - accepted_values: + values: [0] + + - name: is_recurring + description: column might not in use, system auto generate + tests: + - not_null + - accepted_values: + values: [0] + + - name: recurrence_pattern + description: column might not in use, system auto generate + tests: + - dbt_expectations.expect_column_values_to_be_null + + - name: deleted_at + description: system generate datetime that announcment was deleted + + - name: created_at + description: system generate datetime that announcment was created + tests: + - not_null + + - name: updated_at + description: system generate datetime that announcment was updated + tests: + - not_null + + + - name: bank_logs + description: Abandon table + config: + enabled: false + + + - name: banks + description: Table which store banks details. + columns: + - name: id + description: Primary key for 'banks' + tests: + - unique + - not_null + + - name: company_id + description: Foreign keys for 'companies' + tests: + - not_null + + - name: reference + description: A string which write by user/company for the bank transfer reference or other remind ussage + + - name: holder_name + description: Bank holder name + tests: + - not_null + + - name: account_no + description: Bank account number + tests: + - not_null + + + + + - name: booking_logs + description: Raw `bookings` table from Snowflake + columns: + - name: id + description: Primary key for 'booking_logs' + tests: + - unique + - not_null + + + - name: bookings + description: Raw `bookings` table from Snowflake + columns: + - name: id + description: Primary key for 'bookings' + tests: + - unique + - not_null + + + - name: companies + description: Raw `companies` table from Snowflake + columns: + - name: id + description: Primary key for 'companies' + tests: + - unique + - not_null + + + - name: contacts + description: Raw `contacts` table from Snowflake + columns: + - name: id + description: Primary key for 'contacts' + tests: + - unique + - not_null + + + - name: countries + description: Raw `countries` table from Snowflake + columns: + - name: id + description: Primary key for 'countries' + tests: + - unique + - not_null + + + - name: currencies + description: Raw `currencies` table from Snowflake + columns: + - name: id + description: Primary key for 'currencies' + tests: + - unique + - not_null + + + - name: currency_logs + description: Raw `currency_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'currency_logs' + tests: + - unique + - not_null + + + - name: currency_rate_logs + description: Raw `currency_rate_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'currency_rate_logs' + tests: + - unique + - not_null + + + - name: currency_rates + description: Raw `currency_rates` table from Snowflake + columns: + - name: id + description: Primary key for 'currency_rates' + tests: + - unique + - not_null + + + - name: districts + description: Raw `districts` table from Snowflake + columns: + - name: id + description: Primary key for 'districts' + tests: + - unique + - not_null + + + - name: documents + description: Raw `documents` table from Snowflake + columns: + - name: id + description: Primary key for 'documents' + tests: + - unique + - not_null + + + - name: employees + description: Raw `employees` table from Snowflake + columns: + - name: company_id + description: Primary key for 'employees' + + + - name: failed_jobs + description: Raw `failed_jobs` table from Snowflake + columns: + - name: id + description: Primary key for 'failed_jobs' + tests: + - unique + - not_null + + + - name: files + description: Raw `files` table from Snowflake + columns: + - name: id + description: Primary key for 'files' + tests: + - unique + - not_null + + + - name: group_transactions + description: Raw `group_transactions` table from Snowflake + columns: + - name: id + description: Primary key for 'group_transactions' + tests: + - unique + - not_null + + + - name: groups + description: Raw `groups` table from Snowflake + columns: + - name: id + description: Primary key for 'groups' + tests: + - unique + - not_null + + + - name: jobs + description: Raw `jobs` table from Snowflake + columns: + - name: id + description: Primary key for 'jobs' + tests: + - unique + - not_null + + + - name: migrations + description: Raw `migrations` table from Snowflake + columns: + - name: id + description: Primary key for 'migrations' + tests: + - unique + - not_null + + + - name: model_has_roles + description: Raw `model_has_roles` table from Snowflake + columns: + - name: role_id + description: Primary key for 'model_has_roles' + + + - name: notifications + description: Raw `notifications` table from Snowflake + columns: + - name: id + description: Primary key for 'notifications' + tests: + - unique + - not_null + + + - name: password_resets + description: Raw `password_resets` table from Snowflake + columns: + - name: id + description: Primary key for 'password_resets' + tests: + - unique + - not_null + + + - name: permissions + description: Raw `permissions` table from Snowflake + columns: + - name: id + description: Primary key for 'permissions' + tests: + - unique + - not_null + + + - name: role_has_permissions + description: Raw `role_has_permissions` table from Snowflake + columns: + - name: permission_id + description: Primary key for 'role_has_permissions' + + + - name: roles + description: Raw `roles` table from Snowflake + columns: + - name: id + description: Primary key for 'roles' + tests: + - unique + - not_null + + + - name: segment_companies + description: Raw `segment_companies` table from Snowflake + columns: + - name: segment_id + description: Primary key for 'segment_companies' + + + - name: segment_constants + description: Raw `segment_constants` table from Snowflake + columns: + - name: id + description: Primary key for 'segment_constants' + tests: + - unique + - not_null + + + - name: segments + description: Raw `segments` table from Snowflake + columns: + - name: id + description: Primary key for 'segments' + tests: + - unique + - not_null + + + - name: service_types + description: Raw `service_types` table from Snowflake + columns: + - name: id + description: Primary key for 'service_types' + tests: + - unique + - not_null + + + - name: states + description: Raw `states` table from Snowflake + columns: + - name: id + description: Primary key for 'states' + tests: + - unique + - not_null + + + - name: transaction_detail + description: Raw `transaction_detail` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_detail' + tests: + - unique + - not_null + + + - name: transaction_logs + description: Raw `transaction_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_logs' + tests: + - unique + - not_null + + + - name: transactions + description: Raw `transactions` table from Snowflake + columns: + - name: id + description: Primary key for 'transactions' + tests: + - unique + - not_null + + + - name: user_email_verifications + description: Raw `user_email_verifications` table from Snowflake + columns: + - name: id + description: Primary key for 'user_email_verifications' + tests: + - unique + - not_null + + + - name: users + description: Raw `users` table from Snowflake + columns: + - name: id + description: Primary key for 'users' + tests: + - unique + - not_null + + + - name: wallet_logs + description: Raw `wallet_logs` table from Snowflake + columns: + - name: id + description: Primary key for 'wallet_logs' + tests: + - unique + - not_null + + + - name: wallets + description: Raw `wallets` table from Snowflake + columns: + - name: id + description: Primary key for 'wallets' + tests: + - unique + - not_null \ No newline at end of file diff --git a/models/exchange/base/activity_logs/base_exchange__activity_logs.sql b/models/staging/exchange/base/base_exchange__activity_logs.sql similarity index 59% rename from models/exchange/base/activity_logs/base_exchange__activity_logs.sql rename to models/staging/exchange/base/base_exchange__activity_logs.sql index fed1f5d..148da48 100644 --- a/models/exchange/base/activity_logs/base_exchange__activity_logs.sql +++ b/models/staging/exchange/base/base_exchange__activity_logs.sql @@ -3,12 +3,11 @@ WITH activity_logs AS ( SELECT * FROM {{ source('src_exchange_mysql', 'activity_log') }} ), - -- LOGIC activity_logs_rename AS ( SELECT - activity_logs.id, + activity_logs.id AS activity_log_id, activity_logs.log_name, activity_logs.description, activity_logs.subject_id, @@ -16,9 +15,9 @@ activity_logs_rename AS ( activity_logs.causer_id, activity_logs.causer_type, activity_logs.properties, - activity_logs.created_at, - activity_logs.updated_at, - current_timestamp() AS _dbt_ran_at + activity_logs.created_at AS created_datetime, + activity_logs.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM activity_logs ), @@ -28,17 +27,26 @@ activity_logs_rename AS ( final__base_exchange__activity_logs AS ( SELECT - id, + -- ids + activity_log_id, + subject_id, + causer_id, + + -- dimensions log_name, description, - subject_id, subject_type, - causer_id, causer_type, properties, - created_at, - updated_at, - _dbt_ran_at + + -- measures + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime FROM activity_logs_rename ) diff --git a/models/staging/exchange/base/base_exchange__banks.sql b/models/staging/exchange/base/base_exchange__banks.sql new file mode 100644 index 0000000..896af7d --- /dev/null +++ b/models/staging/exchange/base/base_exchange__banks.sql @@ -0,0 +1,64 @@ +-- IMPORTS +WITH banks AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'banks') }} +), + +-- LOGIC +banks_rename AS ( + + SELECT + banks.id AS bank_id, + banks.company_id, + banks.reference AS recipient_reference, + banks.bank_name, + banks.holder_name, + banks.account_no AS account_number, + banks.bank_branch, + banks.swift AS swift_code, + banks.snap AS cnaps_code, + banks.type AS bank_type, + banks.default AS is_default, + banks.status, + banks.country_id, + banks.deleted_at AS deleted_datetime, + banks.created_at AS created_datetime, + banks.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM banks +), + + +-- FINAL +final__base_exchange__banks AS ( + + SELECT + -- ids + bank_id, + company_id, + country_id, + + -- dimensions + status, + bank_type, + is_default, + account_number, + bank_name, + bank_branch, + holder_name, + swift_code, + cnaps_code, + recipient_reference, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM banks_rename +) + +SELECT * FROM final__base_exchange__banks \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__booking_arch_logs.sql b/models/staging/exchange/base/base_exchange__booking_arch_logs.sql new file mode 100644 index 0000000..fce4089 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__booking_arch_logs.sql @@ -0,0 +1,63 @@ +-- IMPORTS +WITH booking_arch_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'booking_logs') }} +), + + +-- LOGIC +booking_arch_logs_rename AS ( + + SELECT + booking_arch_logs.id AS booking_log_id, + booking_arch_logs.booking_id, + booking_arch_logs.company_id, + booking_arch_logs.marking AS booking_marking_id, + booking_arch_logs.service_id AS service_type, + booking_arch_logs.bank_id, + booking_arch_logs.fix_amount AS fix_value, + booking_arch_logs.fix_currency_id, + booking_arch_logs.convertible_currency_id AS quote_currency_id, + booking_arch_logs.conversion_currency_id AS base_currency_id, + booking_arch_logs.status, + booking_arch_logs.deleted_at AS deleted_datetime, + booking_arch_logs.created_at AS created_datetime, + booking_arch_logs.updated_at As updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM booking_arch_logs + +), + +-- FINAL +final_base_exchange__booking_arch_logs AS ( + + SELECT + -- ids + booking_log_id, + booking_id, + booking_marking_id, + company_id, + bank_id, + fix_currency_id, + quote_currency_id, + base_currency_id, + + -- dimensions + status, + service_type, + + -- measures + fix_value, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM booking_arch_logs_rename +) + +SELECT * FROM final_base_exchange__booking_arch_logs \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__bookings.sql b/models/staging/exchange/base/base_exchange__bookings.sql new file mode 100644 index 0000000..8e5115a --- /dev/null +++ b/models/staging/exchange/base/base_exchange__bookings.sql @@ -0,0 +1,61 @@ +-- IMPORTS +WITH bookings AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'bookings') }} +), + + +-- LOGIC +bookings_rename AS ( + + SELECT + bookings.id AS booking_id, + bookings.company_id, + bookings.marking AS booking_marking_id, + bookings.service_id AS service_type, + bookings.bank_id, + bookings.fix_amount AS fix_value, + bookings.fix_currency_id, + bookings.convertible_currency_id AS quote_currency_id, + bookings.conversion_currency_id AS base_currency_id, + bookings.status, + bookings.deleted_at AS deleted_datetime, + bookings.created_at AS created_datetime, + bookings.updated_at As updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM bookings + +), + +-- FINAL +final_base_exchange__bookings AS ( + + SELECT + -- ids + booking_id, + booking_marking_id, + company_id, + bank_id, + fix_currency_id, + quote_currency_id, + base_currency_id, + + -- dimensions + status, + service_type, + + -- measures + fix_value, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM bookings_rename +) + +SELECT * FROM final_base_exchange__bookings \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__companies.sql b/models/staging/exchange/base/base_exchange__companies.sql new file mode 100644 index 0000000..f1b5008 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__companies.sql @@ -0,0 +1,54 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'companies') }} +), + + +-- LOGIC +companies_column_rename AS ( + + SELECT + companies.id AS company_id, + companies.name, + companies.reference AS company_marking_id, + companies.debtor AS autocount_id, + companies.type AS company_type, + companies.business_type, + companies.status, + companies.deleted_at AS deleted_datetime, + companies.created_at AS created_datetime, + companies.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies +), + + +-- FINAL +final__base_exchange__companies AS ( + + SELECT + -- ids + company_id, + company_marking_id, + autocount_id, + + -- dimension + name, + company_type, + business_type, + status, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM companies_column_rename + +) + +SELECT * FROM final__base_exchange__companies \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__companies_brg_segments.sql b/models/staging/exchange/base/base_exchange__companies_brg_segments.sql new file mode 100644 index 0000000..95ab9de --- /dev/null +++ b/models/staging/exchange/base/base_exchange__companies_brg_segments.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH snap_segment_companies AS ( + SELECT * FROM {{ ref('snap_exchange__segment_companies') }} +), + +-- LOGIC +segment_companies_rename AS ( + + SELECT + snap_segment_companies.segment_id, + snap_segment_companies.company_id, + snap_segment_companies.dbt_valid_from AS dbt_valid_from_datetime, + snap_segment_companies.dbt_valid_to AS dbt_valid_to_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM snap_segment_companies +), + + +-- FINAL +final__stg_exchange__companies_brg_segments AS ( + + SELECT + -- ids + segment_id, + company_id, + + -- dimensions + + -- measures + + -- date/times + dbt_valid_from_datetime, + dbt_valid_to_datetime, + + -- metadata + _dbt_ran_datetime + + FROM segment_companies_rename + +) + +SELECT * FROM final__stg_exchange__companies_brg_segments \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__company_addresses.sql b/models/staging/exchange/base/base_exchange__company_addresses.sql new file mode 100644 index 0000000..a828656 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__company_addresses.sql @@ -0,0 +1,81 @@ +-- IMPORTS +WITH addresses AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'addresses') }} +), + +-- LOGIC +addresses_rename AS ( + + SELECT + addresses.id AS company_address_id, + addresses.company_id, + addresses.country_id, + addresses.state_id, + addresses.district_id, + addresses.postcode, + addresses.street_one AS address_line_one, + addresses.street_two AS address_line_two, + addresses.billing AS is_billing_address, + addresses.deleted_at AS deleted_datetime, + addresses.created_at AS created_datetime, + addresses.updated_At AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM addresses +), + +remove_deleted_addresses AS ( + SELECT + * + + FROM addresses_rename + + WHERE + deleted_datetime IS NULL +), + +lastest_billing_and_non_billing_addresses AS ( + SELECT + *, + -- 1 will be latest updated address, rank by updated_at date + row_number() OVER + (PARTITION BY remove_deleted_addresses.company_id, remove_deleted_addresses.is_billing_address + ORDER BY remove_deleted_addresses.updated_datetime DESC) AS lastest_address_rank + + FROM remove_deleted_addresses + + QUALIFY + lastest_address_rank = 1 +), + +-- FINAL +final__base_exchange__company_addresses AS ( + + SELECT + -- ids + company_address_id, + company_id, + country_id, + state_id, + district_id, + + -- dimensions + postcode, + address_line_one, + address_line_two, + is_billing_address, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM lastest_billing_and_non_billing_addresses +) + +SELECT * FROM final__base_exchange__company_addresses \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__company_contacts.sql b/models/staging/exchange/base/base_exchange__company_contacts.sql new file mode 100644 index 0000000..163903a --- /dev/null +++ b/models/staging/exchange/base/base_exchange__company_contacts.sql @@ -0,0 +1,68 @@ +-- IMPORTS +WITH contacts AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'contacts') }} +), + + +-- LOGIC +contacts_rename AS ( + + SELECT + contacts.id AS contact_id, + contacts.company_id, + contacts.country_id, + contacts.reference AS contact_name, + contacts.phone, + contacts.email, + contacts.wechat_id AS wechat, + contacts.deleted_at AS deleted_datetime, + contacts.created_at AS created_datetime, + contacts.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM contacts +), + +get_latest_company_contacts AS ( + SELECT + *, + row_number() OVER + (PARTITION BY company_id + ORDER BY created_datetime DESC) AS last_row_number + + FROM contacts_rename + + QUALIFY + last_row_number = 1 +), + +-- FINAL +final__base_exchange__company_contacts AS ( + + SELECT + -- ids + contact_id, + company_id, + country_id, + + -- dimensions + contact_name, + phone, + email, + wechat, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM get_latest_company_contacts + +) + +SELECT * FROM final__base_exchange__company_contacts \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__countries.sql b/models/staging/exchange/base/base_exchange__countries.sql new file mode 100644 index 0000000..7f106b8 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__countries.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH countries AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'countries') }} +), + +-- LOGIC +countries_rename AS ( + + SELECT + countries.id AS country_id, + countries.name, + countries.short_code, + countries.phone_code, + countries.deleted_at AS deleted_datetime, + countries.created_at AS created_datetime, + countries.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM countries + +), + + +-- FINAL +final__base_exchange__countries AS ( + + SELECT + -- ids + country_id, + + -- dimensions + name, + short_code, + phone_code, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM countries_rename + +) + +SELECT * FROM final__base_exchange__countries \ No newline at end of file diff --git a/models/exchange/base/currencies/base_exchange__currencies.sql b/models/staging/exchange/base/base_exchange__currencies.sql similarity index 53% rename from models/exchange/base/currencies/base_exchange__currencies.sql rename to models/staging/exchange/base/base_exchange__currencies.sql index 72c670d..8e25891 100644 --- a/models/exchange/base/currencies/base_exchange__currencies.sql +++ b/models/staging/exchange/base/base_exchange__currencies.sql @@ -8,15 +8,15 @@ WITH currencies AS ( currencies_rename AS ( SELECT - currencies.id, + currencies.id AS currency_id, currencies.country_id, currencies.name, currencies.short_code, currencies.symbol, - currencies.deleted_at, - currencies.created_at, - currencies.updated_at, - current_timestamp() AS _dbt_ran_at + currencies.deleted_at AS deleted_datetime, + currencies.created_at AS created_datetime, + currencies.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM currencies ), @@ -26,15 +26,15 @@ currencies_rename AS ( final__base_exchange__currencies AS ( SELECT - id, + currency_id, country_id, name, short_code, symbol, - deleted_at, - created_at, - updated_at, - _dbt_ran_at + deleted_datetime, + created_datetime, + updated_datetime, + _dbt_ran_datetime FROM currencies_rename diff --git a/models/exchange/base/currency_rate_logs/base_exchange__currency_rate_logs.sql b/models/staging/exchange/base/base_exchange__currency_rate_logs.sql similarity index 59% rename from models/exchange/base/currency_rate_logs/base_exchange__currency_rate_logs.sql rename to models/staging/exchange/base/base_exchange__currency_rate_logs.sql index f1f39e0..04d4d63 100644 --- a/models/exchange/base/currency_rate_logs/base_exchange__currency_rate_logs.sql +++ b/models/staging/exchange/base/base_exchange__currency_rate_logs.sql @@ -2,6 +2,7 @@ WITH currency_rate_logs AS ( SELECT * FROM {{ source('src_exchange_mysql', 'currency_rate_logs') }} ), + -- LOGIC currency_rate_logs_clean_list AS ( @@ -17,13 +18,13 @@ currency_rate_logs_clean_list AS ( currency_rate_logs_rename_clean_wrong_rate AS ( SELECT - currency_rate_logs.id, + currency_rate_logs.id AS currency_rate_log_id, currency_rate_logs.currency_rate_id, - COALESCE(currency_rate_logs_clean_list.selling, currency_rate_logs.selling) AS base_to_quote_currency_exchange_rate, - currency_rate_logs.created_by, - currency_rate_logs.created_at, - currency_rate_logs.updated_at, - CURRENT_TIMESTAMP() AS _dbt_ran_at + COALESCE(currency_rate_logs_clean_list.selling, currency_rate_logs.selling) AS base_to_quote_currency_exchange_rate, + currency_rate_logs.created_by AS creator_user_id, + currency_rate_logs.created_at AS created_datetime, + currency_rate_logs.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM currency_rate_logs @@ -36,13 +37,22 @@ currency_rate_logs_rename_clean_wrong_rate AS ( final__base_exchange__currency_rate_logs AS ( SELECT - id, + -- ids + currency_rate_log_id, currency_rate_id, + creator_user_id, + + -- dimensions + + -- measures base_to_quote_currency_exchange_rate, - created_by, - created_at, - updated_at, - _dbt_ran_at + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime FROM currency_rate_logs_rename_clean_wrong_rate ) diff --git a/models/staging/exchange/base/base_exchange__currency_rates.sql b/models/staging/exchange/base/base_exchange__currency_rates.sql new file mode 100644 index 0000000..35419c8 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__currency_rates.sql @@ -0,0 +1,51 @@ +-- IMPORTS +WITH currency_rates AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'currency_rates') }} +), + + +-- LOGIC +currency_rates_rename AS ( + + SELECT + currency_rates.id AS currency_rate_id, + currency_rates.currency_id AS quote_currency_id, + currency_rates.selling AS base_to_quote_currency_exchange_rate, + currency_rates.payment_method_type AS payment_method, + currency_rates.service_id AS service_type, + currency_rates.deleted_at AS deleted_datetime, + currency_rates.created_at AS created_datetime, + currency_rates.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM currency_rates +), + + +-- FINAL +final__base_exchange__currency_rates AS ( + + SELECT + -- ids + currency_rate_id, + quote_currency_id, + + -- dimensions + payment_method, + service_type, + + -- measures + base_to_quote_currency_exchange_rate, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM currency_rates_rename +) + +SELECT * FROM final__base_exchange__currency_rates \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__districts.sql b/models/staging/exchange/base/base_exchange__districts.sql new file mode 100644 index 0000000..1c07f52 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__districts.sql @@ -0,0 +1,53 @@ +-- IMPORTS +WITH districts AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'districts') }} +), + + +-- LOGIC +districts_rename AS ( + + SELECT + districts.id AS district_id, + districts.country_id, + districts.state_id, + districts.name, + districts.postcode, + districts.status, + districts.deleted_at AS deleted_datetime, + districts.created_at AS created_datetime, + districts.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM districts +), + + +-- FINAL +final__base_exchange__districts AS ( + + SELECT + -- ids + district_id, + country_id, + state_id, + + -- dimensions + name, + postcode, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM districts_rename +) + +SELECT * FROM final__base_exchange__districts \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__documents.sql b/models/staging/exchange/base/base_exchange__documents.sql new file mode 100644 index 0000000..de5dc17 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__documents.sql @@ -0,0 +1,62 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'documents') }} +), + + +-- LOGIC +documents_rename AS ( + + SELECT + documents.id AS document_id, + documents.owner_type, + documents.owner_id, + documents.document_type, + documents.reference, + documents.status, + documents.issued_date AS issued_datetime, + documents.expired_date AS expired_datetime, + documents.approver AS approver_user_id, + documents.approval_date AS approved_datetime, + documents.deleted_at AS deleted_datetime, + documents.created_at AS created_datetime, + documents.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents +), + + +-- FINAL +final__base_exchange__documents AS ( + + SELECT + -- ids + document_id, + owner_id, + + -- dimensions + owner_type, + document_type, + reference, + status, + + -- measures + + -- date/times + issued_datetime, + expired_datetime, + approver_user_id, + approved_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_rename + +) + +SELECT * FROM final__base_exchange__documents \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__group_costs.sql b/models/staging/exchange/base/base_exchange__group_costs.sql new file mode 100644 index 0000000..834b647 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__group_costs.sql @@ -0,0 +1,62 @@ +-- IMPORTS +WITH groups AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'groups') }} +), + + +-- LOGIC +groups_rename AS ( + + SELECT + groups.id AS group_cost_id, + groups.reference AS group_marking_id, + groups.issuer AS supplier_user_id, + groups.amount AS base_amount, + groups.original_amount AS quote_amount, + groups.currency_id AS base_currency_id, + groups.original_currency_id AS quote_currency_id, + groups.currency_rate AS base_to_quote_currency_exchange_rate, + groups.tax AS base_tax, + groups.service_charge AS base_service_charge, + groups.status, + groups.created_at AS created_datetime, + groups.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM groups +), + + +-- FINAL +final__base_exchange__group_costs AS ( + + SELECT + -- ids + group_cost_id, + group_marking_id, + supplier_user_id, + base_currency_id, + quote_currency_id, + + -- dimensions + status, + + -- measures + base_amount, + quote_amount, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM groups_rename + +) + +SELECT * FROM final__base_exchange__group_costs \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__segments.sql b/models/staging/exchange/base/base_exchange__segments.sql new file mode 100644 index 0000000..de3f2f0 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__segments.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH segments AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'segments') }} +), + +-- LOGIC +segments_rename AS ( + + SELECT + segments.id AS segment_id, + UPPER(segments.name) AS name, + segments.type AS segment_type, + segments.deleted_at AS deleted_datetime, + segments.created_at AS created_datetime, + segments.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM segments +), + + +-- FINAL +final__base_exchange__segments AS ( + + SELECT + -- ids + segment_id, + + -- dimensions + name, + segment_type, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM segments_rename +) + +SELECT * FROM final__base_exchange__segments \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__service_types.sql b/models/staging/exchange/base/base_exchange__service_types.sql new file mode 100644 index 0000000..998e326 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__service_types.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH service_types AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'service_types') }} +), + + +-- LOGIC +service_types_rename AS ( + + SELECT + service_types.id AS service_type_id, + service_types.name, + service_types.status, + service_types.deleted_at AS deleted_datetime, + service_types.created_at AS created_datetime, + service_types.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM service_types +), + + +-- FINAL +final__base_exchange__service_types AS ( + + SELECT + -- ids + service_type_id, + + -- dimensions + name, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM service_types_rename + +) + +SELECT * FROM final__base_exchange__service_types \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__states.sql b/models/staging/exchange/base/base_exchange__states.sql new file mode 100644 index 0000000..a5206b4 --- /dev/null +++ b/models/staging/exchange/base/base_exchange__states.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH states AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'states') }} +), + + +-- LOGIC +states_rename AS ( + + SELECT + states.id AS state_id, + states.country_id, + states.name, + states.status, + states.deleted_at AS deleted_datetime, + states.created_at AS created_datetime, + states.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM states +), + + +-- FINAL +final__base_exchange__states AS ( + + SELECT + -- ids + state_id, + country_id, + + -- dimensions + name, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM states_rename + +) + +SELECT * FROM final__base_exchange__states \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__transaction_arch_logs.sql b/models/staging/exchange/base/base_exchange__transaction_arch_logs.sql new file mode 100644 index 0000000..995e04d --- /dev/null +++ b/models/staging/exchange/base/base_exchange__transaction_arch_logs.sql @@ -0,0 +1,81 @@ +-- IMPORTS +WITH transaction_logs AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'transaction_logs') }} +), + + +-- LOGIC +transaction_logs_rename AS ( + + SELECT + transaction_logs.id AS transaction_log_id, + transaction_logs.transaction_id, + transaction_logs.owner_type, + transaction_logs.owner_id, + transaction_logs.type AS transaction_type, + transaction_logs.issuer AS issuer_user_id, + transaction_logs.receiver AS receiver_user_id, + transaction_logs.recipient_bank_account_id AS recipient_bank_id, + transaction_logs.payment_method, + transaction_logs.payment_reference, + transaction_logs.bill_no AS bill_number, + transaction_logs.amount AS base_value, + transaction_logs.original_amount AS quote_value, + transaction_logs.currency_id AS base_currency_id, + transaction_logs.original_currency_id AS quote_currency_id, + transaction_logs.currency_rate AS base_to_quote_currency_exchange_rate, + transaction_logs.tax AS base_tax, + transaction_logs.service_charge AS base_service_charge, + transaction_logs.expires_on AS expired_datetime, + transaction_logs.status, + transaction_logs.deleted_at AS deleted_datetime, + transaction_logs.created_at AS created_datetime, + transaction_logs.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_logs +), + + +-- FINAL +final__base_exchange__transaction_logs AS ( + + SELECT + -- ids + transaction_log_id, + transaction_id, + owner_id, + issuer_user_id, + receiver_user_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + + -- dimentions + owner_type, + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transaction_logs_rename +) + +SELECT * FROM final__base_exchange__transaction_logs \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__transactions.sql b/models/staging/exchange/base/base_exchange__transactions.sql new file mode 100644 index 0000000..d336dcd --- /dev/null +++ b/models/staging/exchange/base/base_exchange__transactions.sql @@ -0,0 +1,79 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'transactions') }} +), + + +-- LOGIC +transactions_rename AS ( + + SELECT + transactions.id AS transaction_id, + transactions.owner_type, + transactions.owner_id, + transactions.type AS transaction_type, + transactions.issuer AS issuer_user_id, + transactions.receiver AS receiver_user_id, + transactions.recipient_bank_account_id AS recipient_bank_id, + transactions.payment_method, + transactions.payment_reference, + transactions.bill_no AS bill_number, + transactions.amount AS base_value, + transactions.original_amount AS quote_value, + transactions.currency_id AS base_currency_id, + transactions.original_currency_id AS quote_currency_id, + transactions.currency_rate AS base_to_quote_currency_exchange_rate, + transactions.tax AS base_tax, + transactions.service_charge AS base_service_charge, + transactions.expires_on AS expired_datetime, + transactions.status, + transactions.deleted_at AS deleted_datetime, + transactions.created_at AS created_datetime, + transactions.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transactions +), + + +-- FINAL +final__base_exchange__transactions AS ( + + SELECT + -- ids + transaction_id, + owner_id, + issuer_user_id, + receiver_user_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + + -- dimentions + owner_type, + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transactions_rename +) + +SELECT * FROM final__base_exchange__transactions \ No newline at end of file diff --git a/models/exchange/base/user_email_verifications/base_exchange__user_email_verifications.sql b/models/staging/exchange/base/base_exchange__user_email_verifications.sql similarity index 55% rename from models/exchange/base/user_email_verifications/base_exchange__user_email_verifications.sql rename to models/staging/exchange/base/base_exchange__user_email_verifications.sql index b5e2d38..18b2140 100644 --- a/models/exchange/base/user_email_verifications/base_exchange__user_email_verifications.sql +++ b/models/staging/exchange/base/base_exchange__user_email_verifications.sql @@ -3,20 +3,17 @@ WITH user_email_verifications AS ( SELECT * FROM {{ source('src_exchange_mysql', 'user_email_verifications') }} ), - -- LOGIC user_email_verifications_rename AS ( SELECT - user_email_verifications.id, + user_email_verifications.id AS user_email_verification_id, user_email_verifications.email, - user_email_verifications.token, user_email_verifications.is_complete AS is_completed, - user_email_verifications.is_active AS is_activated, user_email_verifications.is_sent, - user_email_verifications.created_at, - user_email_verifications.updated_at, - current_timestamp() AS _dbt_ran_at + user_email_verifications.created_at AS created_datetime, + user_email_verifications.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM user_email_verifications ), @@ -26,15 +23,22 @@ user_email_verifications_rename AS ( final__base_exchange__user_email_verifications AS ( SELECT - id, + -- ids + user_email_verification_id, + + -- dimensions email, - token, is_completed, - is_activated, is_sent, - created_at, - updated_at, - _dbt_ran_at + + -- measures + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime FROM user_email_verifications_rename diff --git a/models/staging/exchange/base/base_exchange__users.sql b/models/staging/exchange/base/base_exchange__users.sql new file mode 100644 index 0000000..3604d4e --- /dev/null +++ b/models/staging/exchange/base/base_exchange__users.sql @@ -0,0 +1,51 @@ +-- IMPORTS +WITH users AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'users') }} +), + +-- LOGIC +users_rename_and_clean_null_data AS ( + + SELECT + users.id AS user_id, + users.name, + users.email, + users.type AS user_type, + users.status, + users.deleted_at AS deleted_datetime, + + -- User without create_datetime because it migrate from old system, we get minimum user create datetime to replace it + COALESCE(users.created_at, CAST('2021-05-13 00:00:00 +08:00' AS DATETIME)) AS created_datetime, + users.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM users +), + +-- FINAL +final__base_exchange__users AS ( + + SELECT + -- ids + user_id, + + -- dimensions + name, + email, + user_type, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM users_rename_and_clean_null_data +) + +SELECT * FROM final__base_exchange__users \ No newline at end of file diff --git a/models/shipping/base/activity_logs/base_shipping__activity_logs.sql b/models/staging/exchange/stg_exchange__activity_logs.sql similarity index 50% rename from models/shipping/base/activity_logs/base_shipping__activity_logs.sql rename to models/staging/exchange/stg_exchange__activity_logs.sql index 8f30a63..a45426b 100644 --- a/models/shipping/base/activity_logs/base_shipping__activity_logs.sql +++ b/models/staging/exchange/stg_exchange__activity_logs.sql @@ -1,14 +1,13 @@ -- IMPORTS WITH activity_logs AS ( - SELECT * FROM {{ source('src_shipping_mysql', 'activity_log') }} + SELECT * FROM {{ source('src_exchange_mysql', 'activity_log') }} ), - -- LOGIC activity_logs_rename AS ( SELECT - activity_logs.id, + activity_logs.id AS activity_log_id, activity_logs.log_name, activity_logs.description, activity_logs.subject_id, @@ -16,31 +15,40 @@ activity_logs_rename AS ( activity_logs.causer_id, activity_logs.causer_type, activity_logs.properties, - activity_logs.created_at AS created_datetime, - activity_logs.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at + activity_logs.created_at AS created_datetime, + activity_logs.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM activity_logs ), -- FINAL -final__base_exchange__activity_logs AS ( +final__stg_exchange__activity_logs AS ( SELECT - id, + -- ids + activity_log_id, + subject_id, + causer_id, + + -- dimensions log_name, description, - subject_id, subject_type, - causer_id, causer_type, properties, + + -- measures + + -- date/times created_datetime, updated_datetime, - _dbt_ran_at + + -- metadata + _dbt_ran_datetime FROM activity_logs_rename ) -SELECT * FROM final__base_exchange__activity_logs \ No newline at end of file +SELECT * FROM final__stg_exchange__activity_logs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__banks.sql b/models/staging/exchange/stg_exchange__banks.sql new file mode 100644 index 0000000..54cdd74 --- /dev/null +++ b/models/staging/exchange/stg_exchange__banks.sql @@ -0,0 +1,81 @@ +-- IMPORTS +WITH banks AS ( + SELECT * FROM {{ ref('base_exchange__banks') }} +), + +bank_types AS ( + SELECT * FROM {{ ref('seed_exchange__bank_types') }} + WHERE category = 'REPHRASE' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +banks_change_column_value AS ( + + SELECT + banks.bank_id, + banks.company_id, + banks.recipient_reference, + banks.bank_name, + banks.holder_name, + banks.account_number, + banks.bank_branch, + banks.swift_code, + banks.cnaps_code, + COALESCE(bank_types.name, banks.bank_type::string) AS bank_type, + banks.is_default, + COALESCE(status.name, banks.status::string) AS status, + banks.country_id, + banks.deleted_datetime, + banks.created_datetime, + banks.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM banks + + LEFT JOIN bank_types + ON (banks.bank_type = bank_types.id) + + LEFT JOIN status + ON (banks.status = status.id) + +), + + +-- FINAL +final__stg_exchange__banks AS ( + + SELECT + -- ids + bank_id, + company_id, + country_id, + + -- dimensions + status, + bank_type, + is_default, + account_number, + bank_name, + bank_branch, + holder_name, + swift_code, + cnaps_code, + recipient_reference, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM banks_change_column_value +) + +SELECT * FROM final__stg_exchange__banks \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__booking_documents.sql b/models/staging/exchange/stg_exchange__booking_documents.sql new file mode 100644 index 0000000..f5f8388 --- /dev/null +++ b/models/staging/exchange/stg_exchange__booking_documents.sql @@ -0,0 +1,50 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + IFF(documents.owner_type = 'App\\Models\\Booking', documents.owner_id, null) AS booking_id, + documents.document_type, + documents.deleted_datetime, + documents.created_datetime, + documents.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents + + WHERE + owner_type = 'App\\Models\\Booking' +), + + +-- FINAL +final__stg_exchange__booking_documents AS ( + + SELECT + -- ids + document_id, + booking_id, + + -- dimensions + document_type, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_join_status + +) + +SELECT * FROM final__stg_exchange__booking_documents \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__booking_logs.sql b/models/staging/exchange/stg_exchange__booking_logs.sql new file mode 100644 index 0000000..cdbede2 --- /dev/null +++ b/models/staging/exchange/stg_exchange__booking_logs.sql @@ -0,0 +1,124 @@ +-- IMPORTS +WITH booking_arch_logs AS ( + SELECT * FROM {{ ref('base_exchange__booking_arch_logs') }} +), + +bookings AS ( + SELECT * FROM {{ ref('base_exchange__bookings') }} +), + +activity_logs AS ( + SELECT * FROM {{ ref('base_exchange__activity_logs') }} +), + +service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +booking_id_brg_user_id AS ( + + SELECT + subject_id AS booking_id, + causer_id AS user_id + FROM + activity_logs + WHERE + subject_type='App\\Models\\Booking' + AND + description='created' +), + +bookings_generate_booking_log_id AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(booking_id, created_datetime)) AS booking_log_id, + * + + FROM bookings +), + +booking_arch_logs_union_bookings AS ( + SELECT * FROM bookings_generate_booking_log_id + UNION + SELECT * FROM booking_arch_logs +), + + +booking_logs_map_column_and_value_transform AS ( + + SELECT + booking_arch_logs_union_bookings.booking_log_id, + booking_arch_logs_union_bookings.booking_id, + booking_arch_logs_union_bookings.booking_marking_id, + booking_arch_logs_union_bookings.company_id, + booking_arch_logs_union_bookings.bank_id, + + booking_id_brg_user_id.user_id, + + COALESCE(service_types.name, booking_arch_logs_union_bookings.service_type::string) AS service_type, + + booking_arch_logs_union_bookings.fix_value, + + booking_arch_logs_union_bookings.fix_currency_id, + booking_arch_logs_union_bookings.quote_currency_id, + booking_arch_logs_union_bookings.base_currency_id, + COALESCE(status.name, booking_arch_logs_union_bookings.status::string) AS status, + + booking_arch_logs_union_bookings.deleted_datetime, + booking_arch_logs_union_bookings.created_datetime, + booking_arch_logs_union_bookings.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM booking_arch_logs_union_bookings + + LEFT JOIN booking_id_brg_user_id + ON (booking_arch_logs_union_bookings.booking_id = booking_id_brg_user_id.booking_id) + + LEFT JOIN service_types + ON (booking_arch_logs_union_bookings.service_type = service_types.service_type_id) + + LEFT JOIN status + ON (booking_arch_logs_union_bookings.status = status.id) +), + +-- FINAL +final__stg_exchange__booking_logs AS ( + + SELECT + -- ids + booking_log_id, + booking_id, + booking_marking_id, + company_id, + user_id, + bank_id, + fix_currency_id, + quote_currency_id, + base_currency_id, + + -- dimensions + service_type, + status, + + -- measures + fix_value, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM booking_logs_map_column_and_value_transform + +) + +SELECT * FROM final__stg_exchange__booking_logs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__bookings.sql b/models/staging/exchange/stg_exchange__bookings.sql new file mode 100644 index 0000000..c342389 --- /dev/null +++ b/models/staging/exchange/stg_exchange__bookings.sql @@ -0,0 +1,104 @@ +-- IMPORTS +WITH bookings AS ( + SELECT * FROM {{ ref('base_exchange__bookings') }} +), + +activity_logs AS ( + SELECT * FROM {{ ref('base_exchange__activity_logs') }} +), + +service_types AS ( + SELECT * FROM {{ ref('base_exchange__service_types') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +booking_id_brg_user_id AS ( + + SELECT + subject_id AS booking_id, + causer_id AS user_id + FROM + activity_logs + WHERE + subject_type='App\\Models\\Booking' + AND + description='created' +), + + +bookings_map_column_and_value_transform AS ( + + SELECT + bookings.booking_id, + bookings.booking_marking_id, + bookings.company_id, + bookings.bank_id, + + booking_id_brg_user_id.user_id, + + COALESCE(service_types.name, bookings.service_type::string) AS service_type, + + bookings.fix_value, + + bookings.fix_currency_id, + bookings.quote_currency_id, + bookings.base_currency_id, + COALESCE(status.name, bookings.status::string) AS status, + + bookings.deleted_datetime, + bookings.created_datetime, + bookings.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM bookings + + LEFT JOIN booking_id_brg_user_id + ON (bookings.booking_id = booking_id_brg_user_id.booking_id) + + LEFT JOIN service_types + ON (bookings.service_type = service_types.service_type_id) + + LEFT JOIN status + ON (bookings.status = status.id) +), + +-- FINAL +final__stg_exchange__booking_logs AS ( + + SELECT + -- ids + booking_id, + booking_marking_id, + company_id, + user_id, + bank_id, + fix_currency_id, + quote_currency_id, + base_currency_id, + + -- dimensions + service_type, + status, + + -- measures + fix_value, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM bookings_map_column_and_value_transform + +) + +SELECT * FROM final__stg_exchange__booking_logs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__companies.sql b/models/staging/exchange/stg_exchange__companies.sql new file mode 100644 index 0000000..3cfe5b0 --- /dev/null +++ b/models/staging/exchange/stg_exchange__companies.sql @@ -0,0 +1,81 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ ref('base_exchange__companies') }} +), + +company_types AS ( + SELECT * FROM {{ ref('seed_exchange__company_types') }} + WHERE category = '2_AS_CORPORATE' +), + +business_types AS ( + SELECT * FROM {{ ref('seed_exchange__business_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +companies_map_constant AS ( + + SELECT + companies.company_id, + companies.name, + companies.company_marking_id, + companies.autocount_id, + + COALESCE(company_types.name, companies.company_type::string) AS company_type, + COALESCE(business_types.name, companies.business_type::string) AS business_type, + COALESCE(status.name, companies.status::string) AS status, + + companies.deleted_datetime, + companies.created_datetime, + companies.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies + + LEFT JOIN company_types + ON (companies.company_type = company_types.id) + + LEFT JOIN business_types + ON (companies.business_type = business_types.id) + + LEFT JOIN status + ON (companies.status = status.id) +), + +-- FINAL +final__stg_companies__exchange AS ( + + SELECT + -- ids + company_id, + company_marking_id, + autocount_id, + + -- dimensions + name, + company_type, + business_type, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM companies_map_constant + +) + +SELECT * FROM final__stg_companies__exchange \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__companies_brg_segments.sql b/models/staging/exchange/stg_exchange__companies_brg_segments.sql new file mode 100644 index 0000000..95ab9de --- /dev/null +++ b/models/staging/exchange/stg_exchange__companies_brg_segments.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH snap_segment_companies AS ( + SELECT * FROM {{ ref('snap_exchange__segment_companies') }} +), + +-- LOGIC +segment_companies_rename AS ( + + SELECT + snap_segment_companies.segment_id, + snap_segment_companies.company_id, + snap_segment_companies.dbt_valid_from AS dbt_valid_from_datetime, + snap_segment_companies.dbt_valid_to AS dbt_valid_to_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM snap_segment_companies +), + + +-- FINAL +final__stg_exchange__companies_brg_segments AS ( + + SELECT + -- ids + segment_id, + company_id, + + -- dimensions + + -- measures + + -- date/times + dbt_valid_from_datetime, + dbt_valid_to_datetime, + + -- metadata + _dbt_ran_datetime + + FROM segment_companies_rename + +) + +SELECT * FROM final__stg_exchange__companies_brg_segments \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__companies_brg_users.sql b/models/staging/exchange/stg_exchange__companies_brg_users.sql new file mode 100644 index 0000000..f7d77c8 --- /dev/null +++ b/models/staging/exchange/stg_exchange__companies_brg_users.sql @@ -0,0 +1,39 @@ +-- IMPORTS +WITH employees AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'employees') }} +), + + +-- LOGIC +employees_column_rename_and_drop_unuse_column AS ( + + SELECT + employees.company_id, + employees.user_id, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM employees +), + + +-- FINAL +final__stg_exchange__companies_brg_users AS ( + + SELECT + -- ids + company_id, + user_id, + + -- dimensions + + -- measures + + -- date/times + + -- metadata + _dbt_ran_datetime + + FROM employees_column_rename_and_drop_unuse_column +) + +SELECT * FROM final__stg_exchange__companies_brg_users \ No newline at end of file diff --git a/models/exchange/staging/company_addresses/stg_exchange__company_addresses.sql b/models/staging/exchange/stg_exchange__company_addresses.sql similarity index 67% rename from models/exchange/staging/company_addresses/stg_exchange__company_addresses.sql rename to models/staging/exchange/stg_exchange__company_addresses.sql index 9fb353b..fe4ba5b 100644 --- a/models/exchange/staging/company_addresses/stg_exchange__company_addresses.sql +++ b/models/staging/exchange/stg_exchange__company_addresses.sql @@ -1,6 +1,6 @@ -- IMPORTS WITH company_addresses AS ( - SELECT * FROM {{ ref('base_exchange__addresses') }} + SELECT * FROM {{ ref('base_exchange__company_addresses') }} ), countries AS ( @@ -19,7 +19,7 @@ districts AS ( company_addressess_join_companies_countries_states_districts AS ( SELECT - company_addresses.id, + company_addresses.company_address_id, company_addresses.company_id, company_addresses.country_id, @@ -37,45 +37,54 @@ company_addressess_join_companies_countries_states_districts AS ( company_addresses.address_line_one, company_addresses.address_line_two, company_addresses.is_billing_address, - company_addresses.deleted_at, - company_addresses.created_at, - company_addresses.updated_At, - CURRENT_TIMESTAMP() AS _dbt_ran_at + company_addresses.deleted_datetime, + company_addresses.created_datetime, + company_addresses.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM company_addresses LEFT JOIN countries - ON (company_addresses.country_id = countries.id) + ON (company_addresses.country_id = countries.country_id) LEFT JOIN states - ON (company_addresses.state_id = states.id) + ON (company_addresses.state_id = states.state_id) LEFT JOIN districts - ON (company_addresses.district_id = districts.id) + ON (company_addresses.district_id = districts.district_id) ), -- FINAL final__stg_exchange__company_addresses AS ( SELECT - id, + -- ids + company_address_id, company_id, country_id, + state_id, + district_id, + + -- dimensions country_name, country_short_code, - country_phone_code - state_id, - state_name, - district_id, + country_phone_code, + state_name, district_name, postcode, address_line_one, address_line_two, is_billing_address, - deleted_at, - created_at, - updated_At, - _dbt_ran_at + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime FROM company_addressess_join_companies_countries_states_districts ) diff --git a/models/staging/exchange/stg_exchange__company_contacts.sql b/models/staging/exchange/stg_exchange__company_contacts.sql new file mode 100644 index 0000000..428c239 --- /dev/null +++ b/models/staging/exchange/stg_exchange__company_contacts.sql @@ -0,0 +1,67 @@ +-- IMPORTS +WITH contacts AS ( + SELECT * FROM {{ ref('base_exchange__company_contacts') }} +), + +countries AS ( + SELECT * FROM {{ ref('base_exchange__countries') }} +), + + +-- LOGIC +companies_join_countries AS ( + + SELECT + contacts.contact_id, + contacts.company_id, + + contacts.country_id, + countries.name AS country_name, + + contacts.contact_name, + contacts.phone, + contacts.email, + contacts.wechat, + contacts.deleted_datetime, + contacts.created_datetime, + contacts.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM contacts + + LEFT JOIN countries + ON (contacts.country_id = countries.country_id) +), + + +-- FINAL +final__stg_exchange__company_contacts AS ( + + SELECT + -- ids + contact_id, + company_id, + country_id, + + -- dimensions + country_name, + contact_name, + phone, + email, + wechat, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM companies_join_countries + +) + +SELECT * FROM final__stg_exchange__company_contacts \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__company_documents.sql b/models/staging/exchange/stg_exchange__company_documents.sql new file mode 100644 index 0000000..5048939 --- /dev/null +++ b/models/staging/exchange/stg_exchange__company_documents.sql @@ -0,0 +1,66 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + IFF(documents.owner_type = 'App\\Models\\Company', documents.owner_id, null) AS company_id, + documents.document_type, + documents.reference, + COALESCE(status.name, documents.status::string) AS status, + documents.approver_user_id, + documents.approved_datetime, + documents.deleted_datetime, + documents.created_datetime, + documents.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents + + LEFT JOIN status + ON (documents.status = status.id) + + WHERE + owner_type = 'App\\Models\\Company' +), + + +-- FINAL +final__stg_exchange__company_documents AS ( + + SELECT + -- ids + document_id, + company_id, + approver_user_id, + + -- dimensions + status, + document_type, + reference, + + -- measures + + -- date/times + approved_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_join_status + +) + +SELECT * FROM final__stg_exchange__company_documents \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__company_wallets.sql b/models/staging/exchange/stg_exchange__company_wallets.sql new file mode 100644 index 0000000..3abfd48 --- /dev/null +++ b/models/staging/exchange/stg_exchange__company_wallets.sql @@ -0,0 +1,51 @@ +-- IMPORTS +WITH wallets AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'wallets') }} +), + +-- LOGIC +wallets_rename AS ( + + SELECT + wallets.id AS wallet_id, + IFF(wallets.owner_type = 'App//Models//Company', wallets.owner_id, null) AS company_id, + wallets.code AS wallet_code, + wallets.currency_id, + wallets.amount, + wallets.deleted_at AS deleted_datetime, + wallets.created_at AS created_datetime, + wallets.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM wallets +), + + +-- FINAL +final__stg_exchange__company_wallets AS ( + + SELECT + -- ids + wallet_id, + wallet_code, + company_id, + currency_id, + + -- dimensions + + -- measures + amount, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM wallets_rename + +) + +SELECT * FROM final__stg_exchange__company_wallets \ No newline at end of file diff --git a/models/exchange/staging/currencies/stg_exchange__currencies.sql b/models/staging/exchange/stg_exchange__currencies.sql similarity index 51% rename from models/exchange/staging/currencies/stg_exchange__currencies.sql rename to models/staging/exchange/stg_exchange__currencies.sql index 8827d2e..7ac8af4 100644 --- a/models/exchange/staging/currencies/stg_exchange__currencies.sql +++ b/models/staging/exchange/stg_exchange__currencies.sql @@ -13,23 +13,23 @@ countries AS ( currencies_join_countries AS ( SELECT - currencies.id, + currencies.currency_id, currencies.country_id, - countries.name AS country_name, + countries.name AS country_name, currencies.name, currencies.short_code, currencies.symbol, - currencies.deleted_at, - currencies.created_at, - currencies.updated_at, - current_timestamp() AS _dbt_ran_at + currencies.deleted_datetime, + currencies.created_datetime, + currencies.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM currencies LEFT JOIN countries - ON (currencies.country_id = countries.id) + ON (currencies.country_id = countries.country_id) ), @@ -37,16 +37,25 @@ currencies_join_countries AS ( final__stg_exchange__currencies AS ( SELECT - id, + -- ids + currency_id, country_id, - country_name, + + -- dimensions name, short_code, symbol, - deleted_at, - created_at, - updated_at, - _dbt_ran_at + country_name, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime FROM currencies_join_countries diff --git a/models/exchange/staging/currency_rate_logs/stg_exchange__currency_rate_logs.sql b/models/staging/exchange/stg_exchange__currency_rate_logs.sql similarity index 71% rename from models/exchange/staging/currency_rate_logs/stg_exchange__currency_rate_logs.sql rename to models/staging/exchange/stg_exchange__currency_rate_logs.sql index f9cf5ed..5dd8e8a 100644 --- a/models/exchange/staging/currency_rate_logs/stg_exchange__currency_rate_logs.sql +++ b/models/staging/exchange/stg_exchange__currency_rate_logs.sql @@ -25,29 +25,29 @@ service_types AS ( currency_rate_logs_join_currency_rates AS ( SELECT - currency_rate_logs.id, + currency_rate_logs.currency_rate_log_id, currency_rate_logs.currency_rate_id, currency_rates.quote_currency_id, - currency_rate_logs.base_to_quote_currency_exchange_rate AS base_to_quote_currency_exchange_rate, + currency_rate_logs.base_to_quote_currency_exchange_rate, currency_rates.payment_method, currency_rates.service_type, - currency_rate_logs.created_by, - currency_rate_logs.created_at, - currency_rate_logs.updated_at + currency_rate_logs.creator_user_id, + currency_rate_logs.created_datetime, + currency_rate_logs.updated_datetime FROM currency_rate_logs LEFT JOIN currency_rates - ON (currency_rate_logs.currency_rate_id = currency_rates.id) + ON (currency_rate_logs.currency_rate_id = currency_rates.currency_rate_id) ), currency_rate_logs_quote_currency_id_join_currencies AS ( SELECT - currency_rate_logs_join_currency_rates.id, + currency_rate_logs_join_currency_rates.currency_rate_log_id, currency_rate_logs_join_currency_rates.currency_rate_id, currency_rate_logs_join_currency_rates.quote_currency_id, @@ -57,37 +57,34 @@ currency_rate_logs_quote_currency_id_join_currencies AS ( currency_rate_logs_join_currency_rates.base_to_quote_currency_exchange_rate, - currency_rate_logs_join_currency_rates.payment_method, - payment_methods.name AS payment_method_name, + COALESCE(payment_methods.name, currency_rate_logs_join_currency_rates.payment_method::string) AS payment_method, + COALESCE(service_types.name, currency_rate_logs_join_currency_rates.service_type::string) AS service_type, - currency_rate_logs_join_currency_rates.service_type, - service_types.name AS service_type_name, - - currency_rate_logs_join_currency_rates.created_by, - currency_rate_logs_join_currency_rates.created_at, - currency_rate_logs_join_currency_rates.updated_at + currency_rate_logs_join_currency_rates.creator_user_id, + currency_rate_logs_join_currency_rates.created_datetime, + currency_rate_logs_join_currency_rates.updated_datetime FROM currency_rate_logs_join_currency_rates LEFT JOIN currencies - ON (currency_rate_logs_join_currency_rates.quote_currency_id = currencies.id) + ON (currency_rate_logs_join_currency_rates.quote_currency_id = currencies.currency_id) LEFT JOIN payment_methods ON (currency_rate_logs_join_currency_rates.payment_method = payment_methods.id) LEFT JOIN service_types - ON (currency_rate_logs_join_currency_rates.service_type = service_types.id) + ON (currency_rate_logs_join_currency_rates.service_type = service_types.service_type_id) ), currency_rate_logs_add_dense_rank AS ( SELECT - currency_rate_logs_quote_currency_id_join_currencies.id, + currency_rate_logs_quote_currency_id_join_currencies.currency_rate_log_id, dense_rank() OVER (PARTITION BY currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id - ORDER BY currency_rate_logs_quote_currency_id_join_currencies.created_at) AS dense_rank_index, + ORDER BY currency_rate_logs_quote_currency_id_join_currencies.created_datetime) AS dense_rank_index, currency_rate_logs_quote_currency_id_join_currencies.currency_rate_id AS currency_rate_id, currency_rate_logs_quote_currency_id_join_currencies.quote_currency_id, @@ -95,44 +92,40 @@ currency_rate_logs_add_dense_rank AS ( currency_rate_logs_quote_currency_id_join_currencies.quote_currency_short_code, currency_rate_logs_quote_currency_id_join_currencies.quote_currency_symbol, currency_rate_logs_quote_currency_id_join_currencies.payment_method, - currency_rate_logs_quote_currency_id_join_currencies.payment_method_name, currency_rate_logs_quote_currency_id_join_currencies.service_type, - currency_rate_logs_quote_currency_id_join_currencies.service_type_name, currency_rate_logs_quote_currency_id_join_currencies.base_to_quote_currency_exchange_rate, - currency_rate_logs_quote_currency_id_join_currencies.created_by AS created_by, - currency_rate_logs_quote_currency_id_join_currencies.created_at AS created_at + currency_rate_logs_quote_currency_id_join_currencies.creator_user_id, + currency_rate_logs_quote_currency_id_join_currencies.created_datetime FROM currency_rate_logs_quote_currency_id_join_currencies ORDER BY currency_rate_id, - created_at + created_datetime ), currency_rate_logs_resturucure AS ( SELECT - currency_rate_logs_add_dense_rank.id, + currency_rate_logs_add_dense_rank.currency_rate_log_id, currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.quote_currency_id, currency_rate_logs_add_dense_rank.quote_currency_name, currency_rate_logs_add_dense_rank.quote_currency_short_code, currency_rate_logs_add_dense_rank.quote_currency_symbol, currency_rate_logs_add_dense_rank.payment_method, - currency_rate_logs_add_dense_rank.payment_method_name, currency_rate_logs_add_dense_rank.service_type, - currency_rate_logs_add_dense_rank.service_type_name, currency_rate_logs_add_dense_rank.base_to_quote_currency_exchange_rate, - currency_rate_logs_add_dense_rank.created_by, + currency_rate_logs_add_dense_rank.creator_user_id, CASE WHEN currency_rate_logs_add_dense_rank.dense_rank_index=1 THEN CAST('1990-01-01 00:00:00 +08:00' AS DATETIME) - ELSE currency_rate_logs_add_dense_rank.created_at - END AS created_date_from, + ELSE currency_rate_logs_add_dense_rank.created_datetime + END AS from_datetime, - COALESCE(currency_rate_logs_add_dense_rank_clone.created_at, - CAST('2999-12-31 23:59:59 +08:00' AS DATETIME)) AS created_date_to, - CURRENT_TIMESTAMP() AS _dbt_ran_at + COALESCE(currency_rate_logs_add_dense_rank_clone.created_datetime, + CAST('2999-12-31 23:59:59 +08:00' AS DATETIME)) AS to_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM currency_rate_logs_add_dense_rank @@ -141,7 +134,7 @@ currency_rate_logs_resturucure AS ( ON currency_rate_logs_add_dense_rank.dense_rank_index = currency_rate_logs_add_dense_rank_clone.dense_rank_index-1 AND currency_rate_logs_add_dense_rank.currency_rate_id = currency_rate_logs_add_dense_rank_clone.currency_rate_id - ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_at + ORDER BY currency_rate_logs_add_dense_rank.currency_rate_id, currency_rate_logs_add_dense_rank.created_datetime ), @@ -149,21 +142,28 @@ currency_rate_logs_resturucure AS ( final__stg_exchange__currency_rate_logs AS ( SELECT - id, + -- ids + currency_rate_log_id, currency_rate_id, quote_currency_id, + creator_user_id, + + -- dimensions quote_currency_name, quote_currency_short_code, quote_currency_symbol, payment_method, - payment_method_name, service_type, - service_type_name, + + -- measures base_to_quote_currency_exchange_rate, - created_by, - created_date_from, - created_date_to, - _dbt_ran_at + + -- date/times + from_datetime, + to_datetime, + + -- metadata + _dbt_ran_datetime FROM currency_rate_logs_resturucure diff --git a/models/staging/exchange/stg_exchange__group_cost_documents.sql b/models/staging/exchange/stg_exchange__group_cost_documents.sql new file mode 100644 index 0000000..7134cce --- /dev/null +++ b/models/staging/exchange/stg_exchange__group_cost_documents.sql @@ -0,0 +1,49 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + IFF(documents.owner_type = 'App\\Models\\Group', documents.owner_id, null) AS group_cost_id, + documents.document_type, + documents.deleted_datetime, + documents.created_datetime, + documents.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents + + WHERE + owner_type = 'App\\Models\\Group' +), + + +-- FINAL +final__stg_exchange__group_cost_documents AS ( + + SELECT + -- ids + document_id, + group_cost_id, + + -- dimensions + document_type, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_join_status +) + +SELECT * FROM final__stg_exchange__group_cost_documents \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__group_costs.sql b/models/staging/exchange/stg_exchange__group_costs.sql new file mode 100644 index 0000000..5e00308 --- /dev/null +++ b/models/staging/exchange/stg_exchange__group_costs.sql @@ -0,0 +1,76 @@ +-- IMPORTS +WITH group_costs AS ( + SELECT * FROM {{ ref('base_exchange__group_costs') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +group_costs_join_currencies_status AS ( + + SELECT + group_costs.group_cost_id, + + group_costs.group_marking_id, + group_costs.supplier_user_id, + group_costs.base_amount, + group_costs.quote_amount, + + group_costs.base_currency_id, + + group_costs.quote_currency_id, + + group_costs.base_to_quote_currency_exchange_rate, + group_costs.base_tax, + group_costs.base_service_charge, + + COALESCE(status.name, group_costs.status::string) AS status, + + group_costs.created_datetime, + group_costs.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM group_costs + + LEFT JOIN status + ON (group_costs.status = status.id) +), + + +-- FINAL +final__stg_exchange__group_costs AS ( + + SELECT + -- ids + group_cost_id, + group_marking_id, + supplier_user_id, + base_currency_id, + quote_currency_id, + + -- dimensions + status, + + -- measures + base_amount, + quote_amount, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM group_costs_join_currencies_status + +) + +SELECT * FROM final__stg_exchange__group_costs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__segments.sql b/models/staging/exchange/stg_exchange__segments.sql new file mode 100644 index 0000000..dcec9c4 --- /dev/null +++ b/models/staging/exchange/stg_exchange__segments.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH segments AS ( + SELECT * FROM {{ ref('base_exchange__segments') }} +), + +segment_types AS ( + SELECT * FROM {{ ref('seed_exchange__segment_types') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +segments_join_segment_types AS ( + + SELECT + segments.segment_id, + segments.name, + COALESCE(segment_types.name, segments.segment_type::string) AS segment_type, + segments.deleted_datetime, + segments.created_datetime, + segments.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM segments + + LEFT JOIN segment_types + ON (segments.segment_type = segment_types.id) +), + + +-- FINAL +final__stg_exchange__segments AS ( + + SELECT + -- ids + segment_id, + + -- dimensions + name, + segment_type, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM segments_join_segment_types + +) + +SELECT * FROM final__stg_exchange__segments \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_cost_logs.sql b/models/staging/exchange/stg_exchange__transaction_cost_logs.sql new file mode 100644 index 0000000..332bf6b --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_cost_logs.sql @@ -0,0 +1,140 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_arch_logs AS ( + SELECT * FROM {{ ref('base_exchange__transaction_arch_logs') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +transactions_generate_transaction_log_id AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(transactions.transaction_id, transactions.created_datetime)) AS transaction_log_id, + * + + FROM transactions +), + + +transaction_arch_logs_union_transactions AS ( + SELECT * FROM transactions_generate_transaction_log_id + UNION + SELECT * FROM transaction_arch_logs +), + + +union_transaction_rename_column_and_column_value AS ( + + SELECT + transaction_arch_logs_union_transactions.transaction_log_id AS transaction_cost_log_id, + transaction_arch_logs_union_transactions.transaction_id AS transaction_cost_id, + + IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Transaction', + transaction_arch_logs_union_transactions.owner_id, + null) AS transaction_order_id, + + COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type, + + transaction_arch_logs_union_transactions.issuer_user_id AS supplier_company_id, + transaction_arch_logs_union_transactions.receiver_user_id AS company_id, + + transaction_arch_logs_union_transactions.recipient_bank_id, + + COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string) AS payment_method, + + transaction_arch_logs_union_transactions.payment_reference, + transaction_arch_logs_union_transactions.bill_number, + + transaction_arch_logs_union_transactions.base_value, + transaction_arch_logs_union_transactions.quote_value, + + transaction_arch_logs_union_transactions.base_currency_id, + + transaction_arch_logs_union_transactions.quote_currency_id, + + transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate, + transaction_arch_logs_union_transactions.base_tax, + transaction_arch_logs_union_transactions.base_service_charge, + transaction_arch_logs_union_transactions.expired_datetime, + + COALESCE(status.name, transaction_arch_logs_union_transactions.status::string) AS status, + + transaction_arch_logs_union_transactions.deleted_datetime, + transaction_arch_logs_union_transactions.created_datetime, + transaction_arch_logs_union_transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_arch_logs_union_transactions + + LEFT JOIN transaction_types + ON (transaction_arch_logs_union_transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transaction_arch_logs_union_transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transaction_arch_logs_union_transactions.status = status.id) + + + WHERE + transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Transaction' + AND + transaction_arch_logs_union_transactions.transaction_type = '3' --BILL +), + +-- FINAL +final__stg_exchange__transaction_log_costs AS ( + + SELECT + --ids + transaction_cost_log_id, + transaction_cost_id, + transaction_order_id, + supplier_company_id, + company_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + + -- dimentions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM union_transaction_rename_column_and_column_value + +) + +SELECT * FROM final__stg_exchange__transaction_log_costs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_costs.sql b/models/staging/exchange/stg_exchange__transaction_costs.sql new file mode 100644 index 0000000..cf26246 --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_costs.sql @@ -0,0 +1,136 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transactions.transaction_id AS transaction_cost_id, + IFF(transactions.owner_type = 'App\\Models\\Transaction', transactions.owner_id, null) AS transaction_order_id, + + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + + transactions.issuer_user_id AS supplier_company_id, + + transactions.recipient_bank_id AS bank_id, + + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + + transactions.payment_reference, + transactions.bill_number, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + transactions.quote_currency_id, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + + transactions.expired_datetime, + + COALESCE(status.name, transactions.status::string) AS status, + + transactions.deleted_datetime, + transactions.created_datetime, + transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + + WHERE + transactions.owner_type = 'App\\Models\\Transaction' + AND + transactions.transaction_type = '3' --BILL + +), + +remove_deleted_row AS ( + SELECT + * + + FROM transactions_join_transaction_types_payment_methods_currencies_status + + WHERE deleted_datetime IS NULL +), + +-- It will seldom offur system error, so we assume that the latest cost is correct. But it is not a correct assumption +remove_system_error_duplicate_row AS ( + SELECT + *, + ROW_NUMBER() OVER (PARTITION BY transaction_order_id ORDER BY updated_datetime DESC) AS row_number_index + + FROM remove_deleted_row + + QUALIFY + row_number_index = 1 +), + + +-- FINAL +final__stg_exchange__transaction_costs AS ( + + SELECT + -- ids + transaction_cost_id, + transaction_order_id, + supplier_company_id, + bank_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM remove_system_error_duplicate_row +) + +SELECT * FROM remove_system_error_duplicate_row \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_costs_brg_group_costs.sql b/models/staging/exchange/stg_exchange__transaction_costs_brg_group_costs.sql new file mode 100644 index 0000000..ef658a3 --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_costs_brg_group_costs.sql @@ -0,0 +1,43 @@ +-- IMPORTS +WITH group_transactions AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'group_transactions') }} +), + + +-- LOGIC +group_transactions_rename AS ( + + SELECT + group_transactions.id AS transaction_costs_brg_group_costs_id, + group_transactions.group_id AS group_cost_id, + group_transactions.transaction_id AS transaction_id, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM group_transactions + +), + + +-- FINAL +final__stg_exchange__transaction_costs_brg_group_costs AS ( + + SELECT + --ids + transaction_costs_brg_group_costs_id, + group_cost_id, + transaction_id, + + -- dimensions + + -- measures + + -- date/times + + -- metadata + _dbt_ran_datetime + + FROM group_transactions_rename + +) + +SELECT * FROM final__stg_exchange__transaction_costs_brg_group_costs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_documents.sql b/models/staging/exchange/stg_exchange__transaction_documents.sql new file mode 100644 index 0000000..136f4dc --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_documents.sql @@ -0,0 +1,65 @@ +-- IMPORTS +WITH documents AS ( + SELECT * FROM {{ ref('base_exchange__documents') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +documents_join_status AS ( + + SELECT + documents.document_id, + IFF(documents.owner_type = 'App\\Models\\Transaction', documents.owner_id, null) AS transaction_id, + documents.document_type, + IFF(document_type = 'CUSTOMER_PAYMENT_PROOF', + COALESCE(status.name, documents.status::string), + null) AS status, + documents.approver_user_id, + documents.approved_datetime, + documents.deleted_datetime, + documents.created_datetime, + documents.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM documents + + LEFT JOIN status + ON (documents.status = status.id) + + WHERE + owner_type = 'App\\Models\\Transaction' +), + + +-- FINAL +final__stg_exchange__transaction_documents AS ( + + SELECT + -- ids + document_id, + transaction_id, + approver_user_id, + + -- dimensions + status, + document_type, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM documents_join_status + +) + +SELECT * FROM final__stg_exchange__transaction_documents \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_order_logs.sql b/models/staging/exchange/stg_exchange__transaction_order_logs.sql new file mode 100644 index 0000000..618d182 --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_order_logs.sql @@ -0,0 +1,138 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_arch_logs AS ( + SELECT * FROM {{ ref('base_exchange__transaction_arch_logs') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +transactions_generate_transaction_log_id AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(transactions.transaction_id, transactions.created_datetime)) AS transaction_log_id, + * + + FROM transactions +), + + +transaction_arch_logs_union_transactions AS ( + SELECT * FROM transactions_generate_transaction_log_id + UNION + SELECT * FROM transaction_arch_logs +), + + +union_transaction_rename_column_and_column_value AS ( + + SELECT + transaction_arch_logs_union_transactions.transaction_log_id AS transaction_order_log_id, + transaction_arch_logs_union_transactions.transaction_id AS transaction_order_id, + + IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Booking', + transaction_arch_logs_union_transactions.owner_id, + null) AS booking_id, + + COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type, + + transaction_arch_logs_union_transactions.receiver_user_id AS company_id, + + transaction_arch_logs_union_transactions.recipient_bank_id, + + COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string) AS payment_method, + + transaction_arch_logs_union_transactions.payment_reference, + transaction_arch_logs_union_transactions.bill_number, + + transaction_arch_logs_union_transactions.base_value, + transaction_arch_logs_union_transactions.quote_value, + + transaction_arch_logs_union_transactions.base_currency_id, + + transaction_arch_logs_union_transactions.quote_currency_id, + + transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate, + transaction_arch_logs_union_transactions.base_tax, + transaction_arch_logs_union_transactions.base_service_charge, + transaction_arch_logs_union_transactions.expired_datetime, + + COALESCE(status.name, transaction_arch_logs_union_transactions.status::string) AS status, + + transaction_arch_logs_union_transactions.deleted_datetime, + transaction_arch_logs_union_transactions.created_datetime, + transaction_arch_logs_union_transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_arch_logs_union_transactions + + LEFT JOIN transaction_types + ON (transaction_arch_logs_union_transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transaction_arch_logs_union_transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transaction_arch_logs_union_transactions.status = status.id) + + + WHERE + transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Booking' + AND + transaction_arch_logs_union_transactions.transaction_type = '1' --PAYMENT +), + +-- FINAL +final__stg_exchange__transaction_order_logs AS ( + + SELECT + --ids + transaction_order_log_id, + transaction_order_id, + booking_id, + company_id, + recipient_bank_id, + base_currency_id, + quote_currency_id, + + -- dimentions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM union_transaction_rename_column_and_column_value + +) + +SELECT * FROM final__stg_exchange__transaction_order_logs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_order_products.sql b/models/staging/exchange/stg_exchange__transaction_order_products.sql new file mode 100644 index 0000000..9e6703b --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_order_products.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH transaction_details AS ( + SELECT * FROM {{ source('src_exchange_mysql', 'transaction_detail') }} +), + + +-- LOGIC +transaction_details_rename AS ( + + SELECT + transaction_details.id AS transaction_order_product_id, + transaction_details.transaction_id AS transaction_order_id, + transaction_details.product_code, + transaction_details.product_name, + transaction_details.quantity, + transaction_details.price, + transaction_details.amount, + transaction_details.deleted_at AS deleted_datetime, + transaction_details.created_at AS created_datetime, + transaction_details.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_details +), + +-- FINAL +final__base_exchange__transaction_order_products AS ( + + SELECT + -- ids + transaction_order_product_id, + transaction_order_id, + + -- dimensions + product_code, + product_name, + + -- measures + quantity, + price, + amount, + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transaction_details_rename + +) + +SELECT * FROM final__base_exchange__transaction_order_products \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_orders.sql b/models/staging/exchange/stg_exchange__transaction_orders.sql new file mode 100644 index 0000000..a05aa7a --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_orders.sql @@ -0,0 +1,116 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_status AS ( + + SELECT + transactions.transaction_id AS transaction_order_id, + IFF(transactions.owner_type = 'App\\Models\\Booking', transactions.owner_id, null) AS booking_id, + + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + + transactions.receiver_user_id AS company_id, + + transactions.recipient_bank_id AS bank_id, + + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + + transactions.payment_reference, + transactions.bill_number, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + transactions.quote_currency_id, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + + transactions.expired_datetime, + + COALESCE(status.name, transactions.status::string) AS status, + + transactions.deleted_datetime, + transactions.created_datetime, + transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + + WHERE + transactions.owner_type = 'App\\Models\\Booking' + AND + transactions.transaction_type = '1' --PAYMENT + +), + + +-- FINAL +final__stg_exchange__transaction_costs AS ( + + SELECT + -- ids + transaction_order_id, + booking_id, + company_id, + bank_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transactions_join_transaction_types_payment_methods_status + +) + +SELECT * FROM final__stg_exchange__transaction_costs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_wallet_logs.sql b/models/staging/exchange/stg_exchange__transaction_wallet_logs.sql new file mode 100644 index 0000000..a18b26b --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_wallet_logs.sql @@ -0,0 +1,133 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_arch_logs AS ( + SELECT * FROM {{ ref('base_exchange__transaction_arch_logs') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +transactions_generate_transaction_log_id AS ( + SELECT + MD5_NUMBER_LOWER64(CONCAT(transactions.transaction_id, transactions.created_datetime)) AS transaction_log_id, + * + + FROM transactions +), + + +transaction_arch_logs_union_transactions AS ( + SELECT * FROM transactions_generate_transaction_log_id + UNION + SELECT * FROM transaction_arch_logs +), + + +union_transaction_rename_column_and_column_value AS ( + + SELECT + transaction_arch_logs_union_transactions.transaction_log_id AS transaction_wallet_log_id, + transaction_arch_logs_union_transactions.transaction_id AS transaction_wallet_id, + + IFF(transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Wallet', + transaction_arch_logs_union_transactions.owner_id, + null) AS wallet_id, + + COALESCE(transaction_types.name, transaction_arch_logs_union_transactions.transaction_type::string) AS transaction_type, + + transaction_arch_logs_union_transactions.receiver_user_id AS company_id, + + COALESCE(payment_methods.name, transaction_arch_logs_union_transactions.payment_method::string) AS payment_method, + + transaction_arch_logs_union_transactions.payment_reference, + transaction_arch_logs_union_transactions.bill_number, + + transaction_arch_logs_union_transactions.base_value, + transaction_arch_logs_union_transactions.quote_value, + + transaction_arch_logs_union_transactions.base_currency_id, + + transaction_arch_logs_union_transactions.quote_currency_id, + + transaction_arch_logs_union_transactions.base_to_quote_currency_exchange_rate, + transaction_arch_logs_union_transactions.base_tax, + transaction_arch_logs_union_transactions.base_service_charge, + transaction_arch_logs_union_transactions.expired_datetime, + + COALESCE(status.name, transaction_arch_logs_union_transactions.status::string) AS status, + + transaction_arch_logs_union_transactions.deleted_datetime, + transaction_arch_logs_union_transactions.created_datetime, + transaction_arch_logs_union_transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transaction_arch_logs_union_transactions + + LEFT JOIN transaction_types + ON (transaction_arch_logs_union_transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transaction_arch_logs_union_transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transaction_arch_logs_union_transactions.status = status.id) + + + WHERE + transaction_arch_logs_union_transactions.owner_type = 'App\\Models\\Wallet' +), + +-- FINAL +final__stg_exchange__transaction_wallet_logs AS ( + + SELECT + --ids + transaction_wallet_log_id, + transaction_wallet_id, + wallet_id, + company_id, + base_currency_id, + quote_currency_id, + + -- dimentions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM union_transaction_rename_column_and_column_value + +) + +SELECT * FROM final__stg_exchange__transaction_wallet_logs \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__transaction_wallets.sql b/models/staging/exchange/stg_exchange__transaction_wallets.sql new file mode 100644 index 0000000..3e8cf3a --- /dev/null +++ b/models/staging/exchange/stg_exchange__transaction_wallets.sql @@ -0,0 +1,110 @@ +-- IMPORTS +WITH transactions AS ( + SELECT * FROM {{ ref('base_exchange__transactions') }} +), + +transaction_types AS ( + SELECT * FROM {{ ref('seed_exchange__transaction_types') }} +), + +payment_methods AS ( + SELECT * FROM {{ ref('seed_exchange__payment_methods') }} +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +transactions_join_transaction_types_payment_methods_currencies_status AS ( + + SELECT + transactions.transaction_id AS transaction_wallet_id, + IFF(transactions.owner_type = 'App\\Models\\Wallet', transactions.owner_id, null) AS wallet_id, + + COALESCE(transaction_types.name, transactions.transaction_type::string) AS transaction_type, + + transactions.receiver_user_id AS company_id, + + COALESCE(payment_methods.name, transactions.payment_method::string) AS payment_method, + + transactions.payment_reference, + transactions.bill_number, + + transactions.base_value, + transactions.quote_value, + + transactions.base_currency_id, + transactions.quote_currency_id, + + transactions.base_to_quote_currency_exchange_rate, + transactions.base_tax, + transactions.base_service_charge, + + transactions.expired_datetime, + + COALESCE(status.name, transactions.status::string) AS status, + + transactions.deleted_datetime, + transactions.created_datetime, + transactions.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM transactions + + LEFT JOIN transaction_types + ON (transactions.transaction_type = transaction_types.id) + + LEFT JOIN payment_methods + ON (transactions.payment_method = payment_methods.id) + + LEFT JOIN status + ON (transactions.status = status.id) + + + WHERE + transactions.owner_type = 'App\\Models\\Wallet' +), + + +-- FINAL +final__stg_exchange__transaction_wallets AS ( + + SELECT + -- ids + transaction_wallet_id, + wallet_id, + company_id, + base_currency_id, + quote_currency_id, + + -- dimensions + transaction_type, + payment_method, + status, + payment_reference, + bill_number, + + -- measures + base_value, + quote_value, + base_to_quote_currency_exchange_rate, + base_tax, + base_service_charge, + + -- date/times + expired_datetime, + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM transactions_join_transaction_types_payment_methods_currencies_status + +) + +SELECT * FROM final__stg_exchange__transaction_wallets \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__user_email_verifications.sql b/models/staging/exchange/stg_exchange__user_email_verifications.sql new file mode 100644 index 0000000..1de2cf6 --- /dev/null +++ b/models/staging/exchange/stg_exchange__user_email_verifications.sql @@ -0,0 +1,56 @@ +-- IMPORTS +WITH user_email_verifications AS ( + SELECT * FROM {{ ref('base_exchange__user_email_verifications') }} +), + +users AS ( + SELECT * FROM {{ ref('base_exchange__users') }} +), + +-- LOGIC +user_email_verifications_join_users AS ( + + SELECT + user_email_verifications.user_email_verification_id, + + users.user_id, + + user_email_verifications.email, + user_email_verifications.is_completed, + user_email_verifications.is_sent, + user_email_verifications.created_datetime, + user_email_verifications.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM user_email_verifications + + LEFT JOIN users + ON (user_email_verifications.email = users.email) +), + +-- FINAL +final__stg_exchange__user_email_verifications AS ( + + SELECT + -- ids + user_email_verification_id, + user_id, + + -- dimensions + email, + is_completed, + is_sent, + + -- measures + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM user_email_verifications_join_users +) + +SELECT * FROM final__stg_exchange__user_email_verifications \ No newline at end of file diff --git a/models/staging/exchange/stg_exchange__users.sql b/models/staging/exchange/stg_exchange__users.sql new file mode 100644 index 0000000..36885a9 --- /dev/null +++ b/models/staging/exchange/stg_exchange__users.sql @@ -0,0 +1,69 @@ +-- IMPORTS +WITH users AS ( + SELECT * FROM {{ ref('base_exchange__users') }} +), + +user_types AS ( + SELECT * FROM {{ ref('seed_exchange__user_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_exchange__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +users_join_user_types_status AS ( + + SELECT + users.user_id, + users.name, + users.email, + + COALESCE(user_types.name, users.user_type::string) AS user_type, + COALESCE(status.name, users.status::string) AS status, + + users.deleted_datetime, + users.created_datetime, + users.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM users + + LEFT JOIN user_types + ON (users.user_type = user_types.id) + + LEFT JOIN status + ON (users.status = status.id) +), + + +-- FINAL +final__stg_exchange__users AS ( + + SELECT + -- ids + user_id, + + -- dimensions + name, + email, + user_type, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM users_join_user_types_status +) + +SELECT * FROM final__stg_exchange__users \ No newline at end of file diff --git a/models/staging/shipping/_shipping__docs.md b/models/staging/shipping/_shipping__docs.md new file mode 100644 index 0000000..7535a29 --- /dev/null +++ b/models/staging/shipping/_shipping__docs.md @@ -0,0 +1,21 @@ +{% docs src_shipping_mysql %} + +Application link: https://izyim.cief-malaysia.com/ + +Data is extract from CIEF Selfhost MYSQL `portal_production`, using custom python code. + +- `_source_loaded_at` will add into the raw data when extract + +- Python will run the SQL in everyday 12:30AM + +- The raw data will use copy into snowflake in 2AM everyday + +- Depend on target.name to determin which snowflake raw schema + +```JINJA +{_%- if target.name == "prod" -%} PROD_CIEF_RAW_DB +{_%- else -%} DEV_CIEF_RAW_DB +{_%- endif -%} +``` + +{% enddocs %} \ No newline at end of file diff --git a/models/staging/shipping/_shipping__sources.yml b/models/staging/shipping/_shipping__sources.yml new file mode 100644 index 0000000..5fcb7a7 --- /dev/null +++ b/models/staging/shipping/_shipping__sources.yml @@ -0,0 +1,293 @@ +version: 2 + +sources: + - name: src_shipping_mysql + description: '{{ doc("src_shipping_mysql") }}' + database: | + {%- if target.name == "prod" -%} PROD_CIEF_RAW_DB + {%- else -%} DEV_CIEF_RAW_DB + {%- endif -%} + schema: SHIPPING_MYSQL + loader: custom_python_code + loaded_at_field: _source_loaded_datetime + freshness: + warn_after: {count: 26, period: hour} + error_after: {count: 48, period: hour} + meta: + owner: "@yam" + model_maturity: prod + tags: + - exchange + - daily + + + tables: + - name: activity_log + description: Contains most table's activity log that CUD (created, updated, deleted) + columns: + - name: id + description: Primary key for 'activity_log' + tests: + - unique + - not_null + + + - name: addresses + description: Raw `addresses` table from Snowflake + columns: + - name: id + description: Primary key for 'addresses' + tests: + - unique + - not_null + + + - name: announcement_segment + description: Raw `announcement_segment` table from Snowflake + columns: + - name: announcement_id + description: Primary key for 'announcement_segment' + tests: + - unique + - not_null + + + - name: announcements + description: Raw `announcements` table from Snowflake + columns: + - name: id + description: Primary key for 'announcements' + tests: + - unique + - not_null + + + - name: companies + description: Raw `companies` table from Snowflake + columns: + - name: id + description: Primary key for 'companies' + tests: + - unique + - not_null + + + - name: company_connections + description: Raw `company_connections` table from Snowflake + columns: + - name: id + description: Primary key for 'company_connections' + tests: + - unique + - not_null + + + - name: company_employees + description: Raw `company_employees` table from Snowflake + columns: + - name: company_module_id + description: Primary key for 'company_employees' + tests: + - not_null + + + - name: company_modules + description: Raw `company_modules` table from Snowflake + columns: + - name: id + description: Primary key for 'company_modules' + tests: + - unique + - not_null + + + - name: connection_segments + description: Raw `connection_segments` table from Snowflake + columns: + - name: id + description: Primary key for 'connection_segments' + tests: + - unique + - not_null + + + - name: contacts + description: Raw `contacts` table from Snowflake + columns: + - name: id + description: Primary key for 'contacts' + tests: + - unique + - not_null + + + - name: container_packing_lists + description: Raw `container_packing_lists` table from Snowflake + columns: + - name: id + description: Primary key for 'container_packing_lists' + tests: + - unique + - not_null + + + - name: containers + description: Raw `containers` table from Snowflake + columns: + - name: id + description: Primary key for 'containers' + tests: + - unique + - not_null + + + - name: documents + description: Raw `documents` table from Snowflake + columns: + - name: id + description: Primary key for 'documents' + tests: + - unique + - not_null + + + - name: failed_jobs + description: Raw `failed_jobs` table from Snowflake + columns: + - name: id + description: Primary key for 'failed_jobs' + tests: + - unique + - not_null + + + - name: files + description: Raw `files` table from Snowflake + columns: + - name: id + description: Primary key for 'files' + tests: + - unique + - not_null + + + - name: order_roles + description: Raw `order_roles` table from Snowflake + columns: + - name: id + description: Primary key for 'order_roles' + tests: + - unique + - not_null + + + - name: orders + description: Raw `orders` table from Snowflake + columns: + - name: id + description: Primary key for 'orders' + tests: + - unique + - not_null + + + - name: packages + description: Raw `packages` table from Snowflake + columns: + - name: id + description: Primary key for 'packages' + tests: + - unique + - not_null + + + - name: packing_lists + description: Raw `packing_lists` table from Snowflake + columns: + - name: id + description: Primary key for 'packing_lists' + tests: + - unique + - not_null + + + - name: password_resets + description: Raw `password_resets` table from Snowflake + columns: + - name: id + description: Primary key for 'password_resets' + tests: + - unique + - not_null + + + - name: remarks + description: Raw `remarks` table from Snowflake + columns: + - name: id + description: Primary key for 'remarks' + tests: + - unique + - not_null + + + - name: schedules + description: Raw `schedules` table from Snowflake + columns: + - name: id + description: Primary key for 'schedules' + tests: + - unique + - not_null + + + - name: segment_constants + description: Raw `segment_constants` table from Snowflake + columns: + - name: id + description: Primary key for 'segment_constants' + tests: + - unique + - not_null + + + - name: transaction_details + description: Raw `transaction_details` table from Snowflake + columns: + - name: id + description: Primary key for 'transaction_details' + tests: + - unique + - not_null + + + - name: transactions + description: Raw `transactions` table from Snowflake + columns: + - name: id + description: Primary key for 'transactions' + tests: + - unique + - not_null + + + - name: transport_arrangements + description: Raw `transport_arrangements` table from Snowflake + columns: + - name: id + description: Primary key for 'transport_arrangements' + tests: + - unique + - not_null + + + - name: users + description: Raw `users` table from Snowflake + columns: + - name: id + description: Primary key for 'users' + tests: + - unique + - not_null + + diff --git a/models/shipping/base/orders/base_shipping__orders.sql b/models/staging/shipping/base/base_shipping__bookings.sql similarity index 67% rename from models/shipping/base/orders/base_shipping__orders.sql rename to models/staging/shipping/base/base_shipping__bookings.sql index ca0f407..fff9113 100644 --- a/models/shipping/base/orders/base_shipping__orders.sql +++ b/models/staging/shipping/base/base_shipping__bookings.sql @@ -8,16 +8,16 @@ WITH orders AS ( orders_rename AS ( SELECT - orders.id, - orders.company_module_id, + orders.id AS booking_id, + orders.company_module_id AS sub_company_id, orders.reference, orders.reference_contract, - orders.type, + orders.type AS service_type, orders.status, orders.deleted_at AS deleted_datetime, orders.created_at AS created_datetime, orders.updated_at AS updated_datetime, - current_timestamp() AS _dbt_ran_at + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime FROM orders @@ -28,16 +28,25 @@ orders_rename AS ( final__base_shipping__orders AS ( SELECT - id, - company_module_id, + -- ids + booking_id, + sub_company_id, + + -- dimensions reference, reference_contract, - type, + service_type, status, + + -- measures + + -- date/times deleted_datetime, created_datetime, updated_datetime, - _dbt_ran_at + + -- metadata + _dbt_ran_datetime FROM orders_rename diff --git a/models/staging/shipping/base/base_shipping__parent_companies.sql b/models/staging/shipping/base/base_shipping__parent_companies.sql new file mode 100644 index 0000000..e8a01d6 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__parent_companies.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH companies AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'companies') }} +), + +-- LOGIC +companies_rename AS ( + + SELECT + companies.id AS parent_company_id, + companies.name, + companies.debtor AS autocount_id, + companies.type AS parent_company_type, + companies.status, + companies.created_at AS created_datetime, + companies.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM companies +), + + +-- FINAL +final__base_shipping__parent_companies AS ( + + SELECT + -- ids + parent_company_id, + autocount_id, + + -- dimensions + name, + parent_company_type, + status, + + -- measures + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM companies_rename +) + +SELECT * FROM final__base_shipping__parent_companies \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__sub_companies.sql b/models/staging/shipping/base/base_shipping__sub_companies.sql new file mode 100644 index 0000000..1b2e114 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__sub_companies.sql @@ -0,0 +1,48 @@ +-- IMPORTS +WITH company_modules AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'company_modules') }} +), + +-- LOGIC +company_modules_rename AS ( + + SELECT + company_modules.id AS sub_company_id, + company_modules.company_id AS parent_company_id, + company_modules.name, + company_modules.type AS sub_company_type, + company_modules.status, + company_modules.created_at AS created_datetime, + company_modules.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM company_modules +), + + +-- FINAL +final__base_shipping__sub_companies AS ( + + SELECT + -- ids + sub_company_id, + parent_company_id, + + -- dimensions + name, + sub_company_type, + status, + + -- measures + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM company_modules_rename +) + +SELECT * FROM final__base_shipping__sub_companies \ No newline at end of file diff --git a/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql b/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql new file mode 100644 index 0000000..79dbd65 --- /dev/null +++ b/models/staging/shipping/base/base_shipping__sub_company_marking_connections.sql @@ -0,0 +1,46 @@ +-- IMPORTS +WITH company_connections AS ( + SELECT * FROM {{ source('src_shipping_mysql', 'company_connections') }} +), + +-- LOGIC +company_connections_rename AS ( + + SELECT + company_connections.id AS sub_company_marking_connection_id, + company_connections.invitee_id AS sub_company_id, + company_connections.invitee_reference AS sub_company_marking_id, + company_connections.status, + company_connections.created_at AS created_datetime, + company_connections.updated_at AS updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM company_connections +), + + +-- FINAL +final__base_shipping__sub_company_marking_connections AS ( + + SELECT + -- ids + sub_company_marking_connection_id, + sub_company_id, + sub_company_marking_id, + + -- dimensions + status, + + -- measures + + -- date/times + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM company_connections_rename +) + +SELECT * FROM final__base_shipping__sub_company_marking_connections \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__bookings.sql b/models/staging/shipping/stg_shipping__bookings.sql new file mode 100644 index 0000000..abfbaf9 --- /dev/null +++ b/models/staging/shipping/stg_shipping__bookings.sql @@ -0,0 +1,65 @@ +-- IMPORT +WITH bookings AS ( + SELECT * FROM {{ ref('base_shipping__bookings') }} +), + +service_types AS ( + SELECT * FROM {{ ref('seed_shipping__service_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + +-- LOGIC +bookings_map_column_values AS ( + SELECT + bookings.booking_id, + bookings.sub_company_id, + bookings.reference, + bookings.reference_contract, + COALESCE(service_types.name, bookings.service_type::string) AS service_type, + COALESCE(status.name, bookings.status::string) AS status, + bookings.deleted_datetime, + bookings.created_datetime, + bookings.updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM bookings + + LEFT JOIN service_types + ON (bookings.service_type = service_types.id) + + LEFT JOIN status + ON (bookings.status = status.id) +), + +-- FINAL +final__stg_shipping__bookings AS ( + SELECT + -- ids + booking_id, + sub_company_id, + + -- dimensions + reference, + reference_contract, + service_type, + status, + + -- measures + + -- date/times + deleted_datetime, + created_datetime, + updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM bookings_map_column_values +) + +SELECT * FROM final__stg_shipping__bookings \ No newline at end of file diff --git a/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql b/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql new file mode 100644 index 0000000..3acf2df --- /dev/null +++ b/models/staging/shipping/stg_shipping__parent_and_sub_companies.sql @@ -0,0 +1,123 @@ +-- IMPORTS +WITH parent_companies AS ( + SELECT * FROM {{ ref('base_shipping__parent_companies') }} +), + +sub_companies AS ( + SELECT * FROM {{ ref('base_shipping__sub_companies') }} +), + +sub_company_marking_connections AS ( + SELECT * FROM {{ ref('base_shipping__sub_company_marking_connections') }} +), + +parent_company_types AS ( + SELECT * FROM {{ ref('seed_shipping__parent_company_types') }} + WHERE category = 'RENAME' +), + +sub_company_types AS ( + SELECT * FROM {{ ref('seed_shipping__sub_company_types') }} + WHERE category = 'DEFAULT' +), + +status AS ( + SELECT * FROM {{ ref('seed_shipping__status') }} + WHERE category = 'DEFAULT' +), + + +-- LOGIC +parent_companies_join_sub_companies AS ( + SELECT + parent_companies.parent_company_id AS parent_company_id, + parent_companies.autocount_id, + parent_companies.name AS parent_company_name, + parent_companies.parent_company_type AS company_type, + parent_companies.status, + parent_companies.created_datetime AS parent_company_created_datetime, + parent_companies.updated_datetime AS parent_company_updated_datetime, + + sub_companies.sub_company_id AS sub_company_id, + sub_companies.name AS sub_company_name, + sub_companies.sub_company_type AS business_type, + sub_companies.created_datetime AS sub_company_created_datetime, + sub_companies.updated_datetime AS sub_company_updated_datetime + + FROM parent_companies + + LEFT JOIN sub_companies + ON (parent_companies.parent_company_id = sub_companies.sub_company_id) +), + +parent_companies_and_sub_companies_map_constants AS ( + SELECT + parent_companies_join_sub_companies.parent_company_id, + parent_companies_join_sub_companies.autocount_id, + parent_companies_join_sub_companies.parent_company_name, + + COALESCE(company_types.name, parent_companies_join_sub_companies.company_type::string) AS company_type, + + COALESCE(status.name, parent_companies_join_sub_companies.status::string) AS status, + + parent_companies_join_sub_companies.parent_company_created_datetime, + parent_companies_join_sub_companies.parent_company_updated_datetime, + parent_companies_join_sub_companies.sub_company_id, + parent_companies_join_sub_companies.sub_company_name, + + COALESCE(business_type.name, parent_companies_join_sub_companies.business_type::string) AS business_type, + + parent_companies_join_sub_companies.sub_company_created_datetime, + parent_companies_join_sub_companies.sub_company_updated_datetime, + + sub_company_marking_connections.sub_company_marking_id, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM parent_companies_join_sub_companies + + LEFT JOIN sub_company_marking_connections + ON (parent_companies_join_sub_companies.sub_company_id = sub_company_marking_connections.sub_company_id) + + LEFT JOIN parent_company_types AS company_types + ON (parent_companies_join_sub_companies.company_type = company_types.id) + + LEFT JOIN sub_company_types AS business_type + ON (parent_companies_join_sub_companies.business_type = business_type.id) + + LEFT JOIN status + ON (parent_companies_join_sub_companies.status = status.id) +), + +-- FINAL +final__stg_shipping__parent_and_sub_companies AS ( + + SELECT + -- ids + parent_company_id, + autocount_id, + sub_company_id, + sub_company_marking_id, + + -- dimensions + status, + parent_company_name, + sub_company_name, + company_type, + business_type, + + -- measures + + -- date/times + parent_company_created_datetime, + parent_company_updated_datetime, + sub_company_created_datetime, + sub_company_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM parent_companies_and_sub_companies_map_constants +) + +SELECT * FROM final__stg_shipping__parent_and_sub_companies diff --git a/seeds/exchange/seed_exchange__segment_types.csv b/seeds/exchange/seed_exchange__segment_types.csv new file mode 100644 index 0000000..6b0b3ce --- /dev/null +++ b/seeds/exchange/seed_exchange__segment_types.csv @@ -0,0 +1,4 @@ +ID,CATEGORY,NAME +1,DEFAULT,STANDARD_SEGMENT +2,DEFAULT,CUSTOM_SEGMENT +3,DEFAULT,LABEL_SEGMENT \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__packing_list_types.csv b/seeds/shipping/seed_shipping__packing_list_types.csv new file mode 100644 index 0000000..2832068 --- /dev/null +++ b/seeds/shipping/seed_shipping__packing_list_types.csv @@ -0,0 +1,8 @@ +ID,CATEGORY,NAME +0,DEFAULT,SUPPLIER_PACKING_LIST +1,DEFAULT,WAREHOUSE_RECEIVE_LIST +2,DEFAULT,SHIPPING_PACKING_LIST +3,DEFAULT,TRANSPORT_PACKING_LIST +4,DEFAULT,DELIVERY_PACKING_LIST +5,DEFAULT,WAREHOUSE_RECEIVE_LIST_REPLICA +6,DEFAULT,SHIPPING_PACKING_LIST_REPLICA \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__parent_company_types.csv b/seeds/shipping/seed_shipping__parent_company_types.csv new file mode 100644 index 0000000..c202ec6 --- /dev/null +++ b/seeds/shipping/seed_shipping__parent_company_types.csv @@ -0,0 +1,5 @@ +ID,CATEGORY,NAME +0,DEFAULT,PERSONAL_BUSINESS +1,DEFAULT,COMPANY_BUSINESS +0,RENAME,PERSONAL +1,RENAME,CORPORATE \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__service_types.csv b/seeds/shipping/seed_shipping__service_types.csv new file mode 100644 index 0000000..2f2569c --- /dev/null +++ b/seeds/shipping/seed_shipping__service_types.csv @@ -0,0 +1,4 @@ +ID,CATEGORY,NAME +0,DEFAULT,SHARED_CONTAINER +1,DEFAULT,DEDICATED_CONTAINER +2,DEFAULT,LAST_MILE_DELIVERY \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__status.csv b/seeds/shipping/seed_shipping__status.csv new file mode 100644 index 0000000..5fbd987 --- /dev/null +++ b/seeds/shipping/seed_shipping__status.csv @@ -0,0 +1,9 @@ +ID,CATEGORY,NAME +0,DEFAULT,PENDING_SUBMISSION +1,DEFAULT,PENDING_VERIFICATION +2,DEFAULT,APPROVED +3,DEFAULT,COMPLETED +4,DEFAULT,REJECTED +5,DEFAULT,SUSPENDED +6,DEFAULT,EXPIRED +7,DEFAULT,REFUNDED \ No newline at end of file diff --git a/seeds/shipping/seed_shipping__sub_company_types.csv b/seeds/shipping/seed_shipping__sub_company_types.csv new file mode 100644 index 0000000..99c7662 --- /dev/null +++ b/seeds/shipping/seed_shipping__sub_company_types.csv @@ -0,0 +1,10 @@ +ID,CATEGORY,NAME +0,DEFAULT,SUPPLIER +1,DEFAULT,IMPORTER +2,DEFAULT,FREIGHT_FORWARDER +3,DEFAULT,WAREHOUSE +4,DEFAULT,CUSTOM_CLEARANCE_AGENT +5,DEFAULT,DRIVER +6,DEFAULT,CURRENCY_VENDOR +7,DEFAULT,ONLINE_SELLER +8,DEFAULT,LAST_MILE_DELIVERY_AGENT \ No newline at end of file diff --git a/snapshots/exchange/snap_exchange__segment_companies.sql b/snapshots/exchange/snap_exchange__segment_companies.sql index 8207332..e9038a6 100644 --- a/snapshots/exchange/snap_exchange__segment_companies.sql +++ b/snapshots/exchange/snap_exchange__segment_companies.sql @@ -9,8 +9,9 @@ }} select - CONCAT(segment_id,'-',company_id) as concat_id, - * + *, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + from {{ source('src_exchange_mysql', 'segment_companies') }} {% endsnapshot %} \ No newline at end of file diff --git a/tests/generic/test_is_positive.sql b/tests/generic/test_is_positive.sql new file mode 100644 index 0000000..4bb58ce --- /dev/null +++ b/tests/generic/test_is_positive.sql @@ -0,0 +1,26 @@ +{% test is_positive(model, column_name) %} + +with validation as ( + + select + {{ column_name }} as value_field + + from {{ model }} + +), + +validation_errors as ( + + select + value_field + + from validation + + where value_field < 0 + +) + +select * +from validation_errors + +{% endtest %} \ No newline at end of file