cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-19 04:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of gitlab.com:CIEFWorldwideSdnBhd/dbt_cloud into create_shipping_modeling

Browse Source
This commit is contained in:
CIEF DataTeam
2023-07-06 08:42:18 +00:00
parent ee8665519c 6571dd6afe
commit 327168bc9d
14 changed files with 999 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/intermediate/int_exchange__user_email_verifications_group_by_user_ids.sql
+47
View File
@@ -0,0 +1,47 @@
-- IMPORT
WITH user_email_verifications AS (
SELECT * FROM {{ ref('stg_exchange__user_email_verifications') }}
),
-- LOGIC
user_email_verifications_group_by_user_ids AS (
SELECT
user_id,
ZEROIFNULL(COUNT(user_email_verification_id)) AS number_of_verification_email_sent,
LISTAGG(DISTINCT email, ',') WITHIN GROUP(ORDER BY email) AS email,
ZEROIFNULL(MAX(is_completed)) AS is_completed,
MIN(created_datetime) AS first_email_verification_created_datetime,
MAX(created_datetime) AS last_email_verification_created_datetime,
MIN(IFF(is_completed = 1, updated_datetime, null)) AS email_verification_completed_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM user_email_verifications
GROUP BY user_id
),
-- FINAL
final__int_exchange__user_email_verifications_group_by_user_ids AS (
SELECT
-- ids
user_id,
-- dimensions
email,
is_completed,
-- measures
number_of_verification_email_sent,
-- date/times
first_email_verification_created_datetime,
last_email_verification_created_datetime,
email_verification_completed_datetime,
-- metadata
_dbt_ran_datetime
FROM user_email_verifications_group_by_user_ids
)
SELECT * FROM final__int_exchange__user_email_verifications_group_by_user_ids
models/marts/reporting/rep_exchange__company_conversion_funnels.sql
+9 -6
View File
@@ -192,7 +192,7 @@ identity_document_verified_activities AS (
FROM company_identity_documents
WHERE
identity_status = '2'
identity_status = 'APPROVED'
GROUP BY
company_id
@@ -316,7 +316,8 @@ union_all_activities_flag_migrated_company AS (
),
get_number_of_conversions AS (
-- This cumulative count is use for showing date to date conversion
generate_cumulative_count AS (
SELECT
company_id,
is_data_cleansing_generate_row,
@@ -357,11 +358,11 @@ final__rep_exchange__company_conversion_funnels AS (
company_id,
-- dimensions
is_data_cleansing_generate_row,
is_migrated_company,
activity,
activtity_rank,
is_company_latest_activity,
is_data_cleansing_generate_row,
is_migrated_company,
-- measures
cumulative_register_activity,
@@ -376,6 +377,7 @@ final__rep_exchange__company_conversion_funnels AS (
include_migrated_cumulative_register_activity,
include_migrated_cumulative_email_verified_activity,
include_migrated_cumulative_identity_document_uploaded_activity,
include_migrated_cumulative_identity_document_verified_activity,
include_migrated_cumulative_first_time_booking_activity,
include_migrated_cumulative_first_time_order_activity,
include_migrated_cumulative_first_time_order_completed_activity,
@@ -387,8 +389,9 @@ final__rep_exchange__company_conversion_funnels AS (
-- metadata
_dbt_ran_datetime
FROM get_number_of_conversions
FROM generate_cumulative_count
)
SELECT * FROM final__rep_exchange__company_conversion_funnels
models/marts/reporting/rep_exchange__daily_orders.sql
+5 -1
View File
@@ -51,6 +51,8 @@ fct_and_dim_joins AS (
companies.district_name AS company_district_name,
companies.postcode AS company_postcode,
companies.has_wallet AS company_has_wallet,
companies.latitude AS company_latitude,
companies.longitude AS company_longitude,
bookings.user_id,
bookings.bank_id,
@@ -133,7 +135,7 @@ fct_and_dim_joins AS (
FROM bookings
LEFT JOIN transaction_orders_get_next_order_created_datetime
RIGHT JOIN transaction_orders_get_next_order_created_datetime
ON (bookings.booking_id = transaction_orders_get_next_order_created_datetime.booking_id)
LEFT JOIN transaction_costs
@@ -191,6 +193,8 @@ final__rep_exchange__daily_orders AS (
company_state_name,
company_district_name,
company_postcode,
company_latitude,
company_longitude,
company_has_wallet,
cost_transaction_type,
cost_payment_method,
models/marts/warehouse/_warehouse_models.yml
+1 -1
View File
@@ -28,7 +28,7 @@ models:
- unique
- not_null
- name: fct_exchange__costs
- name: fct_exchange__transaction_costs
description: exchange transaction_cost
columns:
- name: transaction_cost_id
models/marts/warehouse/dim_exchange__companies.sql
+36
View File
@@ -97,6 +97,24 @@ companies_rename_and_join_table AS (
company_billing_addresses.postcode,
company_billing_addresses.address_line_one,
company_billing_addresses.address_line_two,
company_billing_addresses.google_place_id,
company_billing_addresses.phone_number AS google_api_extracted_phone_number,
company_billing_addresses.google_returned_address,
company_billing_addresses.location_type,
company_billing_addresses.operational_status,
company_billing_addresses.latitude,
company_billing_addresses.longitude,
company_billing_addresses.location_website,
company_billing_addresses.location_overall_rating,
company_billing_addresses.location_number_of_reviews,
company_billing_addresses.monday_operating_hours,
company_billing_addresses.tuesday_operating_hours,
company_billing_addresses.wednesday_operating_hours,
company_billing_addresses.thursday_operating_hours,
company_billing_addresses.friday_operating_hours,
company_billing_addresses.saturday_operating_hours,
company_billing_addresses.sunday_operating_hours,
company_billing_addresses.json_place_details_full_return,
company_wallets.wallet_id,
IFF(company_wallets.wallet_id IS NOT NULL, 1, 0) AS has_wallet,
@@ -152,6 +170,7 @@ final__dim_exchange__companies AS (
wallet_id,
wallet_currency_id,
identity_document_id,
google_place_id,
-- dimensions
name,
@@ -176,6 +195,23 @@ final__dim_exchange__companies AS (
identity_document_type,
identity_reference,
identity_status,
google_returned_address,
google_api_extracted_phone_number,
location_type,
operational_status,
latitude,
longitude,
location_website,
location_overall_rating,
location_number_of_reviews,
monday_operating_hours,
tuesday_operating_hours,
wednesday_operating_hours,
thursday_operating_hours,
friday_operating_hours,
saturday_operating_hours,
sunday_operating_hours,
json_place_details_full_return,
-- measures
wallet_balance,
models/marts/warehouse/dim_exchange__users.sql
+76
View File
@@ -0,0 +1,76 @@
-- IMPORT
WITH users AS (
SELECT * FROM {{ ref('stg_exchange__users') }}
),
user_email_verifications_group_by_user_ids AS (
SELECT * FROM {{ ref('int_exchange__user_email_verifications_group_by_user_ids') }}
),
companies_brg_users AS (
SELECT * FROM {{ ref('stg_exchange__companies_brg_users') }}
),
-- LOGIC
users_data_enrich AS (
SELECT
users.user_id,
users.name,
users.email,
users.user_type,
users.status AS user_status,
users.deleted_datetime AS user_deleted_datetime,
users.created_datetime AS user_created_datetime,
users.updated_datetime AS user_updated_datetime,
user_email_verifications_group_by_user_ids.is_completed AS is_completed_email_verification,
user_email_verifications_group_by_user_ids.number_of_verification_email_sent AS number_of_verification_email_sent,
user_email_verifications_group_by_user_ids.first_email_verification_created_datetime,
user_email_verifications_group_by_user_ids.last_email_verification_created_datetime,
user_email_verifications_group_by_user_ids.email_verification_completed_datetime,
companies_brg_users.company_id,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM users
LEFT JOIN user_email_verifications_group_by_user_ids
ON (users.user_id = user_email_verifications_group_by_user_ids.user_id)
LEFT JOIN companies_brg_users
ON (users.user_id = companies_brg_users.user_id)
),
-- FINAL
final__dim_exchange__users AS (
SELECT
-- ids
user_id,
company_id,
-- dimensions
name,
email,
user_type,
user_status,
is_completed_email_verification,
-- measures
number_of_verification_email_sent,
-- date/times
user_deleted_datetime,
user_created_datetime,
user_updated_datetime,
first_email_verification_created_datetime,
last_email_verification_created_datetime,
email_verification_completed_datetime,
-- metadata
_dbt_ran_datetime
FROM users_data_enrich
)
SELECT * FROM final__dim_exchange__users
models/staging/exchange/_exchange__models.yml
+3 -1
View File
@@ -23,4 +23,6 @@ models:
description: Bridge between `companies` and `users`
- name: stg_exchange__company_addresses
description: Company addresses that input when register
description: Company addresses that input when register
config:
tags: ['incremental']
models/staging/exchange/stg_exchange__company_addresses.sql
+408 -20
View File
@@ -1,3 +1,30 @@
-- A flag indicator to determine whether to run a full_load or an incremental_load
-- If user is a developer, a full load will be executed.
-- If the target schema is set to 'dbt_prod', only an incremental load will be performed.
{% if target.name == 'dbt_prod' %}{% set one_month_incremental_flag = true %}{% else %}{% set one_month_incremental_flag = false %}{% endif %}
-- MODEL CONFIGURATION BLOCK
{{
config(
materialized='incremental',
unique_key='company_id'
)
}}
-- LOCAL VARIABLES
{%- set days_of_week = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] %}
{%- set masked_address_sql = "(address_line_one IN " + var('mask_data_pattern') + " OR address_line_two IN " + var('mask_data_pattern') + " OR address_line_one IS NULL AND address_line_two IS NULL)" %}
{%- set dummy_value = -999 %}
{%- set dummy_long_string = "xxxxxxxxxxxxxxxxxx" %}
{%- set dummy_short_string = "xxxxxx" %}
{%- set dummy_phone = "xxx-xxxxxxxxxx" %}
{%- set dummy_operating_hours = '99:99-99:99' %}
{%- set dummy_website = 'https:www://xxxxxxxxxx.com' %}
-- IMPORTS
WITH company_addresses AS (
SELECT * FROM {{ ref('base_exchange__company_addresses') }}
@@ -15,43 +42,385 @@ districts AS (
SELECT * FROM {{ ref('base_exchange__districts') }}
),
-- LOGIC
company_addresses_with_row_num AS (
SELECT
*,
ROW_NUMBER() OVER (ORDER BY company_address_id) AS row_num
FROM company_addresses
{% if is_incremental() %}
WHERE updated_datetime > ( SELECT max(updated_datetime) FROM {{ this }} )
{%- endif %}
),
company_addressess_join_companies_countries_states_districts AS (
SELECT
company_addresses.company_address_id,
company_addresses.company_id,
company_addresses_with_row_num.row_num,
company_addresses.country_id,
company_addresses_with_row_num.company_address_id,
company_addresses_with_row_num.company_id,
company_addresses_with_row_num.country_id,
countries.name AS country_name,
countries.short_code AS country_short_code,
countries.phone_code AS country_phone_code,
company_addresses.state_id,
company_addresses_with_row_num.state_id,
states.name AS state_name,
company_addresses.district_id,
company_addresses_with_row_num.district_id,
districts.name AS district_name,
company_addresses.postcode,
company_addresses.address_line_one,
company_addresses.address_line_two,
company_addresses.is_billing_address,
company_addresses.deleted_datetime,
company_addresses.created_datetime,
company_addresses.updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM company_addresses
company_addresses_with_row_num.postcode,
company_addresses_with_row_num.address_line_one,
company_addresses_with_row_num.address_line_two,
company_addresses_with_row_num.is_billing_address,
company_addresses_with_row_num.deleted_datetime,
company_addresses_with_row_num.created_datetime,
company_addresses_with_row_num.updated_datetime
FROM company_addresses_with_row_num
LEFT JOIN countries
ON (company_addresses.country_id = countries.country_id)
ON (company_addresses_with_row_num.country_id = countries.country_id)
LEFT JOIN states
ON (company_addresses.state_id = states.state_id)
ON (company_addresses_with_row_num.state_id = states.state_id)
LEFT JOIN districts
ON (company_addresses.district_id = districts.district_id)
ON (company_addresses_with_row_num.district_id = districts.district_id)
),
-- Setting total rows to be iterated
{% if is_incremental() and one_month_incremental_flag %}
-- Get previous 1 month data from current time
{% set estimate_incremental_rows = dbt_utils.get_column_values(
table=ref('base_exchange__company_addresses'),
where='updated_datetime > ADD_MONTHS(CURRENT_DATE(), -1)',
column='company_address_id') %}
{%- set total_rows = estimate_incremental_rows|length %}
{% else %}
{%- set company_address_ids = dbt_utils.get_column_values(table=ref('base_exchange__company_addresses'), column='company_address_id') %}
{%- set total_rows = company_address_ids|length %}
{% endif %}
-- Formulating row batches for batch processing
-- Each batch is recommended to limit to 10 rows to avoid timeout
{%- set batch_size = 10 %}
{%- set num_batches = (total_rows // batch_size) + 1 %}
{%- set current_batch = 1 %}
{%- for batch in range(1, num_batches + 1) %}
{%- set start_row = (batch - 1) * batch_size + 1 %}
{%- set end_row = start_row + batch_size - 1 %}
batched_data_{{batch}} AS (
SELECT * FROM company_addressess_join_companies_countries_states_districts
WHERE row_num >= {{ start_row }} AND row_num <= {{ end_row }}
),
{%- endfor %}
{%- for batch in range(1, num_batches + 1) %}
address_enrich_using_google_api_{{batch}} AS (
SELECT
*,
{{ address_to_coordinates( 'address_line_one', 'address_line_two', 'postcode', 'district_name', 'state_name' ) }} AS json_function_return
FROM batched_data_{{batch}}
),
{%- endfor %}
union_all_batch_address_enriched AS (
{%- for batch in range(1, num_batches + 1) %}
SELECT * FROM address_enrich_using_google_api_{{batch}}
{%- if not loop.last %} UNION{% endif -%}
{% endfor %}
),
operation_hours_json AS (
SELECT
*,
-- Extract operating hour range for each day of the week and perform unicode replacement
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][0]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_monday_operating_hour,
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][1]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_tuesday_operating_hour,
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][2]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_wednesday_operating_hour,
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][3]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_thursday_operating_hour,
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][4]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_friday_operating_hour,
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][5]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_saturday_operating_hour,
REGEXP_REPLACE(REGEXP_REPLACE(json_function_return['output_result']['result']['opening_hours']['weekday_text'][6]::string, '[\u2013\u002d]', '-'), '[\u202F\u2009]', ' ') AS str_sunday_operating_hour
FROM union_all_batch_address_enriched
),
operation_hour_range_extraction AS (
SELECT
*,
{% for day_of_week in days_of_week %}
-- Extract the time range for each day of the week with day prefix removed
REGEXP_REPLACE(str_{{day_of_week}}_operating_hour, '^[a-zA-Z]+: ', '') AS {{day_of_week}}_time_range
{% if not loop.last %}, {% endif %}
{% endfor %}
FROM operation_hours_json
),
operation_hour_to_24_hour_transform AS (
SELECT
*,
{%- for day_of_week in days_of_week %}
CASE
-- Convert time range to 24-hour format for standard operating hour format. "9:00 AM - 10:00 PM"
WHEN REGEXP_LIKE({{day_of_week}}_time_range, '^[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM][ ]?-[ ]?[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM]$') THEN
CONCAT
(
TO_CHAR(TO_TIME(SPLIT_PART({{day_of_week}}_time_range, ' - ', 1)), 'HH24:MI'),
' - ',
TO_CHAR(TO_TIME(SPLIT_PART({{day_of_week}}_time_range, ' - ', 2)), 'HH24:MI')
)
ELSE {{day_of_week}}_time_range
END AS transformed_{{day_of_week}}_operating_hours
{% if not loop.last%}, {% endif %}
{% endfor %}
FROM operation_hour_range_extraction
),
special_single_operation_hour_to_24_hour_transform AS (
SELECT
*,
{%- for day_of_week in days_of_week %}
-- Convert time range to 24-hour format for cases where the AM/PM indicator is missing in the opening hour
CASE WHEN REGEXP_LIKE(transformed_{{day_of_week}}_operating_hours, '^[0-9]{1,2}:[0-9]{2}[ ]?-[ ]?[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM]$') THEN
CONCAT(TO_CHAR(TO_TIME(
CASE
WHEN TO_TIME(SPLIT_PART(transformed_{{day_of_week}}_operating_hours, ' - ', 2)) > '12:00 PM' THEN
SPLIT_PART(transformed_{{day_of_week}}_operating_hours, ' - ', 1) || ' PM'
ELSE
SPLIT_PART(transformed_{{day_of_week}}_operating_hours, ' - ', 1)
END
),'HH24:MI'),
' - ',
TO_CHAR(TO_TIME(SPLIT_PART(transformed_{{day_of_week}}_operating_hours, ' - ', 2)), 'HH24:MI')
)
ELSE transformed_{{day_of_week}}_operating_hours
END AS transformed_{{day_of_week}}_time_range
{% if not loop.last%}, {% endif %}
{% endfor %}
FROM operation_hour_to_24_hour_transform
),
-- This block handles specifically scenarios that has 2 operational hours in a single day "9:00 AM - 3:00 PM, 5:00 PM - 10:00 PM"
special_double_operation_hour_to_24_hour_transform AS (
SELECT
*,
{%- for day_of_week in days_of_week %}
CASE
-- First operating hour
-- Convert time range to 24-hour format for cases where the AM/PM indicator is missing in the operating hour
WHEN REGEXP_LIKE(operation_hour_split.{{day_of_week}}_first_operation_hour, '^[0-9]{1,2}:[0-9]{2}[ ]?-[ ]?[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM]$') THEN
CONCAT(TO_CHAR(TO_TIME(
CASE
WHEN TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 2)) > '12:00 PM' THEN
SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 1) || ' PM'
ELSE
SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 1)
END
),'HH24:MI'),
' - ',
TO_CHAR(TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 2)), 'HH24:MI')
)
-- Convert time range to 24-hour format for standard operating hour format
WHEN REGEXP_LIKE(operation_hour_split.{{day_of_week}}_first_operation_hour, '^[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM][ ]?-[ ]?[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM]$') THEN
CONCAT
(
TO_CHAR(TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 1)), 'HH24:MI'),
' - ',
TO_CHAR(TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 2)), 'HH24:MI')
)
ELSE operation_hour_split.{{day_of_week}}_first_operation_hour
END AS {{day_of_week}}_transformed_first_time_range,
-- Second operating hour
-- Convert time range to 24-hour format for cases where the AM/PM indicator is missing in the operating hour
CASE WHEN REGEXP_LIKE(operation_hour_split.{{day_of_week}}_second_operation_hour, '^[0-9]{1,2}:[0-9]{2}[ ]?-[ ]?[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM]$') THEN
CONCAT(TO_CHAR(TO_TIME(
CASE
WHEN TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_second_operation_hour, ' - ', 2)) > '12:00 PM' THEN
SPLIT_PART(operation_hour_split.{{day_of_week}}_second_operation_hour, ' - ', 1) || ' PM'
ELSE
SPLIT_PART(operation_hour_split.{{day_of_week}}_second_operation_hour, ' - ', 1)
END
),'HH24:MI'),
' - ',
TO_CHAR(TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_second_operation_hour, ' - ', 2)), 'HH24:MI')
)
--- Convert time range to 24-hour format for standard operating hour format
WHEN REGEXP_LIKE(operation_hour_split.{{day_of_week}}_first_operation_hour, '^[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM][ ]?-[ ]?[0-9]{1,2}:[0-9]{2}[ ]?[aApP][mM]$') THEN
CONCAT
(
TO_CHAR(TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 1)), 'HH24:MI'),
' - ',
TO_CHAR(TO_TIME(SPLIT_PART(operation_hour_split.{{day_of_week}}_first_operation_hour, ' - ', 2)), 'HH24:MI')
)
ELSE operation_hour_split.{{day_of_week}}_second_operation_hour
END AS {{day_of_week}}_transformed_second_time_range
{% if not loop.last%}, {% endif %}
{% endfor %}
FROM
-- Subquery to extract first and second operational hour of the day
( SELECT
*,
{%- for day_of_week in days_of_week %}
SPLIT_PART(transformed_{{day_of_week}}_time_range, ', ', 1) AS {{day_of_week}}_first_operation_hour,
SPLIT_PART(transformed_{{day_of_week}}_time_range, ', ', 2) AS {{day_of_week}}_second_operation_hour
{% if not loop.last%}, {% endif %}
{% endfor %}
FROM special_single_operation_hour_to_24_hour_transform
) AS operation_hour_split
),
concatenate_operation_hour AS (
SELECT
*,
{%- for day_of_week in days_of_week %}
CASE
WHEN {{day_of_week}}_transformed_second_time_range <> '' THEN
CONCAT({{day_of_week}}_transformed_first_time_range, ', ', {{day_of_week}}_transformed_second_time_range)
ELSE {{day_of_week}}_transformed_first_time_range
END AS {{day_of_week}}_final_operating_hours
{% if not loop.last%}, {% endif %}
{% endfor %}
FROM special_double_operation_hour_to_24_hour_transform
),
google_place_api_details_extraction AS (
SELECT
*,
json_function_return['output_place_id']['place_id_value']::string AS google_place_id,
json_function_return AS json_place_details_full_return,
json_function_return['function_run_datetime']::datetime AS _function_ran_datetime,
json_function_return['output_result']['result']['formatted_address']::string AS google_returned_address,
json_function_return['output_result']['result']['formatted_phone_number']::string AS phone_number,
json_function_return['output_result']['result']['types']::string AS location_type,
json_function_return['output_result']['result']['business_status']::string AS operational_status,
json_function_return['output_result']['result']['geometry']['location']['lat']::float AS latitude,
json_function_return['output_result']['result']['geometry']['location']['lng']::float AS longitude,
json_function_return['output_result']['result']['website']::string AS location_website,
json_function_return['output_result']['result']['rating']::float AS location_overall_rating,
json_function_return['output_result']['result']['user_ratings_total']::integer AS location_number_of_reviews
FROM concatenate_operation_hour
),
masked_data_transformation AS (
SELECT
company_address_id,
company_id,
country_id,
state_id,
district_id,
CASE
WHEN {{ masked_address_sql }} THEN '{{ dummy_long_string }}'
ELSE google_place_id
END AS google_place_id,
-- dimensions
country_name,
country_short_code,
country_phone_code,
state_name,
district_name,
postcode,
address_line_one,
address_line_two,
is_billing_address,
CASE
WHEN {{ masked_address_sql }} THEN '{{ dummy_short_string }}'
ELSE google_returned_address
END AS google_returned_address,
CASE
WHEN {{ masked_address_sql }} THEN '{{ dummy_phone }}'
ELSE phone_number
END AS phone_number,
CASE
WHEN {{ masked_address_sql }} THEN '[' || '{{ dummy_short_string }}' || ']'
ELSE location_type
END AS location_type,
CASE
WHEN {{ masked_address_sql }} THEN '{{ dummy_short_string }}'
ELSE operational_status
END AS operational_status,
CASE
WHEN {{ masked_address_sql }} THEN {{ dummy_value }}
ELSE latitude
END AS latitude,
CASE
WHEN {{ masked_address_sql }} THEN {{ dummy_value }}
ELSE longitude
END AS longitude,
CASE
WHEN {{ masked_address_sql }} THEN '{{ dummy_website }}'
ELSE location_website
END AS location_website,
CASE
WHEN {{ masked_address_sql }} THEN {{ dummy_value }}
ELSE location_overall_rating
END AS location_overall_rating,
CASE
WHEN {{ masked_address_sql }} THEN {{ dummy_value }}
ELSE location_number_of_reviews
END AS location_number_of_reviews,
{%- for day_of_week in days_of_week %}
CASE
WHEN {{ masked_address_sql }} THEN '{{ dummy_operating_hours }}'
ELSE {{day_of_week}}_final_operating_hours
END AS {{day_of_week}}_operating_hours,
{% endfor %}
CASE
WHEN {{ masked_address_sql }} THEN '{"' || '{{dummy_long_string}}' || '"}'
ELSE json_place_details_full_return
END AS json_place_details_full_return,
-- date/times
deleted_datetime,
created_datetime,
updated_datetime,
-- metadata
_function_ran_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM google_place_api_details_extraction
),
-- FINAL
@@ -64,6 +433,7 @@ final__stg_exchange__company_addresses AS (
country_id,
state_id,
district_id,
google_place_id,
-- dimensions
country_name,
@@ -75,6 +445,23 @@ final__stg_exchange__company_addresses AS (
address_line_one,
address_line_two,
is_billing_address,
google_returned_address,
phone_number,
location_type,
operational_status,
latitude,
longitude,
location_website,
location_overall_rating,
location_number_of_reviews,
monday_operating_hours,
tuesday_operating_hours,
wednesday_operating_hours,
thursday_operating_hours,
friday_operating_hours,
saturday_operating_hours,
sunday_operating_hours,
json_place_details_full_return,
-- measures
@@ -84,9 +471,10 @@ final__stg_exchange__company_addresses AS (
updated_datetime,
-- metadata
_function_ran_datetime,
_dbt_ran_datetime
FROM company_addressess_join_companies_countries_states_districts
)
FROM masked_data_transformation
)
SELECT * FROM final__stg_exchange__company_addresses
models/staging/exchange/stg_exchange__transaction_orders.sql
+4
View File
@@ -1,3 +1,7 @@
-- change name for final__stg_exchange__transaction_costs
-- @todo
-- @yam
-- IMPORTS
WITH transactions AS (
SELECT * FROM {{ ref('base_exchange__transactions') }}