cief/dbt_cloud
1
0
Fork 0
mirror of https://gitlab.com/cief-data/dbt_cloud.git synced 2026-08-19 04:14:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'compile_company_dimension_for_dedupe' into 'main'

Browse Source 
Compile company dimension for dedupe

See merge request cief-data/dbt_cloud!19
This commit is contained in:
Yam ZhengLim
2023-08-02 14:43:30 +00:00
parent 032cdf0960 f8c337e731
commit a365b87988
5 changed files with 396 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dbt_project.yml
+8 -10
View File
@@ -36,16 +36,6 @@ models:
+tags:
- staging
autocount:
+materialized: view
+schema: autocount__staging
+tags:
- autocount
base:
+materialized: view
+schema: autocount__base
crisp:
+materialized: view
+schema: crisp__staging
@@ -66,6 +56,14 @@ models:
+materialized: view
+schema: exchange__base
googlesheet:
+materialized: view
+schema: googlesheet__stagting
base:
+materialized: view
+schema: googlesheet__base
shipping:
+materialized: view
+schema: shipping__staging
models/intermediate/int__dates.sql
+250
View File
@@ -0,0 +1,250 @@
-- CONFIG
{{
config(
materialized='view'
)
}}
-- IMPORT
WITH holiday_and_non_working_day_lists AS (
SELECT * FROM {{ ref('stg_googlesheet__event_and_holiday_lists') }}
),
-- LOGIC
date_spine AS (
{{ dbt_utils.date_spine(
start_date="to_date('01/01/2015', 'mm/dd/yyyy')",
datepart="day",
end_date="dateadd(year, 40, current_date)"
)
}}
),
fiscal_calendar as (
SELECT
date_day,
date_day AS date_actual,
DAYNAME(date_day) AS day_name,
DATE_PART('month', date_day) AS month_actual,
DATE_PART('year', date_day) AS year_actual,
DATE_PART(quarter, date_day) AS quarter_actual,
CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7
ELSE DATE_PART(dayofweek, date_day) END AS day_of_week,
CASE WHEN day_name = 'Mon' THEN date_day
ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week,
CASE WHEN day_name = 'Mon' THEN WEEK(date_day)
ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column
CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1'
THEN '1'
ELSE week_of_year_temp END AS week_of_year,
DATE_PART('day', date_day) AS day_of_month,
ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter,
ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year,
-- @TODO @GERSON ,done
-- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023)
CASE
WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual
WHEN DATE_PART('month', DATEADD( month,-1, date_day) ) = 12 THEN year_actual-1
ELSE (year_actual+1)
END AS fiscal_year,
-- @TODO @GERSON , done
-- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4)
CASE WHEN month_actual in (2,3,4) THEN '1'
WHEN month_actual in (5,6,7) THEN '2'
WHEN month_actual in (8,9,10) THEN '3'
WHEN month_actual in (11,12) THEN '4'
WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4'
-- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4'
ELSE '4' END AS fiscal_quarter,
-- @TODO @GERSON ,done
ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter,
ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year,
TO_CHAR(date_day, 'MMMM') AS month_name,
TRUNC(date_day, 'Month') AS first_day_of_month,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month,
FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year,
FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter,
LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter,
-- @TODO @GERSON ,done
-- Change this 2 too
FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter,
LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter,
-- @TODO @GERSON,done
FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year,
LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year,
-- @TODO @GERSON ,done
DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year,
-- @TODO @GERSON ,done
CASE WHEN EXTRACT('month', date_day) = 1 THEN 12
ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year,
LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week,
(year_actual || '-Q' || fiscal_quarter) AS quarter_name,
-- @TODO @GERSON,done
(fiscal_year || '-' || DECODE(fiscal_quarter,
1, 'Q1',
2, 'Q2',
3, 'Q3',
4, 'Q4')) AS fiscal_quarter_name,
('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy,
DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute,
--here check
fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name,
('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy,
-- @TODO @GERSON
DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter,
IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter,
DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year,
IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year,
DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa,
DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings
FROM date_spine
QUALIFY date_actual >= '2016-01-01'
ORDER BY date_day
),
holiday_group_by_date AS (
SELECT
date,
day,
ARRAY_AGG(DISTINCT event_name) WITHIN GROUP (ORDER BY event_name) AS holiday_event_names,
ARRAY_AGG(DISTINCT event_description) WITHIN GROUP (ORDER BY event_description) AS holiday_event_descriptions,
ARRAY_AGG(DISTINCT state) WITHIN GROUP (ORDER BY state) AS holiday_states,
ARRAY_AGG(DISTINCT country) WITHIN GROUP (ORDER BY country) AS holiday_countries,
ARRAY_AGG(DISTINCT religion) WITHIN GROUP (ORDER BY religion) AS holiday_religions,
ARRAY_AGG(DISTINCT holiday_general_type) WITHIN GROUP (ORDER BY holiday_general_type) AS holiday_event_general_types,
ARRAY_AGG(DISTINCT holiday_specific_type) WITHIN GROUP (ORDER BY holiday_specific_type) AS holiday_event_specific_types,
MAX(is_malaysia_holiday) AS is_malaysia_holiday,
MAX(is_company_holiday) AS is_company_holiday,
MAX(is_china_holiday) AS is_china_holiday,
MAX(is_china_warehouse_holiday) AS is_china_warehouse_holiday
FROM holiday_and_non_working_day_lists
GROUP BY
day,
date
ORDER BY date
),
fiscal_calendar_join_holiday_list AS (
SELECT
fiscal_calendar.*,
holidays.holiday_event_names,
holidays.holiday_event_descriptions,
holidays.holiday_event_general_types,
holidays.holiday_event_specific_types,
holidays.holiday_states,
holidays.holiday_countries,
holidays.holiday_religions,
holidays.is_malaysia_holiday,
holidays.is_company_holiday,
holidays.is_china_holiday,
holidays.is_china_warehouse_holiday,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM fiscal_calendar
LEFT JOIN holiday_group_by_date AS holidays
ON fiscal_calendar.date_day = holidays.date
),
-- FINAL
final__int__dates AS (
SELECT
date_day,
date_actual,
day_name,
month_actual,
year_actual,
quarter_actual,
day_of_week,
first_day_of_week,
week_of_year,
day_of_month,
day_of_quarter,
day_of_year,
fiscal_year,
fiscal_quarter,
day_of_fiscal_quarter,
day_of_fiscal_year,
month_name,
first_day_of_month,
last_day_of_month,
first_day_of_year,
last_day_of_year,
first_day_of_quarter,
last_day_of_quarter,
first_day_of_fiscal_quarter,
last_day_of_fiscal_quarter,
first_day_of_fiscal_year,
last_day_of_fiscal_year,
week_of_fiscal_year,
month_of_fiscal_year,
last_day_of_week,
quarter_name,
fiscal_quarter_name,
fiscal_quarter_name_fy,
fiscal_quarter_number_absolute,
fiscal_month_name,
fiscal_month_name_fy,
last_month_of_fiscal_quarter,
is_first_day_of_last_month_of_fiscal_quarter,
last_month_of_fiscal_year,
is_first_day_of_last_month_of_fiscal_year,
snapshot_date_fpa,
snapshot_date_billings,
holiday_event_names,
holiday_event_descriptions,
holiday_event_general_types,
holiday_event_specific_types,
holiday_countries,
holiday_states,
holiday_religions,
is_malaysia_holiday,
is_company_holiday,
is_china_holiday,
is_china_warehouse_holiday,
_dbt_ran_datetime
FROM fiscal_calendar_join_holiday_list
)
SELECT * FROM final__int__dates
models/staging/exchange/base/base_exchange__seasonal_segments.sql
+55
View File
@@ -0,0 +1,55 @@
-- IMPORTS
WITH source as (
SELECT * FROM {{ source('src_exchange_mysql', 'seasonal_segment') }}
),
-- LOGIC
seasonal_segment_rename AS (
SELECT
id AS seasonal_segment_id,
company_id,
status,
segment_id,
starting_on AS seasonal_segment_started_datetime,
ending_on AS seasonal_segment_ended_datetime,
deleted_at AS seasonal_segment_deleted_datetime,
created_at AS seasonal_segment_created_datetime,
updated_at AS seasonal_segment_updated_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM source
-- id 1 is wrong setup, it should not occur
WHERE id != 1
),
-- FINAL
final__base_exchange__seasonal_segments AS (
SELECT
-- ids
seasonal_segment_id,
company_id,
segment_id,
-- dimensions
status,
-- measures
-- date/times
seasonal_segment_started_datetime,
seasonal_segment_ended_datetime,
seasonal_segment_deleted_datetime,
seasonal_segment_created_datetime,
seasonal_segment_updated_datetime,
-- metadata
_dbt_ran_datetime
FROM seasonal_segment_rename
)
SELECT * from final__base_exchange__seasonal_segments
models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql
+78
View File
@@ -0,0 +1,78 @@
-- IMPORT
WITH holiday_lists AS (
SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_event_and_holiday_lists') }}
),
-- LOGIC
parse_jason_holidays AS (
SELECT
_airbyte_ab_id AS airbyte_id,
parse_json(_airbyte_data) AS holiday_json,
_airbyte_emitted_at AS _airbyte_emitted_datetime
FROM holiday_lists
),
flatten_json AS (
SELECT
airbyte_id,
holiday_json['id']::INTEGER AS holiday_id,
holiday_json['date']::DATE AS date,
holiday_json['day']::STRING AS day,
holiday_json['event_name']::STRING AS event_name,
holiday_json['event_desc']::STRING AS event_description,
holiday_json['general_type']::STRING AS holiday_general_type,
holiday_json['holiday_type']::STRING AS holiday_specific_type,
holiday_json['country']::STRING AS country,
holiday_json['state']::STRING AS state,
holiday_json['event_cultural_origin']::STRING AS religion,
holiday_json['is_malaysia_holiday']::INTEGER AS is_malaysia_holiday,
holiday_json['is_company_holiday']::INTEGER AS is_company_holiday,
holiday_json['is_china_holiday']::INTEGER AS is_china_holiday,
holiday_json['is_china_warehouse_holiday']::INTEGER AS is_china_warehouse_holiday,
holiday_json['special_remark']::STRING AS remark,
_airbyte_emitted_datetime,
'{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime
FROM parse_jason_holidays
),
-- FINAL
final__stg_googlesheet__event_and_holiday_lists AS (
SELECT
-- ids
holiday_id,
airbyte_id,
-- dimensions
day,
event_name,
event_description,
holiday_general_type,
holiday_specific_type,
country,
state,
religion,
is_malaysia_holiday,
is_company_holiday,
is_china_holiday,
is_china_warehouse_holiday,
-- measures
-- date/times
date,
-- metadata
remark,
_airbyte_emitted_datetime,
_dbt_ran_datetime
FROM flatten_json
)
SELECT * FROM final__stg_googlesheet__event_and_holiday_lists
packages.yml
+3
View File
@@ -21,3 +21,6 @@ packages:
- package: dbt-labs/codegen
version: 0.9.0
- package: calogica/dbt_date
version: [">=0.7.0", "<0.8.0"]