diff --git a/dbt_project.yml b/dbt_project.yml index 41345de..d503f08 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -35,16 +35,6 @@ models: staging: +tags: - staging - - autocount: - +materialized: view - +schema: autocount__staging - +tags: - - autocount - - base: - +materialized: view - +schema: autocount__base crisp: +materialized: view @@ -64,7 +54,15 @@ models: base: +materialized: view - +schema: exchange__base + +schema: exchange__base + + googlesheet: + +materialized: view + +schema: googlesheet__stagting + + base: + +materialized: view + +schema: googlesheet__base shipping: +materialized: view diff --git a/models/intermediate/int__dates.sql b/models/intermediate/int__dates.sql new file mode 100644 index 0000000..110b395 --- /dev/null +++ b/models/intermediate/int__dates.sql @@ -0,0 +1,250 @@ +-- CONFIG +{{ + config( + materialized='view' + ) +}} + +-- IMPORT +WITH holiday_and_non_working_day_lists AS ( + SELECT * FROM {{ ref('stg_googlesheet__event_and_holiday_lists') }} +), + +-- LOGIC +date_spine AS ( + + {{ dbt_utils.date_spine( + start_date="to_date('01/01/2015', 'mm/dd/yyyy')", + datepart="day", + end_date="dateadd(year, 40, current_date)" + ) + }} + +), + +fiscal_calendar as ( + + SELECT + date_day, + date_day AS date_actual, + + DAYNAME(date_day) AS day_name, + + DATE_PART('month', date_day) AS month_actual, + DATE_PART('year', date_day) AS year_actual, + + DATE_PART(quarter, date_day) AS quarter_actual, + + CASE WHEN DATE_PART(dayofweek, date_day) = 0 THEN 7 + ELSE DATE_PART(dayofweek, date_day) END AS day_of_week, + + CASE WHEN day_name = 'Mon' THEN date_day + ELSE DATEADD('day', 0, DATE_TRUNC('week', date_day)) END AS first_day_of_week, + + CASE WHEN day_name = 'Mon' THEN WEEK(date_day) + ELSE WEEK(date_day) END AS week_of_year_temp, --remove this column + + CASE WHEN day_name = 'Mon' AND LEAD(week_of_year_temp) OVER (ORDER BY date_day) = '1' + THEN '1' + ELSE week_of_year_temp END AS week_of_year, + + DATE_PART('day', date_day) AS day_of_month, + + ROW_NUMBER() OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS day_of_quarter, + ROW_NUMBER() OVER (PARTITION BY year_actual ORDER BY date_day) AS day_of_year, + + -- @TODO @GERSON ,done + -- BUSINESS LOGIC: Feb - Jan (2022), Feb - Jan (2023) + CASE + WHEN month_actual in (2,3,4,5,6,7,8,9,10,11,12) THEN year_actual + WHEN DATE_PART('month', DATEADD( month,-1, date_day) ) = 12 THEN year_actual-1 + ELSE (year_actual+1) + END AS fiscal_year, + + -- @TODO @GERSON , done + -- BUSINESS LOGIC: Feb - Apr (Q1), May - Jul (Q2), Aug - Oct (Q3), Nov - Jan (Q4) + CASE WHEN month_actual in (2,3,4) THEN '1' + WHEN month_actual in (5,6,7) THEN '2' + WHEN month_actual in (8,9,10) THEN '3' + WHEN month_actual in (11,12) THEN '4' + WHEN DATE_PART('month', DATEADD(month,-1,date_day)) =12 THEN '4' + -- WHEN month_actual= DATE_PART('year', month_actual) +1 and month_actual=1 THEN '4' + ELSE '4' END AS fiscal_quarter, + + -- @TODO @GERSON ,done + ROW_NUMBER() OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS day_of_fiscal_quarter, + ROW_NUMBER() OVER (PARTITION BY fiscal_year ORDER BY date_day) AS day_of_fiscal_year, + + TO_CHAR(date_day, 'MMMM') AS month_name, + + TRUNC(date_day, 'Month') AS first_day_of_month, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual, month_actual ORDER BY date_day) AS last_day_of_month, + + FIRST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS first_day_of_year, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual ORDER BY date_day) AS last_day_of_year, + + FIRST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS first_day_of_quarter, + LAST_VALUE(date_day) OVER (PARTITION BY year_actual, quarter_actual ORDER BY date_day) AS last_day_of_quarter, + + -- @TODO @GERSON ,done + -- Change this 2 too + FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS first_day_of_fiscal_quarter, + LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year, fiscal_quarter ORDER BY date_day) AS last_day_of_fiscal_quarter, + + -- @TODO @GERSON,done + FIRST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS first_day_of_fiscal_year, + LAST_VALUE(date_day) OVER (PARTITION BY fiscal_year ORDER BY date_day) AS last_day_of_fiscal_year, + + -- @TODO @GERSON ,done + DATEDIFF('week', first_day_of_fiscal_year, date_actual)+1 AS week_of_fiscal_year, + + -- @TODO @GERSON ,done + CASE WHEN EXTRACT('month', date_day) = 1 THEN 12 + ELSE EXTRACT('month', date_day) - 1 END AS month_of_fiscal_year, + + LAST_VALUE(date_day) OVER (PARTITION BY first_day_of_week ORDER BY date_day) AS last_day_of_week, + + (year_actual || '-Q' || fiscal_quarter) AS quarter_name, + + -- @TODO @GERSON,done + (fiscal_year || '-' || DECODE(fiscal_quarter, + 1, 'Q1', + 2, 'Q2', + 3, 'Q3', + 4, 'Q4')) AS fiscal_quarter_name, + ('FY' || SUBSTR(fiscal_quarter_name, 3, 7)) AS fiscal_quarter_name_fy, + DENSE_RANK() OVER (ORDER BY fiscal_quarter_name) AS fiscal_quarter_number_absolute, + --here check + fiscal_year || '-' || MONTHNAME(date_day) AS fiscal_month_name, + ('FY' || SUBSTR(fiscal_month_name, 3, 8)) AS fiscal_month_name_fy, + + -- @TODO @GERSON + DATE_TRUNC('month', last_day_of_fiscal_quarter) AS last_month_of_fiscal_quarter, + IFF(DATE_TRUNC('month', last_day_of_fiscal_quarter) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_quarter, + DATE_TRUNC('month', last_day_of_fiscal_year) AS last_month_of_fiscal_year, + IFF(DATE_TRUNC('month', last_day_of_fiscal_year) = date_actual, TRUE, FALSE) AS is_first_day_of_last_month_of_fiscal_year, + + DATEADD('day',7,DATEADD('month',1,first_day_of_month)) AS snapshot_date_fpa, + DATEADD('day',44,DATEADD('month',1,first_day_of_month)) AS snapshot_date_billings + + FROM date_spine + + QUALIFY date_actual >= '2016-01-01' + + ORDER BY date_day +), + +holiday_group_by_date AS ( + + SELECT + date, + day, + ARRAY_AGG(DISTINCT event_name) WITHIN GROUP (ORDER BY event_name) AS holiday_event_names, + ARRAY_AGG(DISTINCT event_description) WITHIN GROUP (ORDER BY event_description) AS holiday_event_descriptions, + ARRAY_AGG(DISTINCT state) WITHIN GROUP (ORDER BY state) AS holiday_states, + ARRAY_AGG(DISTINCT country) WITHIN GROUP (ORDER BY country) AS holiday_countries, + ARRAY_AGG(DISTINCT religion) WITHIN GROUP (ORDER BY religion) AS holiday_religions, + ARRAY_AGG(DISTINCT holiday_general_type) WITHIN GROUP (ORDER BY holiday_general_type) AS holiday_event_general_types, + ARRAY_AGG(DISTINCT holiday_specific_type) WITHIN GROUP (ORDER BY holiday_specific_type) AS holiday_event_specific_types, + MAX(is_malaysia_holiday) AS is_malaysia_holiday, + MAX(is_company_holiday) AS is_company_holiday, + MAX(is_china_holiday) AS is_china_holiday, + MAX(is_china_warehouse_holiday) AS is_china_warehouse_holiday + + FROM holiday_and_non_working_day_lists + + GROUP BY + day, + date + + ORDER BY date +), + +fiscal_calendar_join_holiday_list AS ( + + SELECT + fiscal_calendar.*, + + holidays.holiday_event_names, + holidays.holiday_event_descriptions, + holidays.holiday_event_general_types, + holidays.holiday_event_specific_types, + holidays.holiday_states, + holidays.holiday_countries, + holidays.holiday_religions, + holidays.is_malaysia_holiday, + holidays.is_company_holiday, + holidays.is_china_holiday, + holidays.is_china_warehouse_holiday, + + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM fiscal_calendar + + LEFT JOIN holiday_group_by_date AS holidays + ON fiscal_calendar.date_day = holidays.date +), + +-- FINAL +final__int__dates AS ( + + SELECT + date_day, + date_actual, + day_name, + month_actual, + year_actual, + quarter_actual, + day_of_week, + first_day_of_week, + week_of_year, + day_of_month, + day_of_quarter, + day_of_year, + fiscal_year, + fiscal_quarter, + day_of_fiscal_quarter, + day_of_fiscal_year, + month_name, + first_day_of_month, + last_day_of_month, + first_day_of_year, + last_day_of_year, + first_day_of_quarter, + last_day_of_quarter, + first_day_of_fiscal_quarter, + last_day_of_fiscal_quarter, + first_day_of_fiscal_year, + last_day_of_fiscal_year, + week_of_fiscal_year, + month_of_fiscal_year, + last_day_of_week, + quarter_name, + fiscal_quarter_name, + fiscal_quarter_name_fy, + fiscal_quarter_number_absolute, + fiscal_month_name, + fiscal_month_name_fy, + last_month_of_fiscal_quarter, + is_first_day_of_last_month_of_fiscal_quarter, + last_month_of_fiscal_year, + is_first_day_of_last_month_of_fiscal_year, + snapshot_date_fpa, + snapshot_date_billings, + holiday_event_names, + holiday_event_descriptions, + holiday_event_general_types, + holiday_event_specific_types, + holiday_countries, + holiday_states, + holiday_religions, + is_malaysia_holiday, + is_company_holiday, + is_china_holiday, + is_china_warehouse_holiday, + _dbt_ran_datetime + + FROM fiscal_calendar_join_holiday_list +) + +SELECT * FROM final__int__dates \ No newline at end of file diff --git a/models/staging/exchange/base/base_exchange__seasonal_segments.sql b/models/staging/exchange/base/base_exchange__seasonal_segments.sql new file mode 100644 index 0000000..918906c --- /dev/null +++ b/models/staging/exchange/base/base_exchange__seasonal_segments.sql @@ -0,0 +1,55 @@ +-- IMPORTS +WITH source as ( + SELECT * FROM {{ source('src_exchange_mysql', 'seasonal_segment') }} + +), + +-- LOGIC +seasonal_segment_rename AS ( + + SELECT + id AS seasonal_segment_id, + company_id, + status, + segment_id, + starting_on AS seasonal_segment_started_datetime, + ending_on AS seasonal_segment_ended_datetime, + deleted_at AS seasonal_segment_deleted_datetime, + created_at AS seasonal_segment_created_datetime, + updated_at AS seasonal_segment_updated_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM source + + -- id 1 is wrong setup, it should not occur + WHERE id != 1 +), + +-- FINAL +final__base_exchange__seasonal_segments AS ( + + SELECT + -- ids + seasonal_segment_id, + company_id, + segment_id, + + -- dimensions + status, + + -- measures + + -- date/times + seasonal_segment_started_datetime, + seasonal_segment_ended_datetime, + seasonal_segment_deleted_datetime, + seasonal_segment_created_datetime, + seasonal_segment_updated_datetime, + + -- metadata + _dbt_ran_datetime + + FROM seasonal_segment_rename +) + +SELECT * from final__base_exchange__seasonal_segments diff --git a/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql b/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql new file mode 100644 index 0000000..61fbfb3 --- /dev/null +++ b/models/staging/googlesheet/stg_googlesheet__event_and_holiday_lists.sql @@ -0,0 +1,78 @@ +-- IMPORT +WITH holiday_lists AS ( + SELECT * FROM {{ source('src_googlesheet_airbyte', '_airbyte_raw_event_and_holiday_lists') }} +), + +-- LOGIC +parse_jason_holidays AS ( + + SELECT + _airbyte_ab_id AS airbyte_id, + parse_json(_airbyte_data) AS holiday_json, + _airbyte_emitted_at AS _airbyte_emitted_datetime + + FROM holiday_lists +), + +flatten_json AS ( + + SELECT + airbyte_id, + holiday_json['id']::INTEGER AS holiday_id, + holiday_json['date']::DATE AS date, + holiday_json['day']::STRING AS day, + holiday_json['event_name']::STRING AS event_name, + holiday_json['event_desc']::STRING AS event_description, + holiday_json['general_type']::STRING AS holiday_general_type, + holiday_json['holiday_type']::STRING AS holiday_specific_type, + holiday_json['country']::STRING AS country, + holiday_json['state']::STRING AS state, + holiday_json['event_cultural_origin']::STRING AS religion, + holiday_json['is_malaysia_holiday']::INTEGER AS is_malaysia_holiday, + holiday_json['is_company_holiday']::INTEGER AS is_company_holiday, + holiday_json['is_china_holiday']::INTEGER AS is_china_holiday, + holiday_json['is_china_warehouse_holiday']::INTEGER AS is_china_warehouse_holiday, + holiday_json['special_remark']::STRING AS remark, + + _airbyte_emitted_datetime, + '{{ modules.datetime.datetime.now(modules.pytz.timezone("Asia/Kuala_Lumpur")) }}' AS _dbt_ran_datetime + + FROM parse_jason_holidays +), + +-- FINAL +final__stg_googlesheet__event_and_holiday_lists AS ( + + SELECT + -- ids + holiday_id, + airbyte_id, + + -- dimensions + day, + event_name, + event_description, + holiday_general_type, + holiday_specific_type, + country, + state, + religion, + is_malaysia_holiday, + is_company_holiday, + is_china_holiday, + is_china_warehouse_holiday, + + -- measures + + -- date/times + date, + + -- metadata + remark, + _airbyte_emitted_datetime, + _dbt_ran_datetime + + FROM flatten_json +) + +SELECT * FROM final__stg_googlesheet__event_and_holiday_lists diff --git a/packages.yml b/packages.yml index a87b53f..72d3ad1 100644 --- a/packages.yml +++ b/packages.yml @@ -20,4 +20,7 @@ packages: version: 1.0.0 - package: dbt-labs/codegen - version: 0.9.0 \ No newline at end of file + version: 0.9.0 + + - package: calogica/dbt_date + version: [">=0.7.0", "<0.8.0"] \ No newline at end of file