data-management/crisp/extract/crisp_extract_function.py

from crisp_api import Crisp
import general_function
import pandas as pd
from datetime import date, timedelta, datetime

def connect_crisp(config):
    try:
        client = Crisp()
        client.set_tier("plugin")
        client.authenticate(config["identifier"], config["key"])
        return client
    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-connect_crisp")
        return False
    
def list_website_operators_csv(website_id, client, path, filename):
    try:
        output_location = path + filename
        list_website_operators_json = client.website.list_website_operators(website_id)
        list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='__')

        # Datetime add
        list_website_operators_df['api_called_at'] = date.today() - timedelta(days = 1)

        list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        print("[Download Complete]",filename)
    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-list_website_operators_csv")
        return False         

def list_people_profiles_csv(website_id, client, path, filename):
    try:
        output_location = path + filename
        list_people_profiles_df = pd.DataFrame()

        loop_flag = True
        page_number = 1

        while loop_flag == True:
            list_people_profiles_json = client.website.list_people_profiles(website_id, page_number)
            temp_df = general_function.json_normalize_to_df(list_people_profiles_json, sep='__')
            temp_df['api_page_number'] = page_number
            list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True)
            if not list_people_profiles_json:
                loop_flag = False
            else:
                page_number += 1
        # Timestamp to datetime
        list_people_profiles_df['created_at']=(pd.to_datetime(list_people_profiles_df['created_at'],unit='ms')) 
        list_people_profiles_df['updated_at']=(pd.to_datetime(list_people_profiles_df['updated_at'],unit='ms')) 
        list_people_profiles_df['active-last']=(pd.to_datetime(list_people_profiles_df['active-last'],unit='ms')) 

        # Datetime add
        list_people_profiles_df['api_called_at'] = date.today() - timedelta(days = 1)

        list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-list_people_profiles_csv")
        return False         


def get_people_data_csv(website_id, client, path, filename, list_people_profiles_csv_path):
    try:
        output_location = path + filename
        get_people_data_df = pd.DataFrame()

        person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']

        # Get marking by CustomerService Operator
        for people_id in person_ids:
            get_people_data_json = client.website.get_people_data(website_id, people_id)
            temp_df = general_function.json_normalize_to_df(get_people_data_json)


            temp_df['people_id'] = people_id
            get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)

        # Fix setup error by CX team, eg( exchange-marking, shipping-marking -> should be shipping_marking)
        get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_")
        get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "__")

        # Datetime add
        get_people_data_df['api_called_at'] = date.today() - timedelta(days = 1)

        get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_people_data_csv")
        return False         



def get_people_subscription_status_csv(website_id, client, path, filename, list_people_profiles_csv_path):
    try:
        output_location = path + filename
        get_people_subscription_status_df = pd.DataFrame()

        person_ids = pd.read_csv(list_people_profiles_csv_path, sep='|', encoding='utf-8-sig', quotechar= '"')['people_id']
        
        # Get email subscription status
        for people_id in person_ids:
            pget_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
            temp_df = general_function.json_normalize_to_df(pget_people_subscription_status_df_json, sep='__')
            temp_df['people_id'] = people_id
            get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)

        # Datetime add
        get_people_subscription_status_df['api_called_at'] = date.today() - timedelta(days = 1)

        get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
        get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_people_subscription_status_csv")
        return False