data-management/crisp/extract/crisp_extract_function.py

from crisp_api import Crisp
import general_function
import pandas as pd
from datetime import date, timedelta, datetime

def connect_crisp(config):
    try:
        client = Crisp()
        client.set_tier("plugin")
        client.authenticate(config["identifier"], config["key"])
        return client
    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-connect_crisp")
        return False
    
def list_website_operators_json(website_id, client, path, filename):
    try:
        output_location = path + filename
        list_website_operators_json = client.website.list_website_operators(website_id)
        list_website_operators_df = general_function.json_normalize_to_df(list_website_operators_json, sep='__')

        # Datetime add
        list_website_operators_df['api_called_at'] = datetime.now()

        # list_website_operators_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        list_website_operators_df.to_json(output_location, orient='records', force_ascii = False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-list_website_operators_json")
        return False         

def list_people_profiles_json(website_id, client, path, filename):
    try:
        output_location = path + filename
        list_people_profiles_df = pd.DataFrame()

        loop_flag = True
        page_number = 1

        while loop_flag == True:
            list_people_profiles_json = client.website.list_people_profiles(website_id, page_number)
            temp_df = general_function.json_normalize_to_df(list_people_profiles_json, sep='__')
            temp_df['api_page_number'] = page_number
            list_people_profiles_df = pd.concat([temp_df,list_people_profiles_df], ignore_index=True)
            if not list_people_profiles_json:
                loop_flag = False
            else:
                page_number += 1
        # Timestamp to datetime
        list_people_profiles_df['created_at']=(pd.to_datetime(list_people_profiles_df['created_at'],unit='ms')) 
        list_people_profiles_df['updated_at']=(pd.to_datetime(list_people_profiles_df['updated_at'],unit='ms')) 
        list_people_profiles_df['active__last']=(pd.to_datetime(list_people_profiles_df['active__last'],unit='ms')) 

        # Datetime add
        list_people_profiles_df['api_called_at'] = datetime.now()

        # list_people_profiles_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        list_people_profiles_df.to_json(output_location, orient='records', force_ascii = False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-list_people_profiles_json")
        return False         


def get_people_data_json(website_id, client, path, filename, list_people_profiles_json_path):
    try:
        output_location = path + filename
        get_people_data_df = pd.DataFrame()

        person_ids = pd.read_json(list_people_profiles_json_path)['people_id']

        # Get marking by CustomerService Operator
        for people_id in person_ids:
            get_people_data_json = client.website.get_people_data(website_id, people_id)
            temp_df = general_function.json_normalize_to_df(get_people_data_json)


            temp_df['people_id'] = people_id
            get_people_data_df = pd.concat([temp_df,get_people_data_df], ignore_index=True)

        # Fix setup error by CX team, eg( exchange-marking, shipping-marking -> should be shipping_marking)
        get_people_data_df.columns = get_people_data_df.columns.str.replace("-", "_")
        get_people_data_df.columns = get_people_data_df.columns.str.replace(".", "__")

        # Datetime add
        get_people_data_df['api_called_at'] = datetime.now()

        # get_people_data_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        get_people_data_df.to_json(output_location, orient='records', force_ascii = False)
        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_people_data_json")
        return False         



def get_people_subscription_status_json(website_id, client, path, filename, list_people_profiles_json_path):
    try:
        output_location = path + filename
        get_people_subscription_status_df = pd.DataFrame()

        person_ids = pd.read_json(list_people_profiles_json_path)['people_id']
        
        # Get email subscription status
        for people_id in person_ids:
            get_people_subscription_status_df_json = client.website.get_people_subscription_status(website_id, people_id)
            temp_df = general_function.json_normalize_to_df(get_people_subscription_status_df_json, sep='__')
            temp_df['people_id'] = people_id
            get_people_subscription_status_df = pd.concat([temp_df,get_people_subscription_status_df], ignore_index=True)

        # Datetime add
        get_people_subscription_status_df['api_called_at'] = datetime.now()

        get_people_subscription_status_df.columns = get_people_subscription_status_df.columns.str.replace(".", "-")
        # get_people_subscription_status_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"',  index=False)
        get_people_subscription_status_df.to_json(output_location, orient='records', force_ascii = False)

        print("[Download Complete]",filename)

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_people_subscription_status_json")
        return False         



def list_conversations_json(website_id, client, path, filename):
    try:
        loop_flag = True
        page_number = 1
        output_location = path + filename
        conversations_list_df = pd.DataFrame()

        while loop_flag:
            conversations_list_df_json = client.website.list_conversations(website_id, page_number)

            temp_df = general_function.json_normalize_to_df(conversations_list_df_json, sep='__')
            conversations_list_df = pd.concat([conversations_list_df, temp_df], ignore_index=True)

            if conversations_list_df_json == []:
                loop_flag = False
            else:
                page_number += 1
        
        conversations_list_df.columns = conversations_list_df.columns.str.replace("-", "_")
        conversations_list_df['api_called_at'] = datetime.now()

        conversations_list_df.to_json(output_location, orient='records', force_ascii = False)
        print("[Download Complete]",filename)
        
    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-list_conversations_json")
        return False  


def get_message_in_conversations_json(website_id, client, path, filename, list_conversations_json_path):
    try:
        output_location = path + filename
        get_message_in_conversations_df = pd.DataFrame()
        session_ids = pd.read_json(list_conversations_json_path)['session_id']

        for session_id in session_ids:
            print(session_id) #NEED DELETE

            get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, "")   
            temp_df = general_function.json_normalize_to_df(get_message_in_conversations_json, sep='__')
            get_message_in_conversations_df = pd.concat([temp_df, get_message_in_conversations_df], ignore_index=True)

            while (temp_df.shape[0] == 40):       #Maximum can get 40 conversation per request, Request for date before if the temp_df is full
                query = { "timestamp_before" : str(temp_df.at[0,'timestamp']) }
                get_message_in_conversations_json = client.website.get_messages_in_conversation(website_id, session_id, query)
                temp_df = pd.json_normalize(get_message_in_conversations_json, sep='__')
                get_message_in_conversations_df = pd.concat([temp_df,get_message_in_conversations_df], ignore_index=True)
            
        get_message_in_conversations_df['api_called_at'] = datetime.now()
        get_message_in_conversations_df.to_json(output_location, orient='records', force_ascii = False)
        print("[Download Complete]",filename)
        

    except Exception as e:
        print(e)
        print("[FUNCTION_ERROR]-get_message_in_conversations_json")
        return False