mirror of
https://gitlab.com/cief-data/data-management.git
synced 2026-08-19 04:13:57 +00:00
Crisp ExtractLoad user only
This commit is contained in:
Yam ZhengLim
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"identifier" : "0f8edfb8-1271-4e8c-bdfc-e33de84a0efe",
|
||||
"key" : "408094e63ab670c968b3f9a119387263d4e23721817513b47bd94700bf4349a9",
|
||||
"identifier" : "6f00e404-3e97-4a85-afd5-3980d4f4d6c9",
|
||||
"key" : "3a20c99007db592dadaf9e4aaa85bce26ad8bb2421380639896fb7d17a0e927e",
|
||||
"website_id" : "665dcd41-1edf-4451-8cb9-f1cf9ed35e15"
|
||||
}
|
||||
Binary file not shown.
@@ -42,6 +42,7 @@ def get_peoples_csv(website_id, client, path, filename):
|
||||
else:
|
||||
page_number += 1
|
||||
|
||||
# Get marking by CustomerService Operator
|
||||
for people_id in peoples_list_df.people_id:
|
||||
print(people_id)
|
||||
people_data_json = client.website.get_people_data(website_id, people_id)
|
||||
@@ -51,8 +52,16 @@ def get_peoples_csv(website_id, client, path, filename):
|
||||
|
||||
peoples_list_df = peoples_list_df.merge(peoples_data_df, on='people_id', how='left')
|
||||
|
||||
peoples_list_df.to_csv(output_location, sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
peoples_data_df.to_csv('test.csv', sep=',', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
# Get email subscription status
|
||||
for people_id in peoples_list_df.people_id:
|
||||
people_subscription_status_json = client.website.get_people_subscription_status(website_id, people_id)
|
||||
temp_df = general_function.json_normalize_to_df(people_subscription_status_json)
|
||||
temp_df['people_id'] = people_id
|
||||
people_subscription_status_df = pd.concat([temp_df,people_subscription_status_json], ignore_index=True)
|
||||
|
||||
peoples_list_df = peoples_list_df.merge(people_subscription_status_df, on='people_id', how='left')
|
||||
|
||||
peoples_list_df.to_csv(output_location, sep='|', encoding='utf-8-sig', header='true', quotechar= '"', index=False)
|
||||
print("[Download Complete]",filename)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
type|details.user_id|details.email|details.avatar|details.first_name|details.last_name|details.role|details.title|details.availability|details.has_token
|
||||
operator|15f0ec35-8a0c-457f-83ad-68e04f3b6624|taypeisongtps@gmail.com|https://storage.crisp.chat/users/avatar/operator/5e71c78373d9ac00/potrait-photo_gsa6gd.jpg|Pei Song|Tay|member|Customer Service|away|False
|
||||
operator|293073d9-e65f-40e2-a1ca-54fb9a2e4ff7|ykheng2bb@gmail.com|https://storage.crisp.chat/users/avatar/operator/4a7ce09c05d7b400/whatsapp-image-2022-04-13-at-2_19jwcch.jpeg|YAP|KAH HENG|owner|Customer Service|online|False
|
||||
operator|293073d9-e65f-40e2-a1ca-54fb9a2e4ff7|ykheng2bb@gmail.com|https://storage.crisp.chat/users/avatar/operator/4a7ce09c05d7b400/whatsapp-image-2022-04-13-at-2_19jwcch.jpeg|YAP|KAH HENG|owner|Customer Service|away|False
|
||||
operator|294351f3-d501-41dd-a095-2c512dd2ff4f|ciefworldwide.acc1@gmail.com||Haizat|Zaidi|owner||away|False
|
||||
operator|2a0b5503-97ff-4875-918d-be8185bc58a1|h2009413007@hcu.edu.my|https://storage.crisp.chat/users/avatar/operator/6bba6be706925c00/screenshot-2022-09-30-150004_mst2g.png|Tok|soon wee|member|Consultant|online|False
|
||||
operator|2a0b5503-97ff-4875-918d-be8185bc58a1|h2009413007@hcu.edu.my|https://storage.crisp.chat/users/avatar/operator/6bba6be706925c00/screenshot-2022-09-30-150004_mst2g.png|Tok|soon wee|member|Consultant|away|False
|
||||
operator|413c155c-3f3b-4859-bc18-dddb17dfd3ac|diliniwr10@gmail.com|https://storage.crisp.chat/users/avatar/operator/ed6e6534f8f01000/dili_wntzti.png|Dilini|WR|member||away|False
|
||||
operator|6f231d14-b3c0-47b5-bd02-617579cd2bcb|kangjun8331@gmail.com|https://storage.crisp.chat/users/avatar/operator/9ccd7bf346431800/img1413_d7cf1m.jpg|Kang|Jun|member|Consultant|away|False
|
||||
operator|80708bf4-9c08-4b8d-a37b-3a2f3c9570f1|jhunyanwong@gmail.com|https://storage.crisp.chat/users/avatar/operator/b20140b4cfccf800/214025936361797548903332433376_1r1o1en.jpg|JY|Wong|owner||away|False
|
||||
operator|83fb3451-f17a-4876-b317-244c30b27c8b|shafiqa_sukeri@cief-malaysia.com|https://storage.crisp.chat/users/avatar/operator/6ba11d171dddb800/photo2022-07-0811-15-15_9ulegi.jpg|FAIN|BACKEND|owner|Customer Service|online|False
|
||||
operator|99a1c001-60ac-4f71-91e1-33e8c2205525|enxin1227@gmail.com|https://storage.crisp.chat/users/upload/operator/bb7c7077d72bc800/21064570005_zf9jf6.jpg|Tiffany|Ooi|member||online|False
|
||||
operator|83fb3451-f17a-4876-b317-244c30b27c8b|shafiqa_sukeri@cief-malaysia.com|https://storage.crisp.chat/users/avatar/operator/6ba11d171dddb800/photo2022-07-0811-15-15_9ulegi.jpg|FAIN|BACKEND|owner|Customer Service|away|False
|
||||
operator|99a1c001-60ac-4f71-91e1-33e8c2205525|enxin1227@gmail.com|https://storage.crisp.chat/users/upload/operator/bb7c7077d72bc800/21064570005_zf9jf6.jpg|Tiffany|Ooi|member||away|False
|
||||
operator|9df5d57c-8eb6-4e43-a8c5-cc6282564de9|ciefworldwide@gmail.com||PM|Wong|owner|CEO|away|False
|
||||
operator|a8079229-94c0-4a33-a027-bcbaed17a1c0|uldvstar@gmail.com|https://storage.crisp.chat/users/avatar/operator/8f0372fcb5eb0000/download-1_17y48zb.png|Omair|Saleh|owner||away|False
|
||||
operator|d8e200bc-709b-4b80-a706-bab8ec612e0b|infociefworldwide@gmail.com||CIEF|Marketing|owner|Marketing|away|False
|
||||
operator|d9ef0024-60c9-4137-919d-1d25bc2fab0e|operationteam@cief-malaysia.com||Operation|Team|member||away|False
|
||||
operator|db57d7a6-9421-43b9-9614-0867db22739a|ziwen38@gmail.com|https://storage.crisp.chat/users/avatar/operator/eed9c8d96123e00/resume_1x03t8n.jpeg|Zi Wen|Lee|member||online|False
|
||||
operator|dd8d2dcb-07a4-4343-b5bd-93038d08bd54|lingzunsien1211@gmail.com|https://storage.crisp.chat/users/avatar/operator/67b2c5d990ba9800/resume-pic_1q3r11v.jpg|Zun Sien|Ling|member||online|False
|
||||
operator|db57d7a6-9421-43b9-9614-0867db22739a|ziwen38@gmail.com|https://storage.crisp.chat/users/avatar/operator/eed9c8d96123e00/resume_1x03t8n.jpeg|Zi Wen|Lee|member||away|False
|
||||
operator|dd8d2dcb-07a4-4343-b5bd-93038d08bd54|lingzunsien1211@gmail.com|https://storage.crisp.chat/users/avatar/operator/67b2c5d990ba9800/resume-pic_1q3r11v.jpg|Zun Sien|Ling|member||away|False
|
||||
operator|e486c123-0501-43e7-aa00-0231fb589c2e|potter2186@gmail.com||Harry|Wong|owner|Manager|away|False
|
||||
operator|f43c636e-c023-4169-a555-be63a1f34a12|sookni98@gmail.com|https://storage.crisp.chat/users/avatar/operator/db5dc231ebfc1000/ksn-5_unmi7u.jpeg|Sook Ni|Kho|member||online|False
|
||||
operator|f43c636e-c023-4169-a555-be63a1f34a12|sookni98@gmail.com|https://storage.crisp.chat/users/avatar/operator/db5dc231ebfc1000/ksn-5_unmi7u.jpeg|Sook Ni|Kho|member||away|False
|
||||
operator|fcbaf32c-29dc-4a24-af7d-77ceeb8616b9|yamzhenglim0427@gmail.com||ZhengLim|Yam|owner||online|False
|
||||
sandbox|631dc1f5-f672-41a6-b32a-0dc05339c97f|yamzhenglim0427@gmail.com||ZhengLim|Yam||||
|
||||
|
||||
|
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@@ -0,0 +1,21 @@
|
||||
from google.cloud import storage
|
||||
|
||||
def login_cloudstorage_credential(credential):
|
||||
try:
|
||||
storage_client = storage.Client.from_service_account_json(credential)
|
||||
return storage_client
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-login_cloudstorage_credential")
|
||||
return False
|
||||
|
||||
def upload_to_bucket(storage_client, blob_name, file_path):
|
||||
try:
|
||||
crisp_bucket = storage_client.get_bucket('crisp_bucket_raw')
|
||||
blob = crisp_bucket.blob(blob_name)
|
||||
blob.upload_from_filename(file_path)
|
||||
return print("[Upload Complete]",file_path)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("[FUNCTION_ERROR]-upload_to_bucket",blob_name)
|
||||
return False
|
||||
@@ -6,6 +6,7 @@ import exchange.load.exchange_load_function as exchange_load_func
|
||||
import shipping.extract.shipping_extract_function as shipping_extract_func
|
||||
import shipping.load.shipping_load_function as shipping_load_func
|
||||
import crisp.extract.crisp_extract_function as crisp_extract_func
|
||||
import crisp.load.crisp_load_function as crisp_load_func
|
||||
|
||||
|
||||
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
|
||||
@@ -93,9 +94,28 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
#Get operators csv
|
||||
crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv')
|
||||
|
||||
# #Get users csv
|
||||
#Get users(people) csv
|
||||
crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv')
|
||||
|
||||
def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
#Config file name
|
||||
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
|
||||
|
||||
#Connect to cloudstorage
|
||||
cirsp_storage_client = crisp_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
|
||||
|
||||
#Get the filename and path in temp csv storage
|
||||
temp_file_list = []
|
||||
for (dirpath, dirnames, filenames) in walk(CRISP_CSV_TEMP_STORAGE_PATH):
|
||||
temp_file_list.extend(filenames)
|
||||
|
||||
#Loop all the file and upload to cloudstorage
|
||||
for file_name in temp_file_list:
|
||||
file_path = os.path.join(dirpath,file_name)
|
||||
crisp_load_func.upload_to_bucket(cirsp_storage_client, file_name, file_path)
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
#Make sure cron in the file directory
|
||||
general_function.change_to_current_directory()
|
||||
@@ -120,14 +140,16 @@ def main():
|
||||
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
|
||||
# #Make sure cron in the file directory
|
||||
# general_function.change_to_current_directory()
|
||||
#Make sure cron in the file directory
|
||||
general_function.change_to_current_directory()
|
||||
|
||||
# #Extract Load Exchange Data
|
||||
# CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
|
||||
# crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
|
||||
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
#Delete Temp CSV
|
||||
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
|
||||
|
||||
if True:
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user