cief/data-management
1
0
Fork 0
mirror of https://gitlab.com/cief-data/data-management.git synced 2026-08-19 04:13:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Crisp ExtractLoad user only

Browse Source
This commit is contained in:
Yam ZhengLim
2022-12-06 22:18:48 +08:00
parent 8f2b2b1896
commit 828a29d052
8 changed files with 1857 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+28 -6
View File
@@ -6,6 +6,7 @@ import exchange.load.exchange_load_function as exchange_load_func
import shipping.extract.shipping_extract_function as shipping_extract_func
import shipping.load.shipping_load_function as shipping_load_func
import crisp.extract.crisp_extract_function as crisp_extract_func
import crisp.load.crisp_load_function as crisp_load_func
def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH):
@@ -93,9 +94,28 @@ def crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH):
#Get operators csv
crisp_extract_func.get_operators_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_operators.csv')
# #Get users csv
#Get users(people) csv
crisp_extract_func.get_peoples_csv(CRISP_WEBSITE_ID, CLIENT, CRISP_CSV_TEMP_STORAGE_PATH, 'crisp_peoples.csv')
def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH):
#Config file name
CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json"
#Connect to cloudstorage
cirsp_storage_client = crisp_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE)
#Get the filename and path in temp csv storage
temp_file_list = []
for (dirpath, dirnames, filenames) in walk(CRISP_CSV_TEMP_STORAGE_PATH):
temp_file_list.extend(filenames)
#Loop all the file and upload to cloudstorage
for file_name in temp_file_list:
file_path = os.path.join(dirpath,file_name)
crisp_load_func.upload_to_bucket(cirsp_storage_client, file_name, file_path)
def main():
#Make sure cron in the file directory
general_function.change_to_current_directory()
@@ -120,14 +140,16 @@ def main():
general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH)
# #Make sure cron in the file directory
# general_function.change_to_current_directory()
#Make sure cron in the file directory
general_function.change_to_current_directory()
# #Extract Load Exchange Data
# CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
# crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/"
crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH)
crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH)
#Delete Temp CSV
general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH)
if True:
main()