diff --git a/README.md b/README.md index 53d3529..45e4adf 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ data-management ├─ .gitignore # Ignore the file to push into gitlab │ ├─ config # Folder store all the config file to use in the code +│ ├─ C2M_CONFIG.json # C2M webscraping configuration │ ├─ CLOUD-STORAGE-ADMIN-CREDENTIAL.json # Get from GCP Service Account cloud-storage-admin-service-account │ ├─ CRISP_CONFIG.json # Config for CRISP API key, secret & web-id │ ├─ EXCHANGE_CONFIG.json # Manually config the MySQL ID, Password, etc... @@ -64,6 +65,17 @@ data-management │ ├─ SHIPPING_CONFIG.json # Manually config the MySQL ID, Password, etc... │ └─ SHIPPING_EXTRACT_SQL.json # SQL use in MYSQL to extract the data we need to save in temp's csv │ +├─ c2m # C2M webscrapping +│ ├─ scrape # All script use to scrape the competitor data is in here +│ │ ├─ c2m_scrape_app.py # C2M webscrap, call in main.py or seperate run it in your local environment +│ │ ├─ c2m_temp_scrape_files # Temp folder to store all the csv or error.png from webscraper (Auto Create) +│ │ │ └─ .gitkeep # Keep empty folder +│ │ └─ local_run_files # Folder to store all the data or error from webscraper which run at local pc +│ │ └─ error # Folder to store error.png from webscraper which run at local pc +│ │ └─ .gitkeep # Keep empty folder +│ └─ load +│ └─ c2m_load_function.py # C2M load code function to import in main.py +│ ├─ crisp # CRISP-CRM │ ├─ extract # All crisp extract needed code is in here │ │ ├─ crisp_extract_function.py # Crisp extract code function to import in main.py diff --git a/c2m/load/c2m_load_function.py b/c2m/load/c2m_load_function.py new file mode 100644 index 0000000..50151ed --- /dev/null +++ b/c2m/load/c2m_load_function.py @@ -0,0 +1,21 @@ +from google.cloud import storage + +def login_cloudstorage_credential(credential): + try: + storage_client = storage.Client.from_service_account_json(credential) + return storage_client + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-login_cloudstorage_credential") + return False + +def upload_to_bucket(storage_client, blob_name, file_path): + try: + crisp_bucket = storage_client.get_bucket('exchange_competitor_bucket_raw') + blob = crisp_bucket.blob(blob_name) + blob.upload_from_filename(file_path) + return print("[Upload Complete]",file_path) + except Exception as e: + print(e) + print("[FUNCTION_ERROR]-upload_to_bucket",blob_name) + return False \ No newline at end of file diff --git a/c2m/scrape/c2m_scrape_app.py b/c2m/scrape/c2m_scrape_app.py new file mode 100644 index 0000000..ab05f05 --- /dev/null +++ b/c2m/scrape/c2m_scrape_app.py @@ -0,0 +1,152 @@ +from playwright.sync_api import Playwright, sync_playwright +import re +import time +import csv +import json +import os +import pandas as pd +from datetime import datetime + + +def run(playwright: Playwright, DATA_PATH, ERROR_PATH, CONFIG) -> None: + try: + dt = datetime.now() + date = dt.strftime('%Y-%m-%d') + hour = dt.hour + + RANGE = CONFIG['range'] + PRODUCT_NAME = CONFIG['product_name'] + SITE_NAME = CONFIG['site_name'] + + browser = playwright.chromium.launch(headless=False) + context = browser.new_context() + page = context.new_page() + + page.goto(CONFIG["base_url"]) + page.click("text=选择语言") + + with page.expect_navigation(): + page.click("text=English (US)") + + page.click("text=Please login to your account or register an account for free. >> a") + page.click("input[name=\"login_id\"]") + page.fill("input[name=\"login_id\"]", "yamzhenglim@gmail.com") + page.click("input[name=\"login_pass\"]") + page.fill("input[name=\"login_pass\"]", "Abc_123456") + + with page.expect_navigation(): + page.click("button:has-text(\"Login\")") + + page.click(":nth-match(:text(\"Request Service\"), 2)") + + time.sleep(2) + page.eval_on_selector('.btn-agree', '$(".btn-agree").button( "option", "disabled", true | false )') + page.click("text=I Agree") + + for option in CONFIG['options']: # Not Implemented "ALLINK", "GH", "NH", "OB" + try: + + option_selector = page.locator("select[name=\"reload[acc_type]\"]") + option_selector.select_option(option) + + data = [] + for i in range (RANGE['start'], RANGE['stop']+1, RANGE['step']): + d = {} + myr_selector = '#reload_myr_amount' + myr_locator = page.locator(myr_selector) + myr_locator.clear() + myr_locator.fill(f'{i}') + + page.press("text=Transaction Amount", 'Tab') + time.sleep(0.5) + + myr = page.evaluate('$("#reload_myr_amount").val()') + cny = page.evaluate('$("#reload_cny_amount").val()') + + q_staff = page.query_selector('.lime') + if q_staff: + staff_status = q_staff.inner_text().strip() + else: + staff_status = 'Offline' + + orders_selector = page.query_selector('.in-processing') + if orders_selector: + orders = re.findall('\d+', orders_selector.inner_text()) + + svc = page.evaluate('$("#service_charge").val()') + total_payable = page.evaluate('$("#total_amount").val()') + current_rate = page.query_selector('#ex_rate').text_content() + # today_rate = page.query_selector('.today_rate').text_content() + # last_rate_update = page.query_selector('span:below(.today_rate)').text_content() + last_rate_update = page.query_selector('.text-muted.help-inline').text_content().strip() + + d['product_name'] = PRODUCT_NAME + d['site_name'] = SITE_NAME + d['service_type'] = option # need confirmation / add to config + d['pending_orders'] = int(orders[0]) + d['CNY'] = cny + d['MYR'] = myr + d['handeling_fee'] = svc + d['payable'] = total_payable + d['current_rate'] = current_rate + # d['day_rate'] = today_rate + d['last_rate_update'] = last_rate_update + d['staff_status'] = staff_status + d['scrape_date'] = datetime.now().strftime('%Y-%m-%d') + d['scrape_hour'] = datetime.now().strftime('%H') + + data.append(d) + + df = pd.DataFrame(data) + + except Exception as e: + print(e) + stamp = datetime.now().strftime('%Y_%m_%d') + page.screenshot(path = f"{ERROR_PATH}/ERROR_{stamp}.png") + + df.to_csv(f"{DATA_PATH}/{option}_{date}_{hour}.csv", index=False) + + except Exception as e: + print(e) + stamp = datetime.now().strftime('%Y_%m_%d') + page.screenshot(path = f"{ERROR_PATH}/ERROR_{stamp}.png") + +if __name__ == '__main__': + + # Define a dictionary + conf_dic = { + "base_url": "https://www.c2m.my/reload.php", + "product_name": "Exchange", + "site_name": "c2m", + "range":{ + "start":1, + "stop":100001, + "step":1000 + }, + "options":["TBDF", "ALTR"] + } + + # Convert the dictionary to JSON format + conf = json.dumps(conf_dic) + + CONFIG = json.loads(conf) + + DATA_PATH = "./c2m/scrape/local_run_files/" + ERROR_PATH ="./c2m/scrape/local_run_files/error/" + os.makedirs(DATA_PATH, exist_ok=True) + + with sync_playwright() as playwright: + run(playwright, DATA_PATH, ERROR_PATH, CONFIG) + +else: + def run_app(C2M_FILES_TEMP_STORAGE_PATH, C2M_CONFIG): + # dt = datetime.now() + # date = dt.strftime('%Y-%m-%d') + # hour = dt.hour + + DATA_PATH = C2M_FILES_TEMP_STORAGE_PATH + ERROR_PATH = C2M_FILES_TEMP_STORAGE_PATH + os.makedirs(DATA_PATH, exist_ok=True) + + with sync_playwright() as playwright: + run(playwright, DATA_PATH, ERROR_PATH, C2M_CONFIG) diff --git a/c2m/scrape/c2m_temp_scrape_files/.gitkeep b/c2m/scrape/c2m_temp_scrape_files/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/c2m/scrape/local_run_files/error/.gitkeep b/c2m/scrape/local_run_files/error/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/C2M_CONFIG.json b/config/C2M_CONFIG.json new file mode 100644 index 0000000..f0fc711 --- /dev/null +++ b/config/C2M_CONFIG.json @@ -0,0 +1,11 @@ +{ + "base_url": "https://www.c2m.my/reload.php", + "product_name": "Exchange", + "site_name": "c2m", + "range":{ + "start":1, + "stop":100001, + "step":1000 + }, + "options":["TBDF", "ALTR"] +} \ No newline at end of file diff --git a/general_function.py b/general_function.py index 925640f..a0dcd28 100644 --- a/general_function.py +++ b/general_function.py @@ -22,6 +22,13 @@ def delete_csv_in_path(path): os.unlink(path) print("[Delete Temp Complete]", path) +def delete_csv_or_png_in_path(path): + for folder, subfolders, files in os.walk(path): + for file in files: + if file.endswith('.csv') or file.endswith('.png'): + path = os.path.join(folder, file) + os.unlink(path) + print("[Delete Temp Complete]", path) def change_to_current_directory(): script_dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/main.py b/main.py index b977aaa..b2e16be 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,8 @@ import shipping.extract.shipping_extract_function as shipping_extract_func import shipping.load.shipping_load_function as shipping_load_func import crisp.extract.crisp_extract_function as crisp_extract_func import crisp.load.crisp_load_function as crisp_load_func - +import c2m.scrape.c2m_scrape_app as c2m_scrape_app +import c2m.load.c2m_load_function as c2m_load_func def exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH): #Config file name @@ -127,52 +128,91 @@ def crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH): file_path = os.path.join(dirpath,file_name) crisp_load_func.upload_to_bucket(cirsp_storage_client, file_name, file_path) +def c2m_scrape(C2M_FILES_TEMP_STORAGE_PATH): + C2M_CONFIG_FILE = "C2M_CONFIG.json" + C2M_FILES_TEMP_STORAGE_PATH = C2M_FILES_TEMP_STORAGE_PATH + + #Read config file + C2M_CONFIG = general_function.read_json_file("./config/",C2M_CONFIG_FILE) + + #Run the c2m_scrape_app + c2m_scrape_app.run_app(C2M_FILES_TEMP_STORAGE_PATH, C2M_CONFIG) + +def c2m_load_to_cloudstorage(C2M_FILES_TEMP_STORAGE_PATH): + #Config file name + CLOUDSTORAGE_CREDENTIAL_FILE = "./config/CLOUD_STORAGE_ADMIN_CREDENTIAL.json" + + #Connect to cloudstorage + c2m_storage_client = c2m_load_func.login_cloudstorage_credential(CLOUDSTORAGE_CREDENTIAL_FILE) + + #Get the filename and path in temp csv storage + temp_file_list = [] + for (dirpath, dirnames, filenames) in walk(C2M_FILES_TEMP_STORAGE_PATH): + temp_file_list.extend(filenames) + + #Loop all the file and upload to cloudstorage + for file_name in temp_file_list: + file_path = os.path.join(dirpath,file_name) + c2m_load_func.upload_to_bucket(c2m_storage_client, file_name, file_path) def main(): print("RUNNING MAIN_PY") - # Exchange - #Make sure cron in the file directory - general_function.change_to_current_directory() - #Extract Load Exchange Data - EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" - print("STARTING EXCHANGE EXTRACT") - exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) - print("STARTING EXCHANGE LOAD") - exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) - #Delete Temp CSV - print("DELETING EXCHANGE CSV") - general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) + # # Exchange + # #Make sure cron in the file directory + # general_function.change_to_current_directory() + # #Extract Load Exchange Data + # EXCHANGE_CSV_TEMP_STORAGE_PATH = "./exchange/extract/exchange_temp_extract_csv/" + # print("STARTING EXCHANGE EXTRACT") + # exchange_extract(EXCHANGE_CSV_TEMP_STORAGE_PATH) + # print("STARTING EXCHANGE LOAD") + # exchange_load_to_cloudstorage(EXCHANGE_CSV_TEMP_STORAGE_PATH) + # #Delete Temp CSV + # print("DELETING EXCHANGE CSV") + # general_function.delete_csv_in_path(EXCHANGE_CSV_TEMP_STORAGE_PATH) - # Shipping + # # Shipping + # #Make sure cron in the file directory + # general_function.change_to_current_directory() + # #Extract Load SHIPPING Data + # SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/" + # print("STARTING SHIPPING EXTRACT") + # shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH) + # print("STARTING SHIPPING LOAD") + # shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH) + # #Delete Temp CSV + # print("DELETING SHIPPING CSV") + # general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH) + + # C2M #Make sure cron in the file directory general_function.change_to_current_directory() - #Extract Load SHIPPING Data - SHIPPING_CSV_TEMP_STORAGE_PATH = "./shipping/extract/shipping_temp_extract_csv/" - print("STARTING SHIPPING EXTRACT") - shipping_extract(SHIPPING_CSV_TEMP_STORAGE_PATH) - print("STARTING SHIPPING LOAD") - shipping_load_to_cloudstorage(SHIPPING_CSV_TEMP_STORAGE_PATH) + #Scrape C2M Data + C2M_FILES_TEMP_STORAGE_PATH = "./c2m/scrape/c2m_temp_scrape_files/" + print("STARTING C2M SCRAPE") + c2m_scrape(C2M_FILES_TEMP_STORAGE_PATH) + print("STARTING C2M LOAD") + c2m_load_to_cloudstorage(C2M_FILES_TEMP_STORAGE_PATH) #Delete Temp CSV print("DELETING SHIPPING CSV") - general_function.delete_csv_in_path(SHIPPING_CSV_TEMP_STORAGE_PATH) + general_function.delete_csv_or_png_in_path(C2M_FILES_TEMP_STORAGE_PATH) - # CRISP - #Make sure cron in the file directory - general_function.change_to_current_directory() - #Extract Load Exchange Data - CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/" - print("STARTING CRISP EXTRACT") - crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH) - print("STARTING CRISP LOAD") - crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH) - #Delete Temp CSV - print("DELETING CRISP CSV") - general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH) + # # CRISP + # #Make sure cron in the file directory + # general_function.change_to_current_directory() + # #Extract Load Exchange Data + # CRISP_CSV_TEMP_STORAGE_PATH = "./crisp/extract/crisp_temp_extract_csv/" + # print("STARTING CRISP EXTRACT") + # crisp_extract(CRISP_CSV_TEMP_STORAGE_PATH) + # print("STARTING CRISP LOAD") + # crisp_load_to_cloudstorage(CRISP_CSV_TEMP_STORAGE_PATH) + # #Delete Temp CSV + # print("DELETING CRISP CSV") + # general_function.delete_csv_in_path(CRISP_CSV_TEMP_STORAGE_PATH) - print("END MAIN_PY") - print("------------------------------------------------------------------------------------------------") + # print("END MAIN_PY") + # print("------------------------------------------------------------------------------------------------") if True: main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9aaa1cb..e991e77 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ mysql-connector-python==8.0.29 google-cloud-storage==2.5.0 crisp-api==1.1.13 -pandas==1.5.2 \ No newline at end of file +pandas==1.5.2 +playwright==1.34.0 \ No newline at end of file