add config

e84b5900 · 岳巧源 · c62ff215 · e84b5900 · e84b5900 · e84b5900
Commit e84b5900 authored Jul 31, 2024 by 岳巧源
Showing with 66 additions and 20 deletions
push.py
waldom_data_process_RMB.py
waldom_data_process_USD.py
--- a/push.py
+++ b/push.py
@@ -2,11 +2,13 @@ import pika
 import toml
 import logging
+# 正式环境配置文件位置
 config_toml = "/data/golang/src/europa-erp-go/manifest/config/config.toml"
 config = toml.load(config_toml)
-logging.basicConfig(level=logging.INFO, filename=log_path, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, filename='result.log', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-logger = logging.getLogger("download.py")
+logger = logging.getLogger("push.py")
 # 生产环境请修改rabbitmq连接信息
 rabbit_mq_host = config["producer"]["rabbit_mq_host"]
@@ -35,4 +37,8 @@ class Producer:
            self.get_connn()
            logger.info("rabbitmq重新获取连接！")
        self.channel.basic_publish(exchange=rabbit_mq_exchange, routing_key=rabbit_mq_routing_key,
                                   body=str.encode(message))
\ No newline at end of file
+    def close(self):
+        self.channel.close()
+        self.channel = None
\ No newline at end of file
--- a/waldom_data_process_RMB.py
+++ b/waldom_data_process_RMB.py
@@ -6,13 +6,13 @@ import time
 import requests
 import push
+import os
 import pandas as pd
 file_name = 'Tianyang_Inventory_Feed_RMB.xlsx'
 url = "https://www.waldomchina.com/dailyFeed/Tianyang_Inventory_Feed_RMB.xlsx"
-logging.basicConfig(level=logging.INFO, filename='waldom_usd.log',
+logging.basicConfig(level=logging.INFO, filename='waldom_rmb.log',
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger('waldom_data_process_RMB.py')
@@ -44,15 +44,21 @@ ladder_map = {
    'Price Break Qty 9': 'Price Break 9',
 }
+producer = push.Producer()
+producer.get_connn()
 def download_file(url=""):
+    start = time.time()
    response = requests.get(url=url)
    with open(file_name, 'wb') as f:
        f.write(response.content)
-    logger.info("文件 %s 下载完成" % file_name)
+    end = time.time()
+    logger.info("文件 %s 下载完成, 用时 %s" % (file_name, str(end - start)))
 class HandleCSV:
    def parse(self, path):
-        result = []
        df = pd.read_excel(path, sheet_name=0)
        if not self.validate(df.columns.values):
            logger.error("文件 %s 格式错误" % path)
@@ -107,9 +113,9 @@ class HandleCSV:
                                    batch_sn_map[key] = int(value)
            table['batch_sn'] = batch_sn_map
            json_str = json.dumps(table, ensure_ascii=False)
-            result.append(json_str)
+            self.push_to_mq(json_str)
            print(json_str)
-        return result
@@ -121,6 +127,9 @@ class HandleCSV:
                count += 1
        return count == size
+    def push_to_mq(self, msg:str):
+        producer.push(msg)
 if __name__ == '__main__':
@@ -128,11 +137,14 @@ if __name__ == '__main__':
    # 首先下载所需的文件
    download_file(url)
    time.sleep(1)
-    # 处理成为json字符串数组
+    # 处理并推送json字符串数组
-    res = HandleCSV().parse(file_name)
+    HandleCSV().parse(file_name)
-    # 推送mq队列
+    time.sleep(1)
+    producer.close()
+    # 清理下载的文件
+    os.remove(file_name)
    end = time.time()
-    logger.info("spend time: " + str(end - start))
+    logger.info("rmb file total spend time: " + str(end - start))

--- a/waldom_data_process_USD.py
+++ b/waldom_data_process_USD.py
 import csv
 import json
 import logging
+import os
 import time
+import requests
+import push
-logging.basicConfig(level=logging.INFO, filename='waldom_dollar.log', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger('waldom_data_process_USD.py')
 # Tianyang_InventoryFeed title字段含义
@@ -23,10 +26,24 @@ ladder_map = {
            'Price Break Qty 4', 'Price Break Qty 5', 'Price Break Qty 6',
             'Price Break Qty 7', 'Price Break Qty 8', 'Price Break Qty 9',
 }
+file_name = "Tianyang_InventoryFeed.csv"
+url = "https://www.waldomapac.com/dailyFeed/Tianyang_InventoryFeed.csv"
+producer = push.Producer()
+producer.get_connn()
+def download_file(url=""):
+    start = time.time()
+    response = requests.get(url=url)
+    with open(file_name, 'wb') as f:
+        f.write(response.content)
+    end = time.time()
+    logger.info("文件 %s 下载完成, 用时 %s" % (file_name, str(end - start)))
 class HandleCSV:
    def parse(self, path):
        with open(path, encoding='gbk', errors='ignore') as f:
-            result = []
            first_line = []
            reader = csv.reader(f)
            for index, row in enumerate(reader):
@@ -38,10 +55,10 @@ class HandleCSV:
                        return []
                else:
                    json_str = self.generate_json(row, first_line)
+                    self.push_to_mq(json_str)
                    print(json_str)
-                    result.append(json_str)
        logger.info("文件 %s 已解析" % path)
-        return result
    # 校验excel表头符合既定格式
    def validate(self, arr: list) -> bool:
@@ -103,9 +120,20 @@ class HandleCSV:
        table['batch_sn'] = batch_sn_map
        return json.dumps(table, ensure_ascii=False)
+    def push_to_mq(self, msg: str):
+        producer.push(msg)
 if __name__ == '__main__':
-    result = HandleCSV().parse('Tianyang_InventoryFeed.csv')
+    start = time.time()
-    print(result)
+    download_file(url)
+    time.sleep(1)
+    HandleCSV().parse(file_name)
+    time.sleep(1)
+    producer.close()
+    os.remove(file_name)
+    end = time.time()
+    logger.info("usd file total spend time: " + str(end - start))