add config

push.py

@@ -2,11 +2,13 @@ import pika
 import toml
 import logging
 
+# 正式环境配置文件位置
+
 config_toml = "/data/golang/src/europa-erp-go/manifest/config/config.toml"
 config = toml.load(config_toml)
 
-logging.basicConfig(level=logging.INFO, filename=log_path, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-logger = logging.getLogger("download.py")
+logging.basicConfig(level=logging.INFO, filename='result.log', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger("push.py")
 
 # 生产环境请修改rabbitmq连接信息
 rabbit_mq_host = config["producer"]["rabbit_mq_host"]
@@ -35,4 +37,8 @@ class Producer:
             self.get_connn()
             logger.info("rabbitmq重新获取连接！")
         self.channel.basic_publish(exchange=rabbit_mq_exchange, routing_key=rabbit_mq_routing_key,
-                                   body=str.encode(message))
\ No newline at end of file
+                                   body=str.encode(message))
+
+    def close(self):
+        self.channel.close()
+        self.channel = None
\ No newline at end of file

waldom_data_process_RMB.py

@@ -6,13 +6,13 @@ import time
 import requests
 
 import push
-
+import os
 import pandas as pd
 
 file_name = 'Tianyang_Inventory_Feed_RMB.xlsx'
 url = "https://www.waldomchina.com/dailyFeed/Tianyang_Inventory_Feed_RMB.xlsx"
 
-logging.basicConfig(level=logging.INFO, filename='waldom_usd.log',
+logging.basicConfig(level=logging.INFO, filename='waldom_rmb.log',
                     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger('waldom_data_process_RMB.py')
 
@@ -44,15 +44,21 @@ ladder_map = {
     'Price Break Qty 9': 'Price Break 9',
 }
 
+producer = push.Producer()
+producer.get_connn()
+
+
+
 def download_file(url=""):
+    start = time.time()
     response = requests.get(url=url)
     with open(file_name, 'wb') as f:
         f.write(response.content)
-    logger.info("文件 %s 下载完成" % file_name)
+    end = time.time()
+    logger.info("文件 %s 下载完成, 用时 %s" % (file_name, str(end - start)))
 
 class HandleCSV:
     def parse(self, path):
-        result = []
         df = pd.read_excel(path, sheet_name=0)
         if not self.validate(df.columns.values):
             logger.error("文件 %s 格式错误" % path)
@@ -107,9 +113,9 @@ class HandleCSV:
                                     batch_sn_map[key] = int(value)
             table['batch_sn'] = batch_sn_map
             json_str = json.dumps(table, ensure_ascii=False)
-            result.append(json_str)
+            self.push_to_mq(json_str)
             print(json_str)
-        return result
+
 
 
 
@@ -121,6 +127,9 @@ class HandleCSV:
                 count += 1
         return count == size
 
+    def push_to_mq(self, msg:str):
+        producer.push(msg)
+
 
 
 if __name__ == '__main__':
@@ -128,11 +137,14 @@ if __name__ == '__main__':
     # 首先下载所需的文件
     download_file(url)
     time.sleep(1)
-    # 处理成为json字符串数组
-    res = HandleCSV().parse(file_name)
-    # 推送mq队列
+    # 处理并推送json字符串数组
+    HandleCSV().parse(file_name)
+    time.sleep(1)
+    producer.close()
+    # 清理下载的文件
+    os.remove(file_name)
     end = time.time()
-    logger.info("spend time: " + str(end - start))
+    logger.info("rmb file total spend time: " + str(end - start))
 
 
 

waldom_data_process_USD.py

 import csv
 import json
 import logging
+import os
 import time
+import requests
+import push
+
 
-logging.basicConfig(level=logging.INFO, filename='waldom_dollar.log', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger('waldom_data_process_USD.py')
 
 # Tianyang_InventoryFeed title字段含义
@@ -23,10 +26,24 @@ ladder_map = {
             'Price Break Qty 4', 'Price Break Qty 5', 'Price Break Qty 6',
              'Price Break Qty 7', 'Price Break Qty 8', 'Price Break Qty 9',
 }
+
+file_name = "Tianyang_InventoryFeed.csv"
+url = "https://www.waldomapac.com/dailyFeed/Tianyang_InventoryFeed.csv"
+producer = push.Producer()
+producer.get_connn()
+
+
+def download_file(url=""):
+    start = time.time()
+    response = requests.get(url=url)
+    with open(file_name, 'wb') as f:
+        f.write(response.content)
+    end = time.time()
+    logger.info("文件 %s 下载完成, 用时 %s" % (file_name, str(end - start)))
+
 class HandleCSV:
     def parse(self, path):
         with open(path, encoding='gbk', errors='ignore') as f:
-            result = []
             first_line = []
             reader = csv.reader(f)
             for index, row in enumerate(reader):
@@ -38,10 +55,10 @@ class HandleCSV:
                         return []
                 else:
                     json_str = self.generate_json(row, first_line)
+                    self.push_to_mq(json_str)
                     print(json_str)
-                    result.append(json_str)
         logger.info("文件 %s 已解析" % path)
-        return result
+
 
     # 校验excel表头符合既定格式
     def validate(self, arr: list) -> bool:
@@ -103,9 +120,20 @@ class HandleCSV:
         table['batch_sn'] = batch_sn_map
         return json.dumps(table, ensure_ascii=False)
 
+    def push_to_mq(self, msg: str):
+        producer.push(msg)
+
 if __name__ == '__main__':
-    result = HandleCSV().parse('Tianyang_InventoryFeed.csv')
-    print(result)
+    start = time.time()
+    download_file(url)
+    time.sleep(1)
+    HandleCSV().parse(file_name)
+    time.sleep(1)
+    producer.close()
+    os.remove(file_name)
+    end = time.time()
+    logger.info("usd file total spend time: " + str(end - start))
+