new script

element_14_data_process.py

+import json
 import logging
 import os.path
 import time
 import zipfile
 
-# import push
+import push
 import requests
 
 url = "http://pfd.premierfarnell.com/eCat_element14_CN_BI.zip"
-file_name = "eCat_element14_CN_BI.zip"
-unzip_name = "eCat_element14_CN_BI.txt"
+file_name = "/data/golang/src/europa-erp-go/scripts/eCat_element14_CN_BI.zip"
+unzip_name = "/data/golang/src/europa-erp-go/scripts/eCat_element14_CN_BI.txt"
 
 # 解压缩的文件保存目录
 save_path = "/data/golang/src/europa-erp-go/scripts/"
@@ -16,9 +17,21 @@ save_path = "/data/golang/src/europa-erp-go/scripts/"
 logger = logging.getLogger("element_14_data_process.py")
 
 title_map = {
-    "Brand", ""
+    "Manufacturer Name", "Element14 Order Code", "Manufacturer Part Number",
+    "Quantity_Break_1", "Quantity_Break2", "Quantity_Break3", "Quantity_Break4",
+    "Quantity_Break5", "Quantity_Break6", "Quantity_Break7", "Quantity_Break8",
+    "Quantity_Break9", "Quantity_Break10", "Total Stock available to country",
+    "US ECCN", "MinimumOrderQuantity", "MultipleOrderQuantity",
 }
 
+ladder_price_map = {
+    "Quantity_Break_1", "Quantity_Break2", "Quantity_Break3", "Quantity_Break4",
+    "Quantity_Break5", "Quantity_Break6", "Quantity_Break7", "Quantity_Break8",
+    "Quantity_Break9", "Quantity_Break10",
+}
+
+producer = push.Producer()
+
 def download_zip_file(url=""):
     start = time.time()
     response = requests.get(url=url)
@@ -37,16 +50,54 @@ def unzip_file(file=""):
 
 class HandleTxT:
     def parse(self, path:str):
-        with open(path, encoding='gbk', errors='ignore') as file_object:
+        with open(path, encoding='utf-8', errors='ignore') as file_object:
             lines = file_object.readlines()
         data_string = []
         # 去除行末的换行符
         for line in lines:
             data_line = line.strip("\n")
             data_string.append(data_line)
-        arr = data_string[0].split("|")
-        for i in range(len(arr)):
-            print(arr[i])
+        first_row = data_string[0].split("|")
+        count_msg = 0
+        for index in range(len(data_string)):
+            if index == 0:
+                continue
+            if index > 10:
+                break
+            row = data_string[index].split("|")
+            table = dict()
+            table['price_is_us'] = False
+            table['ladder_price'] = []
+            table['supplier_name'] = 'Element14'
+            table['batch_sn'] = dict()
+            for j in range(len(first_row)):
+                if first_row[j] in title_map:
+                    if first_row[j] == "Manufacturer Name":
+                        table['brand_name'] = row[j]
+                    if first_row[j] == "Total Stock available to country":
+                        table['stock'] = str(row[j])
+                    if first_row[j] == "MinimumOrderQuantity":
+                        table['moq'] = str(row[j])
+                    if first_row[j] == "MultipleOrderQuantity":
+                        table['multiple'] = str(row[j])
+                    if first_row[j] == "Manufacturer Part Number":
+                        table['goods_name'] = str(row[j])
+                    if first_row[j] == "US ECCN":
+                        table['eccn'] = str(row[j])
+                    if first_row[j] == "Element14 Order Code":
+                        table['goods_sn'] = str(row[j])
+                    if first_row[j] in ladder_price_map:
+                        if len(str(row[j]).strip()) != 0:
+                            if int(str(row[j]).strip()) != 0:
+                                ladder_item = dict()
+                                ladder_item['purchases'] = int(str(row[j]).strip())
+                                ladder_item['price_us'] = float(0)
+                                ladder_item['price_cn'] = float(str(row[j+1]).strip())
+                                table['ladder_price'].append(ladder_item)
+            json_str = json.dumps(table)
+            producer.push(json_str)
+            count_msg += 1
+            print(json_str)
 
 
 
@@ -54,10 +105,18 @@ class HandleTxT:
 
 
 if __name__ == '__main__':
-    # unzip_file(file_name)
+    start = time.time()
+    # 下载压缩文件
+    download_zip_file(url)
+    # 解压缩
+    unzip_file(file_name)
+    # 获取rabbit_mq连接
+    producer.get_connn()
     if not os.path.exists(unzip_name):
         logger.error("文件 %s 不存在" % unzip_name)
         exit(1)
+    # 解析并推送
     HandleTxT().parse(unzip_name)
-
+    end = time.time()
+    logger.info("handle file: eCat_element14_CN_BI.txt total spend time: " + str(end - start))