add script

push.py

+import pika
+import toml
+import logging
+
+config_toml = "/data/golang/src/europa-erp-go/manifest/config/config.toml"
+config = toml.load(config_toml)
+
+logging.basicConfig(level=logging.INFO, filename=log_path, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger("download.py")
+
+# 生产环境请修改rabbitmq连接信息
+rabbit_mq_host = config["producer"]["rabbit_mq_host"]
+rabbit_mq_queue = config["producer"]["rabbit_mq_queue"]
+rabbit_mq_exchange = config["producer"]["rabbit_mq_exchange"]
+rabbit_mq_routing_key = config["producer"]["rabbit_mq_routing_key"]
+rabbit_mq_port = config["producer"]["rabbit_mq_port"]
+rabbit_mq_user = config["producer"]["rabbit_mq_user"]
+rabbit_mq_password = config["producer"]["rabbit_mq_password"]
+
+class Producer:
+
+    def __init__(self, channel=None):
+        self.channel = channel
+
+    def get_connn(self):
+        credentials = pika.PlainCredentials(username=rabbit_mq_user, password=rabbit_mq_password)
+        conn = pika.BlockingConnection(pika.ConnectionParameters(host=rabbit_mq_host, port=rabbit_mq_port, credentials=credentials))
+        channel = conn.channel()
+        channel.queue_declare(queue=rabbit_mq_queue, durable=True)
+        self.channel = channel
+
+    def push(self, message):
+        if not self.channel:
+            logger.error("rabbitmq连接异常！")
+            self.get_connn()
+            logger.info("rabbitmq重新获取连接！")
+        self.channel.basic_publish(exchange=rabbit_mq_exchange, routing_key=rabbit_mq_routing_key,
+                                   body=str.encode(message))
\ No newline at end of file

waldom_data_process_RMB.py

+
+import json
+import logging
+import time
+
+import requests
+
+import push
+
+import pandas as pd
+
+file_name = 'Tianyang_Inventory_Feed_RMB.xlsx'
+url = "https://www.waldomchina.com/dailyFeed/Tianyang_Inventory_Feed_RMB.xlsx"
+
+logging.basicConfig(level=logging.INFO, filename='waldom_usd.log',
+                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger('waldom_data_process_RMB.py')
+
+title_map = {
+    'Manufacturer Name', 'Part Number', 'Stock EMEA', 'Stock USA', 'Stock APAC', 'Stock CN', 'MOQ',
+    'Price Break Qty 1', 'Price Break Qty 2', 'Price Break Qty 3', 'Price Break Qty 4',
+    'Price Break Qty 5', 'Price Break Qty 6', 'Price Break Qty 7', 'Price Break Qty 8',
+    'Price Break Qty 9', 'Date Code'
+}
+
+title_arr = [
+    'Manufacturer Name', 'Part Number', 'Stock EMEA', 'Stock USA', 'Stock APAC', 'Stock CN', 'MOQ',
+    'Price Break Qty 1',  'Price Break Qty 2',
+    'Price Break Qty 3',  'Price Break Qty 4',
+    'Price Break Qty 5',   'Price Break Qty 6',
+    'Price Break Qty 7',  'Price Break Qty 8',
+    'Price Break Qty 9',   'Date Code'
+]
+
+ladder_map = {
+    'Price Break Qty 1': 'Price Break 1',
+    'Price Break Qty 2': 'Price Break 2',
+    'Price Break Qty 3': 'Price Break 3',
+    'Price Break Qty 4': 'Price Break 4',
+    'Price Break Qty 5': 'Price Break 5',
+    'Price Break Qty 6': 'Price Break 6',
+    'Price Break Qty 7': 'Price Break 7',
+    'Price Break Qty 8': 'Price Break 8',
+    'Price Break Qty 9': 'Price Break 9',
+}
+
+def download_file(url=""):
+    response = requests.get(url=url)
+    with open(file_name, 'wb') as f:
+        f.write(response.content)
+    logger.info("文件 %s 下载完成" % file_name)
+
+class HandleCSV:
+    def parse(self, path):
+        result = []
+        df = pd.read_excel(path, sheet_name=0)
+        if not self.validate(df.columns.values):
+            logger.error("文件 %s 格式错误" % path)
+            return []
+        for i in range(len(df.index.values)):
+            # print(df.loc[i, title_arr[0]])
+            table = dict()
+            table['price_is_us'] = False
+            table['ladder_price'] = []
+            table['supplier_name'] = 'Waldom人民币'
+            table['goods_sn'] = ''
+            table['multiple'] = '0'
+            batch_sn_map = dict()
+            for j in range(len(title_arr)):
+                ladder_item_map = dict()
+                data = df.loc[i, title_arr[j]]
+                if title_arr[j] == 'Manufacturer Name':
+                    table['brand_name'] = data
+                elif title_arr[j] == 'Part Number':
+                    table['goods_name'] = data
+                elif title_arr[j] == 'Stock EMEA' or \
+                    title_arr[j] == 'Stock USA' or \
+                    title_arr[j] == 'Stock APAC' or \
+                    title_arr[j] == 'Stock CN':
+                    if len(str(data).strip()) > 0:
+                        if 'stock' in table:
+                            table['stock'] += int(data)
+                        else:
+                            table['stock'] = int(data)
+                elif title_arr[j] == 'MOQ':
+                    table['moq'] = str(data)
+                elif title_arr[j] in ladder_map:
+                    if len(str(data).strip()) != 0 and str(data).strip() != "nan" and str(data).strip() != "NaN":
+                        ladder_item_map['purchases'] = int(data)
+                        ladder_item_map['price_us'] = float(0)
+                        ladder_item_map['price_cn'] = float(df.loc[i, ladder_map[title_arr[j]]])
+                        table['ladder_price'].append(ladder_item_map)
+                elif title_arr[j] == 'Date Code':
+                    item_arr = str(data).split(sep=";")
+                    for k in range(len(item_arr)):
+                        item = item_arr[k]
+                        split_item_arr = item.split(sep="|")
+                        if len(split_item_arr) == 3:
+                            number = split_item_arr[0].strip()
+                            year = split_item_arr[1].strip()
+                            if len(number) > 0 and len(year) >= 4 and number.isdigit() and str(year[len(year)-4:]).isdigit():
+                                key = str(year[len(year)-2:]) + "+"
+                                value = number
+                                if key in batch_sn_map:
+                                    batch_sn_map[key] += int(value)
+                                else:
+                                    batch_sn_map[key] = int(value)
+            table['batch_sn'] = batch_sn_map
+            json_str = json.dumps(table, ensure_ascii=False)
+            result.append(json_str)
+            print(json_str)
+        return result
+
+
+
+    def validate(self, arr: list) -> bool:
+        size = len(title_map)
+        count = 0
+        for i in range(len(arr)):
+            if arr[i] in title_map:
+                count += 1
+        return count == size
+
+
+
+if __name__ == '__main__':
+    start = time.time()
+    # 首先下载所需的文件
+    download_file(url)
+    time.sleep(1)
+    # 处理成为json字符串数组
+    res = HandleCSV().parse(file_name)
+    # 推送mq队列
+    end = time.time()
+    logger.info("spend time: " + str(end - start))
+
+
+
+
+
+

waldom_data_process_USD.py

+import csv
+import json
+import logging
+import time
+
+logging.basicConfig(level=logging.INFO, filename='waldom_dollar.log', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger('waldom_data_process_USD.py')
+
+# Tianyang_InventoryFeed title字段含义
+# Manufacturer  ----------------> brand_name
+#  库存相加  HIJK 列
+#  加一个 multiple 列
+# TODO 缺少goods_sn  推送不要一次性推送 要一条一条推送
+
+title_map = {'Part Number', 'Manufacturer', 'Stock EMEA', 'Stock USA', 'Stock APAC', 'Stock CN', 'MOQ', 'Multiplier',
+             'Price Break Qty 1', 'Price Break Qty 2', 'Price Break Qty 3',
+             'Price Break Qty 4', 'Price Break Qty 5', 'Price Break Qty 6',
+             'Price Break Qty 7', 'Price Break Qty 8', 'Price Break Qty 9',
+             'Date Code'
+             }
+ladder_map = {
+            'Price Break Qty 1',  'Price Break Qty 2',  'Price Break Qty 3',
+            'Price Break Qty 4', 'Price Break Qty 5', 'Price Break Qty 6',
+             'Price Break Qty 7', 'Price Break Qty 8', 'Price Break Qty 9',
+}
+class HandleCSV:
+    def parse(self, path):
+        with open(path, encoding='gbk', errors='ignore') as f:
+            result = []
+            first_line = []
+            reader = csv.reader(f)
+            for index, row in enumerate(reader):
+                if index == 0:
+                    first_line = row
+                    flag = self.validate(first_line)
+                    if not flag:
+                        logger.error("文件 %s 格式错误" % path)
+                        return []
+                else:
+                    json_str = self.generate_json(row, first_line)
+                    print(json_str)
+                    result.append(json_str)
+        logger.info("文件 %s 已解析" % path)
+        return result
+
+    # 校验excel表头符合既定格式
+    def validate(self, arr: list) -> bool:
+        size = len(title_map)
+        count = 0
+        for i in range(len(arr)):
+            if arr[i] in title_map:
+                count += 1
+        return count == size
+
+    def generate_json(self, arr: list, title: list) ->str:
+        table = dict()
+        table['price_is_us'] = True
+        table['ladder_price'] = []
+        table['supplier_name'] = 'Waldom美金'
+        table['goods_sn'] = ''
+        batch_sn_map = dict()
+        for i in range(len(title)):
+            ladder_item_map = dict()
+            if title[i] in title_map:
+                if title[i] == 'Part Number':
+                    table['goods_name'] = arr[i]
+                elif title[i] == 'Manufacturer':
+                    table['brand_name'] = arr[i]
+                elif title[i] == 'Stock EMEA' or \
+                    title[i] == 'Stock USA' or \
+                    title[i] == 'Stock APAC' or \
+                    title[i] == 'Stock CN':
+                    if len(str(arr[i])) != 0:
+                        if 'stock' not in table:
+                            table['stock'] = int(arr[i])
+                        else:
+                            table['stock'] += int(arr[i])
+                elif title[i] == 'MOQ':
+                    table['moq'] = arr[i]
+                elif title[i] == 'Multiplier':
+                    table['multiple'] = arr[i]
+                elif title[i] in ladder_map:
+                    if len(str(arr[i]).strip()) != 0:
+                        ladder_item_map['purchases'] = int(arr[i])
+                        ladder_item_map['price_us'] = float(arr[i+1])
+                        ladder_item_map['price_cn'] = float(0)
+                        table['ladder_price'].append(ladder_item_map)
+                elif title[i] == 'Date Code':
+                    items_arr = str(arr[i]).split(sep=";")
+                    for j in range(len(items_arr)):
+                        item = items_arr[j]
+                        split_item_arr = item.split(sep="|")
+                        if len(split_item_arr) == 3:
+                            number = split_item_arr[0].strip()
+                            year = split_item_arr[1].strip()
+                            if len(number) > 0 and len(year) >= 4 and number.isdigit() and str(year[len(year)-4:]).isdigit():
+                                key = str(year[len(year)-2:]) + "+"
+                                value = number
+                                if key in batch_sn_map:
+                                    batch_sn_map[key] += int(value)
+                                else:
+                                    batch_sn_map[key] = int(value)
+        table['batch_sn'] = batch_sn_map
+        return json.dumps(table, ensure_ascii=False)
+
+if __name__ == '__main__':
+    result = HandleCSV().parse('Tianyang_InventoryFeed.csv')
+    print(result)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+