添加日志文件

rs_data.py

@@ -8,6 +8,7 @@ import config
 from utils import ftp
 from utils import file_handle
 from utils import rabbit_mq
+from utils import log_manage
 
 
 
@@ -24,10 +25,11 @@ handle_file_name_1 = "rs_components_HK-cn_chiplink.csv"
 handle_file_name_3 = "rs_components_cn_chiplink.csv"
 
 conf = config.load_config_map()
-
+logger = log_manage.get_logger("rs_data")
 
 
 def handle_hk_csv(file_name, conf: dict, supplier_id: int):
+    logger.info("处理香港csv")
     """handle the HK_cn file"""
     p = rabbit_mq.Producer(
         user=conf["rabbit_mq"]["producer"]["user"],
@@ -98,6 +100,7 @@ def handle_hk_csv(file_name, conf: dict, supplier_id: int):
 
 
 def handle_cn_csv(file_name, conf: dict, supplier_id: int):
+    logger.info("处理国内csv")
     """handle the RMB file"""
     p = rabbit_mq.Producer(
         user=conf["rabbit_mq"]["producer"]["user"],
@@ -203,11 +206,12 @@ if __name__ == '__main__':
                              port=conf_environment["ftp"]["port"],
                              passive=True,
                              username=conf_environment["ftp"]["username"],
-                             password=conf_environment["ftp"]["password"]
-                             )
+                             password=conf_environment["ftp"]["password"],
+                             logger = logger)
     if not os.path.exists(conf_environment["tmp_file_path"]):
         os.makedirs(conf_environment["tmp_file_path"])
     """download the zip file"""
+    logger.info("下载压缩包")
     ftp_client.download(remote_file_name_1, conf_environment["tmp_file_path"] + local_file_name_1)
 
     ftp_client.download(remote_file_name_3, conf_environment["tmp_file_path"] + local_file_name_3)
@@ -216,7 +220,9 @@ if __name__ == '__main__':
         os.makedirs(extract_dir)
 
     """unzip the zip file"""
+
     try:
+        logger.info("解压压缩包")
         file_handle.unzip(conf_environment["tmp_file_path"] + local_file_name_1, extract_dir)
 
         file_handle.unzip(conf_environment["tmp_file_path"] + local_file_name_3, extract_dir)
@@ -225,16 +231,19 @@ if __name__ == '__main__':
         rs_hk_supplier_id = get_supplier_id(supplier_name='Rs', conf=conf_environment)
         rs_cn_supplier_id = get_supplier_id(supplier_name='RS-CN', conf=conf_environment)
         if rs_cn_supplier_id == -1 or rs_hk_supplier_id == -1:
+            logger.info("不支持的supplier_id")
             sys.exit("can't query the correct supplier id.")
         handle_hk_csv(extract_dir + handle_file_name_1, conf_environment, rs_hk_supplier_id)
         handle_cn_csv(extract_dir + handle_file_name_3, conf_environment, rs_cn_supplier_id)
     except Exception as e:
-        print(e)
+        logger.error("报错：{}".format(e))
     """remove the temp file, close the connection"""
+    logger.info("删除压缩包")
     remove(extract_dir + handle_file_name_1)
     remove(extract_dir + handle_file_name_3)
     remove(conf_environment["tmp_file_path"] + local_file_name_1)
     remove(conf_environment["tmp_file_path"] + local_file_name_3)
+    logger.info("运行完成")
 
 
 

utils/ftp.py

@@ -11,12 +11,14 @@ class FTPUtil:
             port=21,
             passive=True,
             username="",
-            password=""):
+            password="",
+            logger):
         ftp = ftplib.FTP()
         ftp.connect(host, port)
         ftp.login(username, password)
         ftp.set_pasv(passive)
         self.ftp = ftp
+        self.logger = logger
 
     def get_connection(self):
         return self.ftp
@@ -42,9 +44,9 @@ class FTPUtil:
                 # 开始下载文件
                 self.ftp.retrbinary('RETR {}'.format(remote_file), callback)
 
-            print(f'{local_file} 文件下载完成')
+            self.logger.info(f'{local_file} 文件下载完成')
         except Exception as e:
-            print(f'下载 {local_file} 过程中出现错误: {e}')
+            self.logger.error(f'下载 {local_file} 过程中出现错误: {e}')
 
     # def download(self, remote_path, local_path):
     #     """

utils/log_manage.py

+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+import logging
+import os
+from logging.handlers import RotatingFileHandler
+
+loggerLevel = logging.INFO
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+log_dir = root_path + '/logs'
+console_formatter = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s'
+json_formatter = '%(message)s'
+logger_dict = dict()
+
+
+def create_logger(log_name, log_type):
+    g_logger = logging.getLogger(log_name)
+    rt = log_name.split('_')[0]
+    log_path = "%s/%s" % (log_dir, rt)
+    if not os.path.exists(log_path):
+        os.makedirs(log_path)
+    logfile = log_name + ".log"
+    log_file = "%s/%s" % (log_path, logfile)
+    console = logging.StreamHandler()
+    console.setFormatter(logging.Formatter(console_formatter))
+    handler = RotatingFileHandler(log_file, maxBytes=2 * 1204 * 1024, backupCount=1)
+    fmt = json_formatter if log_type == 'json' else console_formatter
+    handler.setFormatter(logging.Formatter(fmt))
+    g_logger.addHandler(console)
+    g_logger.addHandler(handler)
+    g_logger.setLevel(loggerLevel)
+    return g_logger
+
+
+def get_logger(log_name, log_type='file'):
+    if log_name not in logger_dict:
+        create_logger(log_name, log_type)
+        logger_dict[log_name] = logging.getLogger(log_name)
+    return logging.getLogger(log_name)