Skip to content

岳巧源 / europa-erp-python

Go to a project
Commit e0e96851 by 刘豪
Options

添加日志文件

parent 5d7e1fec
Inline Side-by-side
Showing with 57 additions and 7 deletions
rs_data.py
...@@ -8,6 +8,7 @@ import config ...@@ -8,6 +8,7 @@ import config
from utils import ftp from utils import ftp
from utils import file_handle from utils import file_handle
from utils import rabbit_mq from utils import rabbit_mq
from utils import log_manage
...@@ -24,10 +25,11 @@ handle_file_name_1 = "rs_components_HK-cn_chiplink.csv" ...@@ -24,10 +25,11 @@ handle_file_name_1 = "rs_components_HK-cn_chiplink.csv"
handle_file_name_3 = "rs_components_cn_chiplink.csv" handle_file_name_3 = "rs_components_cn_chiplink.csv"
conf = config.load_config_map() conf = config.load_config_map()
logger = log_manage.get_logger("rs_data")
def handle_hk_csv(file_name, conf: dict, supplier_id: int): def handle_hk_csv(file_name, conf: dict, supplier_id: int):
logger.info("处理香港csv")
"""handle the HK_cn file""" """handle the HK_cn file"""
p = rabbit_mq.Producer( p = rabbit_mq.Producer(
user=conf["rabbit_mq"]["producer"]["user"], user=conf["rabbit_mq"]["producer"]["user"],
...@@ -98,6 +100,7 @@ def handle_hk_csv(file_name, conf: dict, supplier_id: int): ...@@ -98,6 +100,7 @@ def handle_hk_csv(file_name, conf: dict, supplier_id: int):
def handle_cn_csv(file_name, conf: dict, supplier_id: int): def handle_cn_csv(file_name, conf: dict, supplier_id: int):
logger.info("处理国内csv")
"""handle the RMB file""" """handle the RMB file"""
p = rabbit_mq.Producer( p = rabbit_mq.Producer(
user=conf["rabbit_mq"]["producer"]["user"], user=conf["rabbit_mq"]["producer"]["user"],
...@@ -203,11 +206,12 @@ if __name__ == '__main__': ...@@ -203,11 +206,12 @@ if __name__ == '__main__':
port=conf_environment["ftp"]["port"], port=conf_environment["ftp"]["port"],
passive=True, passive=True,
username=conf_environment["ftp"]["username"], username=conf_environment["ftp"]["username"],
password=conf_environment["ftp"]["password"] password=conf_environment["ftp"]["password"],
) logger = logger)
if not os.path.exists(conf_environment["tmp_file_path"]): if not os.path.exists(conf_environment["tmp_file_path"]):
os.makedirs(conf_environment["tmp_file_path"]) os.makedirs(conf_environment["tmp_file_path"])
"""download the zip file""" """download the zip file"""
logger.info("下载压缩包")
ftp_client.download(remote_file_name_1, conf_environment["tmp_file_path"] + local_file_name_1) ftp_client.download(remote_file_name_1, conf_environment["tmp_file_path"] + local_file_name_1)
ftp_client.download(remote_file_name_3, conf_environment["tmp_file_path"] + local_file_name_3) ftp_client.download(remote_file_name_3, conf_environment["tmp_file_path"] + local_file_name_3)
...@@ -216,7 +220,9 @@ if __name__ == '__main__': ...@@ -216,7 +220,9 @@ if __name__ == '__main__':
os.makedirs(extract_dir) os.makedirs(extract_dir)
"""unzip the zip file""" """unzip the zip file"""
try: try:
logger.info("解压压缩包")
file_handle.unzip(conf_environment["tmp_file_path"] + local_file_name_1, extract_dir) file_handle.unzip(conf_environment["tmp_file_path"] + local_file_name_1, extract_dir)
file_handle.unzip(conf_environment["tmp_file_path"] + local_file_name_3, extract_dir) file_handle.unzip(conf_environment["tmp_file_path"] + local_file_name_3, extract_dir)
...@@ -225,16 +231,19 @@ if __name__ == '__main__': ...@@ -225,16 +231,19 @@ if __name__ == '__main__':
rs_hk_supplier_id = get_supplier_id(supplier_name='Rs', conf=conf_environment) rs_hk_supplier_id = get_supplier_id(supplier_name='Rs', conf=conf_environment)
rs_cn_supplier_id = get_supplier_id(supplier_name='RS-CN', conf=conf_environment) rs_cn_supplier_id = get_supplier_id(supplier_name='RS-CN', conf=conf_environment)
if rs_cn_supplier_id == -1 or rs_hk_supplier_id == -1: if rs_cn_supplier_id == -1 or rs_hk_supplier_id == -1:
logger.info("不支持的supplier_id")
sys.exit("can't query the correct supplier id.") sys.exit("can't query the correct supplier id.")
handle_hk_csv(extract_dir + handle_file_name_1, conf_environment, rs_hk_supplier_id) handle_hk_csv(extract_dir + handle_file_name_1, conf_environment, rs_hk_supplier_id)
handle_cn_csv(extract_dir + handle_file_name_3, conf_environment, rs_cn_supplier_id) handle_cn_csv(extract_dir + handle_file_name_3, conf_environment, rs_cn_supplier_id)
except Exception as e: except Exception as e:
print(e) logger.error("报错：{}".format(e))
"""remove the temp file, close the connection""" """remove the temp file, close the connection"""
logger.info("删除压缩包")
remove(extract_dir + handle_file_name_1) remove(extract_dir + handle_file_name_1)
remove(extract_dir + handle_file_name_3) remove(extract_dir + handle_file_name_3)
remove(conf_environment["tmp_file_path"] + local_file_name_1) remove(conf_environment["tmp_file_path"] + local_file_name_1)
remove(conf_environment["tmp_file_path"] + local_file_name_3) remove(conf_environment["tmp_file_path"] + local_file_name_3)
logger.info("运行完成")
......
utils/ftp.py
...@@ -11,12 +11,14 @@ class FTPUtil: ...@@ -11,12 +11,14 @@ class FTPUtil:
port=21, port=21,
passive=True, passive=True,
username="", username="",
password=""): password="",
logger):
ftp = ftplib.FTP() ftp = ftplib.FTP()
ftp.connect(host, port) ftp.connect(host, port)
ftp.login(username, password) ftp.login(username, password)
ftp.set_pasv(passive) ftp.set_pasv(passive)
self.ftp = ftp self.ftp = ftp
self.logger = logger
def get_connection(self): def get_connection(self):
return self.ftp return self.ftp
...@@ -42,9 +44,9 @@ class FTPUtil: ...@@ -42,9 +44,9 @@ class FTPUtil:
# 开始下载文件 # 开始下载文件
self.ftp.retrbinary('RETR {}'.format(remote_file), callback) self.ftp.retrbinary('RETR {}'.format(remote_file), callback)
print(f'{local_file} 文件下载完成') self.logger.info(f'{local_file} 文件下载完成')
except Exception as e: except Exception as e:
print(f'下载 {local_file} 过程中出现错误: {e}') self.logger.error(f'下载 {local_file} 过程中出现错误: {e}')
# def download(self, remote_path, local_path): # def download(self, remote_path, local_path):
# """ # """
......
utils/log_manage.py 0 → 100644
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import logging
import os
from logging.handlers import RotatingFileHandler
loggerLevel = logging.INFO
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
log_dir = root_path + '/logs'
console_formatter = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s'
json_formatter = '%(message)s'
logger_dict = dict()
def create_logger(log_name, log_type):
g_logger = logging.getLogger(log_name)
rt = log_name.split('_')[0]
log_path = "%s/%s" % (log_dir, rt)
if not os.path.exists(log_path):
os.makedirs(log_path)
logfile = log_name + ".log"
log_file = "%s/%s" % (log_path, logfile)
console = logging.StreamHandler()
console.setFormatter(logging.Formatter(console_formatter))
handler = RotatingFileHandler(log_file, maxBytes=2 * 1204 * 1024, backupCount=1)
fmt = json_formatter if log_type == 'json' else console_formatter
handler.setFormatter(logging.Formatter(fmt))
g_logger.addHandler(console)
g_logger.addHandler(handler)
g_logger.setLevel(loggerLevel)
return g_logger
def get_logger(log_name, log_type='file'):
if log_name not in logger_dict:
create_logger(log_name, log_type)
logger_dict[log_name] = logging.getLogger(log_name)
return logging.getLogger(log_name)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment