Skip to content

lichenggang / bom_identify

Go to a project
Commit b9ac888b by lichenggang
Options

添加异常处理,添加文本识别

parent 00d16687
Inline Side-by-side
Showing with 238 additions and 82 deletions
classify_server.py
......@@ -5,8 +5,8 @@ import grpc
from utils.log_manager import bom_log
from protobuf import classify_pb2
from protobuf import classify_pb2_grpc
import traceback
from predic_fac import PredictorFac
from utils.status import code2msg
class Classify(classify_pb2_grpc.classifyServicer):
......@@ -16,32 +16,17 @@ class Classify(classify_pb2_grpc.classifyServicer):
def Classify(self, request, context):
bom_log.info(f'grpc收到数据: {request.keyword}')
err_msg = None
try:
dic_data = json.loads(request.keyword)
res = self.predictorfac.predict(dic_data)
res['status'] = 1
if '数量' not in str(res):
err_msg = '未识别到数量列'
if '参数' not in str(res) and '型号' not in str(res):
err_msg = '未识别到参数列'
code, res = self.predictorfac.predict(dic_data)
except json.decoder.JSONDecodeError:
res = self.predictorfac.predict(request.keyword)
res['status'] = 1
except:
res = {
'status': 0,
'err_msg': '未知'
}
bom_log.error(f'grpc报错: \n' + traceback.format_exc())
if err_msg:
res = {
'status': 0,
'err_msg': err_msg
}
bom_log.info(f'返回结果: ' + str(res))
return classify_pb2.ClassifyReply(message='result {msg}'.format(msg=res))
code, res = self.predictorfac.predict(request.keyword)
if code != 1:
res = code2msg(code)
bom_log.error(f'grpc识别报错: ' + res['message'])
else:
res['status'] = code
return classify_pb2.ClassifyReply(message=json.dumps(res))
def fac_test_predic(self, data):
res = self.predictorfac.predict(data)
......
http_server.py
......@@ -5,10 +5,10 @@ from urllib.parse import unquote
import tornado.web
import tornado.ioloop
import traceback
from utils.log_manager import bom_log
from predic_fac import PredictorFac
from utils.status import code2msg
class BaseHandler(tornado.web.RequestHandler):
......@@ -18,53 +18,33 @@ class BaseHandler(tornado.web.RequestHandler):
class KwHandler(BaseHandler):
async def get(self):
try:
target = unquote(self.get_argument('keyword'))
predict_type = unquote(self.get_argument('type', 'all'))
bom_log.info(f'http收到识别关键词: {target}, 预测类型为{predict_type}')
res = self.predictorfac.predict(target, predict_type)
res['status'] = 1
except:
res = {
'status': 0,
'err_msg': '未知'
}
bom_log.error(f'http关键词识别报错: \n' + traceback.format_exc())
target = unquote(self.get_argument('keyword'))
predict_type = unquote(self.get_argument('type', 'all'))
bom_log.info(f'http收到识别关键词: {target}, 预测类型为{predict_type}')
code, res = self.predictorfac.predict(target, predict_type)
if code != 1:
res = code2msg(code)
bom_log.error(f'http关键词识别报错: ' + res['message'])
else:
res['status'] = code
self.write(res)
class DictHandler(BaseHandler):
async def post(self):
err_msg = None
try:
bom_log.info(f'http收到识别字典: {self.request.body}')
bom_log.info(f'http收到识别体: {self.request.body}')
dic_data = json.loads(self.request.body)
res = self.predictorfac.predict(dic_data)
res['status'] = 1
if '数量' not in str(res):
err_msg = '未识别到数量列'
if '参数' not in str(res) and '型号' not in str(res):
err_msg = '未识别到参数列'
code, res = self.predictorfac.predict(dic_data)
except json.decoder.JSONDecodeError:
res = self.predictorfac.predict(self.request.body)
res['status'] = 1
except:
res = {
'status': 0,
'err_msg': '未知'
}
bom_log.error(f'http字典识别报错: \n' + traceback.format_exc())
if err_msg:
res = {
'status': 0,
'err_msg': err_msg
}
bom_log.info(f'返回结果: ' + str(res))
code, res = self.predictorfac.predict(self.request.body)
if code != 1:
res = code2msg(code)
bom_log.error(f'http识别报错: ' + res['message'])
else:
res['status'] = code
self.write(res)
def gen_app():
return tornado.web.Application(handlers=[(k, v) for k, v in register_tornado_handlers.items()])
......
id_server.py
......@@ -15,10 +15,12 @@ def run(http_port, grpc_port):
p_http.start()
p_grpc.start()
def http_run(http_port):
http_server(http_port)
if __name__ == "__main__":
http_port = 50052
grpc_port = 50051
run(http_port, grpc_port)
# run(http_port, grpc_port)
http_run(http_port)
predic_fac.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import json
from predict import dict_predict, kw_predict
import json
from predict import dict_predict, kw_predict, txt_predict
from utils.status import code2msg, Status, StatusCode
class PredictorFac:
Status = Status
def __init__(self):
self.kw_predictor = kw_predict.KwPredict('single')
self.dict_predictor = dict_predict.DicPredict('dict')
self.txt_predictor = txt_predict.TxtPredict('txt')
def _get_predictor(self, data):
if isinstance(data, dict):
return self.dict_predictor
elif isinstance(data, list):
return self.txt_predictor
else:
return self.kw_predictor
def predict(self, dic_data, predict_type='all'):
predictor = self._get_predictor(dic_data)
res = predictor.predict(dic_data, predict_type)
return res
try:
res = predictor.predict(dic_data, predict_type)
if isinstance(predictor, dict_predict.DicPredict):
if '数量' not in str(res):
raise self.Status.NoQtyError
if '参数' not in str(res) and '型号' not in str(res):
raise self.Status.NoParamGnError
return 1, res
except Status.StatusErrTup as e:
stat_code = e.value
return stat_code, code2msg(stat_code)
if __name__ == "__main__":
def get_test_data():
import pandas as pd
import json
df = pd.read_excel(r'C:\Users\ICHUNT\Desktop\bomlist\sky-新建 XLSX 工作表 (2).xlsx', header=None)
df = pd.read_excel(r'C:\Users\ICHUNT\Desktop\bomlist\BOM_型号匹配问题.xls', header=None)
df.fillna('?', inplace=True)
dic_dft = df.to_dict(orient='list')
return json.dumps(dic_dft)
data = get_test_data()
return dic_dft
p = PredictorFac()
data = json.loads(data)
pretty_col = {'第%s列' % k: v for k, v in data.items()}
print(pretty_col)
res = p.predict(pretty_col, predict_type='all')
# data = get_test_data()
# data = {}
# pretty_col = {'第%s列' % k: v for k, v in data.items()}
# print(pretty_col)
data=['lm358dt 3', 'C0806fwefhkds 5k']
data = json.dumps(data)
res = p.predict(data, predict_type='all')
print(res)
predict/base_handler.py
......@@ -2,6 +2,8 @@
# -*- coding:utf-8 -*-
import re, os, pickle
from collections import Counter
from copy import deepcopy
from sklearn.externals import joblib
from config import model_config
......@@ -9,7 +11,7 @@ from utils.log_manager import get_logger
from utils.robots import dd_send_msg
import pandas as pd
from static_config import *
from utils.status import StatusCode, code2msg
# 数量列阈值
RIGHT_LEVEL = 0.7
# 序号列阈值
......@@ -39,6 +41,7 @@ class ClassBasePredictorMeta(type):
model = joblib.load(modextr_path + i)
setattr(cls, i, model)
def __new__(mcs, name, bases, attrs):
return type.__new__(mcs, name, bases, attrs)
......@@ -50,7 +53,6 @@ class BasePredictor(metaclass=ClassBasePredictorMeta):
# 占位符
PLACEHOLDER = '?'
pd = pd
def __init__(self, name):
self.name = name
......@@ -100,7 +102,7 @@ class BasePredictor(metaclass=ClassBasePredictorMeta):
是否是数量列辅助函数
"""
if isinstance(kw, str):
return re.match(r'(\d+|\d+(\.\d+))($|(K)|([\u4E00-\u9FA5]{1,3}))$', kw, re.M | re.I)
return re.match(r'(\d+|\d+(\.\d+))($|(K)|[个十百千万])$', kw, re.M | re.I)
else:
return False
......@@ -171,6 +173,7 @@ class BasePredictor(metaclass=ClassBasePredictorMeta):
@classmethod
def get_comprehensive_res(cls, head_result, model_result, major='head', vote_type='standard'):
if vote_type == 'standard':
vote_count = {k: [] for k in STD_FIELDS_MAP.keys()}
else:
......@@ -183,8 +186,11 @@ class BasePredictor(metaclass=ClassBasePredictorMeta):
for k, v in head_result.items():
vote_count[v].append(k)
vote_count_copy = deepcopy(vote_count)
for k, v in model_result.items():
if len(vote_count_copy[v]) == 1:
continue
vote_count[v].append(k)
# 此处并没有处理表头和模型两者预测冲突的情况, 但因为repeat_max方法是稳定的,所以冲突后会取第一个也就是表头预测的结果, 之后按照major参数决定冲突结果
......@@ -286,5 +292,6 @@ class BasePredictor(metaclass=ClassBasePredictorMeta):
return round(count / len(data), 3) >= SIMPLE_ENCAP_LEVEL
if __name__ == "__main__":
print(BasePredictor.is_ref(
['?', '位置', '?', '4C33', '3C5,11C8', '3C6,3C7,2C13,2C14,C283,C285,C286', '?', 'C287,C575,10C33,10C34,10C36,10C37', '4C6,4C26,4C49,4C65', '10C7,10C10,10C11,10C29', '8C4,8C5', '8C40,8C41', '1C7,5C2,5C3,10C2,1C25,1C41,2C11,2C21,2C25,2C37,2C40,4C62,8C29,8C30', '10C12', '10C26,10C35', '1C24,1C43', '10C16,10C20,10C22,10C23', '10C19,10C25', '1C1,1C6,2C1,2C3,2C4,2C6,2C7,3C1,3C2,3C4,3C8,4C2,6C6,6C8,6C9,7C6,7C9,8C7,8C8,8C9,10C4,10C5,10C6,10C9,1C10,1C13,11C4,1C15,1C16,11C6,11C7,1C19,1C22,1C28,1C37,1C44,1C57,1C59,1C63,2C15,2C17,2C29,2C33,2C36,2C38,2C39,3C13,4C12,4C13,4C14,4C15,4C16,4C18,4C19,4C20,4C22,4C23,4C24,4C25,4C28,4C30,4C32,4C34,4C41,4C44,4C45,4C46,4C48,4C51,4C54,4C55,4C57,4C58,4C59,4C60,4C61,6C14,6C16,7C13,8C11,8C17,8C21,10C14,10C17,10C18,10C21,10C28,10C30,10C32,10C42,4C111,4C118,4C121,4C122,C28,C97,C292,C293,C294,C296,C298,C299,C300,C301,C304,C571\n', '10C3,10C15', '2C2,2C9,3C3,4C4,5C6,5C7,5C8,5C9,6C7,7C3,7C7,8C3,2C16,2C18,2C22,2C23,2C26,2C27,2C31,4C17,4C47,4C56,4C63,4C93,5C10,6C46,8C10,8C16,8C18,8C23,8C25,8C31,8C32,8C47,9C21,9C28,4C100,C98,11C9,C290,C291,C295,C297,11C10', '10C8,10C24', '5C1,5C5,2C20,2C30,4C36,4C86,9C27,2C10', '2C12,8C53,9C22', '2C5,3C14,8C37,8C44,10C27', '1C9,4C1,4C7,10C1,1C11,1C14,1C17,1C18,1C20,1C21,1C53,1C58,1C60,1C62,1C64,2C34,2C35,3C15,4C11,4C29,4C31,4C42,4C64,4C82,4C83,8C15,8C42,10C31,4C157,C96,C288,C289,C302,C303,C574', '1C2,1C8,1C30,1C32,1C38,1C70', '1C4,1C5,2C8,4C3,7C4,1C12,1C23,1C26,1C29,1C31,1C34,1C42,1C48,2C32,7C11,7C14,C1,C570', '1C51,10C39,10C41', '1C66,1C71', '?', '?', '10R9,3R24,8R45,8R46', '3C10,7R6,R154,R155,2R21,3R25,7R25,9R32,9R33,9R34,9R35,9R36,9R37,9R38,9R39,9R40,9R41,9R42,9R43,9R44,9R45,9R46,9R47,9R48,9R49,9R50,9R51,9R52,9R53,10R18', 'R1,R2,R3,R4', '7R3,7R4,7R7,7R8,7R23,7R24,8R39,8R41,8R42,8R44', '1R1,1R2,1R16,1R19,1R26,1R49', '10R1,10R8,10R10,10R13', '2R7,2R8,3R3,3R4,3R5,3R6,3R7,2R10,3R20,3R33,3R34,3R35,3R36,3R37', '8R3,8R29,8R30', '6R3', '8R33,8R34,8R35,8R36', '?', '8R2,3R16', '8R37,8R38,8R40,8R43', '?', '2R4,2R5,10R2,10R3,3R17,5R29,5R30,5R32,5R33,5R38,9R13,9R14', '?', '3R15', '5R42', '4R10,4R11,4R12,4R28,4R35', '?', '10R4,10R14', '6R2', '9R5', '1R50,1R53,1R54,1R55,1R56,2R19,5R14,5R19,5R24,5R25,11R11,11R12', '5R40', '1R51', '8R1,4R15,4R16,4R23,4R24,4R33,4R34,10R22,10R30,10R40,10R41', '8R5,8R7', 'R156,R161,1R6,2R2,10R5,8R24,9R15,9R16,11R13,11R15,11R16', '5R28,5R41', '8R18', 'R158', '1R7', '8R16', '2R9', '7R1,8R4,8R6,9R1,10R6,1R21,1R42,1R57,1R58,R160,2R26,3R19,3R23,8R20,8R22,10R11,R167,9R54', '7R5,1R22,1R27,5R16', '1R43', '8R21,8R28', '10R17', '2R12', '1R31', '1R28,10R12', '1R3,5R4,5R37', '1R32', '1R5', '1R4,1R13,1R17,1R23,1R33', '1R14,1R39', '1R15', '5R2,1R12,1R20,1R44,5R11,5R12,5R13,5R17,5R18,5R35', '8R26,8R31', 'R164', '1R29', '2R3', '1R34', '5R3,1R10,1R41,1R60,5R36,11R14', 'R163', '1R18', 'R162,R165', 'R159', '5RN3,5RN4,5RN5,5RN6', '?', '?', '7FB5', '8FB1,10FB1,11FB1,FB4,FB6,FB8,8L2', '?', '10FB2,10FB3', '1L3,1L4', '?', '1L2', 'L2,3L1', 'L1', '1L5', '1L1', '1L6', '5L1,5L2', '10L1,10L2,10L3,10L4', '?', '?', 'D45', 'Q23', '1Q1,1Q8,3Q1,5Q3,5Q4,1Q5,1Q7', '1Q3', '?', '1Q6', '5Q2', '5Q5', '5D3,5D4,5D5,5D6', '5D7,5D8,5D9,7D1,7D2,7D3,7D4,7D5,8D3,8D4,11D1,11D2,5D10,5D11,5D12,5D20,5D22,5D23,5D24', '8D1', '8D2', 'T4', 'U1', 'U7,U18', '?', 'U9', 'U17', '1U2,1U3,1U10', '1U4,1U6,1U7', '2U1', '4U1,4U2,4U3,4U4', '?', '1U1,1U9', '1U5', '8U1', '2U2', '1U11', '3U2', '10U1', '3Y2', '?', '2Y1', 'Y1', '3J1', '8T1', '8JA5', 'CN13', 'J45', '?', '5JA2,5JA3', '1CN2', 'J2,J3', 'J53,J56,J58', '2CN1', 'J4', 'J1', 'J52', 'P2', 'J43']))
print(BasePredictor.is_num(['1k','1k','1k','1k']))
print(re.match(r'(\d+|\d+(\.\d+))($|(K)|[个十百千万])$', '1k', re.M | re.I))
\ No newline at end of file
predict/txt_predict.py 0 → 100644
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from predict.base_handler import BasePredictor
from static_config import EN_TO_ZH_MAP
import re
class TxtPredict(BasePredictor):
@classmethod
def isnum(cls, kw):
"""
是否是数字串
"""
if str(kw).startswith('0'):
return False
return re.match(r'(\d+|\d+(\.\d+))($|(K)|[个十百千万])$', kw, re.M | re.I)
def pre_deal(self, li: list):
data = []
resp = {}
for row_index, i in enumerate(li):
raw_items = re.split(r'(?:，|,|;|\s)\s*', i)
no_null_raw_items = [j.strip() for j in raw_items if j.strip()]
resp_data = {
'qty': None,
'brand': None,
'gn_or_param': None,
'statu_code': 1
}
for item in no_null_raw_items:
if self.isnum(item):
resp_data['qty'] = item
no_null_raw_items.remove(item)
break
else:
resp_data = {'message': '未检测到数量', 'statu_code': 0}
resp[row_index] = resp_data
data.append((row_index, resp_data, no_null_raw_items))
continue
if not no_null_raw_items:
resp_data = {'message': '未检测到参数/型号', 'statu_code': 0}
resp[row_index] = resp_data
data.append((row_index, resp_data, no_null_raw_items))
return data
def predict(self, li: list, predict_type='all'):
self.info.info('预测类型为: %s, 接收数据: %s' % (predict_type, li))
data = self.pre_deal(li)
resp = {}
for row_index, resp_data, nonull_nonum_items in data:
if resp_data['statu_code'] == 0:
resp[row_index] = resp_data
continue
if len(nonull_nonum_items) == 1:
resp_data['gn_or_param'] = nonull_nonum_items[0]
elif len(nonull_nonum_items) >= 2:
result_1, prab = self.get_single_predict(nonull_nonum_items[-1], BasePredictor.model, BasePredictor.extractor)
result0, prab = self.get_single_predict(nonull_nonum_items[0], BasePredictor.model, BasePredictor.extractor)
if EN_TO_ZH_MAP[result_1] == '品牌':
resp_data['brand'] = nonull_nonum_items[-1]
resp_data['gn_or_param'] = ' '.join(nonull_nonum_items[:-1])
elif EN_TO_ZH_MAP[result0] == '品牌':
resp_data['brand'] = nonull_nonum_items[0]
resp_data['gn_or_param'] = ' '.join(nonull_nonum_items[1:])
else:
resp_data['gn_or_param'] = ' '.join(nonull_nonum_items)
resp[row_index] = resp_data
return {'results': resp}
if __name__ == "__main__":
print(TxtPredict.is_num('1K'))
utils/status.py 0 → 100644
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import inspect
def code2msg(code):
if code in StatusCode.StatusDict:
return {"status": code, "message": StatusCode.StatusDict[code][1]}
else:
return {"status": "0", "message": StatusCode.StatusDict["0"][1]}
class StatusCode:
Succeed = "1"
Unknown = "0"
ParamError = "100001"
MissingParam = "100002"
NoQtyError = "100003"
NoParamGnError = "100004"
StatusDict = {
"1": ("succeed", "正常"),
"0": ("unknown", "未知错误"),
"100001": ("param error", "输入参数错误"),
"100002": ("missing param", "缺少输入的参数"),
"100003": ("no qty", "未检测到数量"),
"100004": ("no param or gn", "未检测到参数/型号列"),
}
class BaseStatusError(Exception):
def __init__(self, *args, **kwargs):
super().__init__(*args)
class_name = kwargs["class_name"] if "class_name" in kwargs else self.__class__.__name__
self.value = kwargs["value"] if "value" in kwargs else getattr(StatusCode, class_name)
if "msg" in kwargs:
self.msg = kwargs["msg"]
elif args:
self.msg = args[0]
else:
self.msg = StatusCode.StatusDict[self.value][0]
error_logging = False if "off_error_logging" in kwargs else True
self.error_logging = bool(kwargs["error_logging"]) if "error_logging" in kwargs else error_logging
self.frames = inspect.getouterframes(inspect.currentframe())[1] if self.error_logging else None
def __str__(self):
return str(self.msg)
class Status:
class Succeed(BaseStatusError):
class NotFound(BaseStatusError):
def __init__(self, *args, **kwargs):
super().__init__(off_error_logging=True, *args, **kwargs)
def __init__(self, *args, **kwargs):
super().__init__(off_error_logging=True, *args, **kwargs)
self.result = args[0]
class MissingParam(BaseStatusError):
def __init__(self, *args, **kwargs):
super().__init__(off_error_logging=True, *args, **kwargs)
class ParamError(BaseStatusError):
def __init__(self, *args, **kwargs):
super().__init__(off_error_logging=True, *args, **kwargs)
class NoQtyError(BaseStatusError):
def __init__(self, *args, **kwargs):
super().__init__(off_error_logging=True, *args, **kwargs)
class NoParamGnError(BaseStatusError):
def __init__(self, *args, **kwargs):
super().__init__(off_error_logging=True, *args, **kwargs)
class Unknown(BaseStatusError):
pass
StatusErrTup = (Succeed.NotFound, Unknown,
ParamError, MissingParam, ParamError, NoQtyError, NoParamGnError)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment