Skip to content

lzzzzl / param_data

Go to a project
Commit 5f74c23a by lzzzzl
Options

添加处理全部参数逻辑

parent 2e0478ed
Inline Side-by-side
Showing with 235 additions and 374 deletions
config/db_config_local.py deleted 100644 → 0
# _*_ coding:utf-8 _*_
import sys
import pymysql
from elasticsearch import Elasticsearch
zy_conn = {
'host': '172.18.137.33',
'port': 3306,
'user': 'LxDDUsedRead',
'password': '0o9u0U2oixoYmosflmxXtZmyt',
'db': 'liexin_data',
'charset': 'utf8'
}
local_conn = {
'host': 'localhost',
'port': 3306,
'user': 'root',
'password': 'qaz',
'db': 'bigdata',
'charset': 'utf8'
}
# SKU
# db-user :LxiCSpuR
# db-pwd :Mysx3Tyzlo00oxlmllyR
# db-host : spu-slave.ichunt.db
local_test = {
'host': 'localhost',
'port': 3306,
'user': 'root',
'password': 'qaz',
'db': 'test',
'charset': 'utf8'
}
local_mapping_conn = {
'host': 'localhost',
'port': 3306,
'user': 'root',
'password': 'qaz',
'db': 'mapping',
'charset': 'utf8'
}
dashboard_conn = {
'host': 'localhost',
'port': 3306,
'user': 'dashboard',
'password': 'ichunt5Dashboard@',
'db': 'dashboard',
'charset': 'utf8'
}
order_conn = {
'host': '172.18.137.22',
'port': 3306,
'user': 'huntdbslave',
'password': 'mLssy2@@!!@$#yy',
'db': 'hunt2016',
'charset': 'utf8'
}
learning_conn = {
'host': 'localhost',
'port': 3306,
'user': 'lEaUx',
'password': 'Xp#psOlT@!@#lfs^%wl',
'db': 'learning_data',
'charset': 'utf8'
}
bigdata_conn = {
'host': '192.168.2.232',
'port': 3306,
'user': 'bigdata',
'password': 'bigdata2019',
'db': 'digikey',
'charset': 'utf8'
}
online_bigdata_conn = {
'host': '192.168.2.232',
'port': 3306,
'user': 'bigdata',
'password': 'bigdata2019',
'db': 'digikey',
'charset': 'utf8'
}
class_conn = {
'host': '192.168.2.250',
'port': 3306,
'user': 'root',
'password': 'root',
'db': 'liexin_pool_class',
'charset': 'utf8'
}
class_conn_online = {
'host': '192.168.2.250',
'port': 3306,
'user': 'root',
'password': 'root',
'db': 'liexin_pool_class',
'charset': 'utf8'
}
# db-user :LxiCSpuR
# db-pwd :Mysx3Tyzlo00oxlmllyR
# db-host : spu-slave.ichunt.db
spu_conn = {
'host': '192.168.1.235',
'port': 3306,
'user': 'spu',
'password': 'spu',
'db': 'liexin_spu',
'charset': 'utf8'
}
spu_conn_online = {
'host': '192.168.1.235',
'port': 3306,
'user': 'spu',
'password': 'spu',
'db': 'liexin_spu',
'charset': 'utf8'
}
local_learning_conn = {
'host': '192.168.1.235',
'port': 3306,
'user': 'learning_data',
'password': 'learning_data#zsyM',
'db': 'learning_data',
'charset': 'utf8'
}
local_spu = {
'host': '192.168.1.235',
'port': 3306,
'user': 'spu',
'password': 'spu',
'charset': 'utf8'
}
local_es = [{
'host': '192.168.2.232',
'port': 9200
}]
online_es = [{
'host': '192.168.2.232',
'port': 9200
}]
def get_env():
if sys.platform in ('darwin', 'win32'):
return 'test'
else:
return 'produce'
class DBConn:
@staticmethod
def local_conn():
return pymysql.connect(local_conn['host'],
local_conn['user'],
local_conn['password'],
local_conn['db'],
charset=local_conn['charset'])
@staticmethod
def local_test():
return pymysql.connect(local_test['host'],
local_test['user'],
local_test['password'],
local_test['db'],
charset=local_test['charset'])
@staticmethod
def local_mapping_conn():
return pymysql.connect(local_mapping_conn['host'],
local_mapping_conn['user'],
local_mapping_conn['password'],
local_mapping_conn['db'],
charset=local_mapping_conn['charset'])
@staticmethod
def zy_conn():
return pymysql.connect(zy_conn['host'],
zy_conn['user'],
zy_conn['password'],
zy_conn['db'],
charset=zy_conn['charset'])
@staticmethod
def order_conn():
return pymysql.connect(order_conn['host'],
order_conn['user'],
order_conn['password'],
order_conn['db'],
charset=order_conn['charset'])
@staticmethod
def dashboard_conn():
return pymysql.connect(dashboard_conn['host'],
dashboard_conn['user'],
dashboard_conn['password'],
dashboard_conn['db'],
charset=dashboard_conn['charset'])
@staticmethod
def learning_conn():
return pymysql.connect(learning_conn['host'],
learning_conn['user'],
learning_conn['password'],
learning_conn['db'],
charset=learning_conn['charset'])
@staticmethod
def local_es():
return Elasticsearch(local_es)
@staticmethod
def online_es():
return Elasticsearch(online_es)
@staticmethod
def es():
if get_env() == 'test':
return Elasticsearch(local_es)
else:
return Elasticsearch(online_es)
@staticmethod
def db_learning():
if get_env() == 'test':
return pymysql.connect(local_learning_conn['host'],
local_learning_conn['user'],
local_learning_conn['password'],
local_learning_conn['db'],
charset=local_learning_conn['charset'])
else:
return pymysql.connect(learning_conn['host'],
learning_conn['user'],
learning_conn['password'],
learning_conn['db'],
charset=learning_conn['charset'])
@staticmethod
def db_dashboard():
if get_env() == 'test':
return pymysql.connect(local_mapping_conn['host'],
local_mapping_conn['user'],
local_mapping_conn['password'],
local_mapping_conn['db'],
charset=local_mapping_conn['charset'])
else:
return pymysql.connect(dashboard_conn['host'],
dashboard_conn['user'],
dashboard_conn['password'],
dashboard_conn['db'],
charset=dashboard_conn['charset'])
@staticmethod
def db_bigdata():
if get_env() == 'test':
return pymysql.connect(bigdata_conn['host'],
bigdata_conn['user'],
bigdata_conn['password'],
bigdata_conn['db'],
charset=bigdata_conn['charset'])
else:
return pymysql.connect(online_bigdata_conn['host'],
online_bigdata_conn['user'],
online_bigdata_conn['password'],
online_bigdata_conn['db'],
charset=online_bigdata_conn['charset'])
@staticmethod
def db_class():
if get_env() == 'test':
return pymysql.connect(class_conn['host'],
class_conn['user'],
class_conn['password'],
class_conn['db'],
charset=class_conn['charset'])
else:
return pymysql.connect(class_conn_online['host'],
class_conn_online['user'],
class_conn_online['password'],
class_conn_online['db'],
charset=class_conn_online['charset'])
@staticmethod
def db_spu():
if get_env() == 'test':
return pymysql.connect(spu_conn['host'],
spu_conn['user'],
spu_conn['password'],
spu_conn['db'],
charset=spu_conn['charset'])
else:
return pymysql.connect(spu_conn_online['host'],
spu_conn_online['user'],
spu_conn_online['password'],
spu_conn_online['db'],
charset=spu_conn_online['charset'])
fun/param_conn.py
# _*_ coding:utf-8 _*_
import redis
import pymysql
import config.db_config as config
class PARAM_CONN:
class ParamConn:
@staticmethod
def online_sku_conn(index):
......@@ -19,4 +20,9 @@ class PARAM_CONN:
config.online_sku['user'],
config.online_sku['password'],
"liexin_sku_%d" % index,
charset="utf8")
\ No newline at end of file
charset="utf8")
@staticmethod
def local_redis_conn():
return redis.StrictRedis(
host="localhost", port=6379, db=0, password="", charset="UTF-8", encoding="UTF-8")
\ No newline at end of file
fun/param_data.py
# _*_ coding:utf-8 _*_
from utils.utilClass import LazyProperty
from utils.excel_handler import ExcelHandler
from utils.db_handler import DBHandler
from utils.log_handler import LogHandler
......@@ -9,29 +8,11 @@ from utils.date_handler import DateHandler
now_ts = DateHandler.now_datetime()
class PARAM_DATA(object):
class ParamData(object):
def __init__(self):
self.log = LogHandler('param_extract')
def class_mapping(self, conn):
"""
cat_id =>
:return:
"""
def get_attr_mapping(class_id):
sql = "SELECT attr_id,attr_name FROM lie_class_attr WHERE class_id = %d" % class_id
return {row[1]: row[0] for row in DBHandler.read(conn, sql)}
sql = "SELECT cat_id,class_id FROM lie_class"
attr_dict = {row[0]: get_attr_mapping(row[1]) for row in DBHandler.read(conn, sql) if get_attr_mapping(row[1])}
return attr_dict
def get_unit_value(self, conn):
sql = "SELECT attr_unit_id,attr_unit_name FROM lie_class_attr_unit"
res = DBHandler.read(conn, sql)
return {row[1]: row[0] for row in res}
def get_class_attr(self, conn):
sql = "SELECT attr_id,class_id,attr_unit_id FROM lie_class_attr"
return DBHandler.read(conn, sql)
......@@ -134,8 +115,51 @@ class PARAM_DATA(object):
写入class attr
"""
@staticmethod
def write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, class_db):
sql = "INSERT INTO lie_class_attr (attr_name,class_id,attr_unit_id,remark,status,add_time,update_time) \
VALUES ('%s','%s','%s','%s','%s','%s','%s')" % (basic_attr_name, class_id, unit_id, lx_attr_name, "1", DateHandler.now_datetime(), DateHandler.now_datetime())
def write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, dgk_attr_id, class_db):
sql = "INSERT INTO lie_class_attr (attr_name,class_id,attr_unit_id,remark,status,dgk_attr_id,add_time,update_time) \
VALUES ('%s','%s','%s','%s','%s','%s','%s', '%s')" % (basic_attr_name, class_id, unit_id, lx_attr_name,
"1", dgk_attr_id, DateHandler.now_datetime(),
DateHandler.now_datetime())
DBHandler.insert(class_db, sql)
"""
分类映射
"""
@staticmethod
def class_mapping(class_conn):
def get_attr_mapping(class_id):
sql = "SELECT attr_id,attr_name FROM lie_class_attr WHERE class_id = %d" % class_id
return {row[1]: row[0] for row in DBHandler.read(class_conn, sql)}
sql = "SELECT cat_id,class_id FROM lie_class"
attr_dict = {row[0]: get_attr_mapping(row[1]) for row in DBHandler.read(class_conn, sql)
if get_attr_mapping(row[1])}
return attr_dict
"""
获取单位值
"""
@staticmethod
def get_unit_value(class_conn):
sql = "SELECT attr_unit_id,attr_unit_name FROM lie_class_attr_unit"
res = DBHandler.read(class_conn, sql)
return {row[1]: row[0] for row in res}
"""
根据id获取dgk商品
"""
@staticmethod
def get_dgk_goods_by_ids(i, ids, pams, bg_conn):
sql = "SELECT attr_name,attr_value,attr_id,cat_id FROM lie_goods_attr_fields%d WHERE cat_id IN (%s) AND \
attr_id IN (%s) AND attr_value != ''" % (i, ids, pams)
return DBHandler.read(bg_conn, sql)
"""
写入attr value
"""
@staticmethod
def write_attr_value(attr_id, value, attr_unit_id, status, remark, add_time, update_time, class_conn):
sql = "INSERT INTO lie_class_attr_value (attr_id,value,attr_unit_id,status,remark,add_time,update_time) VALUES \
('%s','%s','%s','%s','%s','%s','%s')" % \
(attr_id, value, attr_unit_id, status, remark, add_time, update_time)
DBHandler.insert(class_conn, sql)
fun/param_trans.py
......@@ -29,6 +29,11 @@ encap_list = ["0030", "008004", "01005", "015008", "0201", "02016", "0202", "020
"3925", "3931", "4030", "4032", "4122", "4520", "4823", "4850", "5040", "5329", "5829", "5929",
"6028", "6030", "6031", "6039", "6054", "6560"]
temp_map = ["C0G", "NP0", "COG", "NPO", "X7R", "X5R", "Y5V", "X6S", "X7S", "X7T", "SL", "U2J", "UJ", "X7U", "X8R",
"Z5U", "C0H", "COH", "U2K", "X6T", "X8G", "X8L", "Y5R", "Y5U", "ZLM"]
sep_temp_map = ["C0G", "NP0", "COG", "NPO"]
param_func = {
'阻值（欧姆）': 'trans_resistance',
'容值': 'trans_capacity',
......@@ -39,14 +44,36 @@ param_func = {
'直流电阻（内阻）': 'trans_dc_resistance',
'精度': 'trans_accuracy',
'封装': 'trans_encap',
'温漂系数（介质材料)': 'trans_temp'
'温漂系数（介质材料)': 'trans_temp_coeff',
'温度': 'trans_temp',
'时间': 'trans_time',
'频率': 'trans_frequency',
}
param_unit = {
'阻值（欧姆）': 3,
'容值': 8,
'电感值': 13,
'额定电压': 20,
'额定电流': 18,
'功率': 22,
'直流电阻（内阻）': 3,
'精度': 24,
'封装': 26,
'温漂系数（介质材料)': 26,
'温度': 27,
'时间': 28,
'频率': 32
}
empty_unit_id = 26
class PARAM_TRANS:
class ParamTrans:
def __init__(self):
self.param_func = param_func
self.param_unit = param_unit
self.empty_unit_id = empty_unit_id
@classmethod
def base_trans(cls, trans_rule, key):
......@@ -68,7 +95,7 @@ class PARAM_TRANS:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]Ω|µΩ|Ω)$'
key = str(key).replace("Ohms", 'Ω')
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_capacity(cls, key):
......@@ -79,7 +106,7 @@ class PARAM_TRANS:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]F|µF|F)$'
key = str(key).split("(")[0]
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_inductance(cls, key):
......@@ -89,7 +116,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]H|µH|H)$'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_rated_voltage(cls, key):
......@@ -99,7 +126,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]V|V)$'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_rated_current(cls, key):
......@@ -110,7 +137,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]A|A)$'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_power(cls, key):
......@@ -120,7 +147,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]W|W)$'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_dc_resistance(cls, key):
......@@ -134,7 +161,7 @@ class PARAM_TRANS:
for ct in cast_map:
if str(key).find(ct) != -1:
key = str(key).replace(ct, cast_map[ct])
return PARAM_TRANS.trans_resistance(key)
return ParamTrans.trans_resistance(key)
@classmethod
def trans_accuracy(cls, key):
......@@ -144,7 +171,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(±\d+|±\d+.\d+|\d+|\d+\.\d+)([a-zA-Z]+|%)$'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_encap(cls, key):
......@@ -162,6 +189,23 @@ class PARAM_TRANS:
return key, ''
@classmethod
def trans_temp_coeff(cls, key):
"""
温漂系数
:param key:
:return:
"""
if key == '-' or key == '':
return 0, 0
for sep_temp in sep_temp_map:
if str(key).find(sep_temp) != -1:
return "C0G", 0
for temp in temp_map:
if str(key).find(temp) != -1:
return temp, 0
return key, ''
@classmethod
def trans_temp(cls, key):
"""
温度
......@@ -169,7 +213,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]W|W)$'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_time(cls, key):
......@@ -179,7 +223,7 @@ class PARAM_TRANS:
:return:
"""
trans_rule = r'(\d+|\d+\.\d+)([a-zA-Z]s|cs|µs|ms|ns|ps)'
return PARAM_TRANS.base_trans(trans_rule, key)
return ParamTrans.base_trans(trans_rule, key)
@classmethod
def trans_frequency(cls, key):
......@@ -188,8 +232,9 @@ class PARAM_TRANS:
:param key:
:return:
"""
trans_rule = r'(±\d+|±\d+.\d+|\d+|\d+\.\d+)([a-zA-Z]ppm|ppm)$'
value, unit = PARAM_TRANS.base_trans(trans_rule, key)
return value * 10, unit
trans_rule = r'(±\d+|±\d+.\d+|\d+|\d+\.\d+)([a-zA-Z]ppm|ppm|[a-zA-Z]Hz|Hz)$'
value, unit = ParamTrans.base_trans(trans_rule, key)
if 'ppm' in unit:
return value * 10, 'Hz'
else:
return value, unit
param/class_extract.py
......@@ -4,7 +4,6 @@ from utils.log_handler import LogHandler
from utils.excel_handler import ExcelHandler
from utils.db_handler import DBHandler
from utils.date_handler import DateHandler
from fun.unit_trans import UnitTrans
from urllib import parse
import config.zy_config as config
import redis
......
param_task/value_task.py
# _*_ coding:utf-8 _*_
from utils.log_handler import LogHandler
from utils.excel_handler import ExcelHandler
from utils.date_handler import DateHandler
from config.db_config import DBConn
from fun.param_data import PARAM_DATA
from fun.param_trans import PARAM_TRANS
from fun.param_data import ParamData
from fun.param_trans import ParamTrans
from fun.param_conn import ParamConn
"""
值任务
"""
class VALUE_TASK:
class ValueTask:
def __init__(self):
self.class_list = list()
self.param_dict = dict()
self.redis = ParamConn.local_redis_conn()
self.class_db = DBConn.db_class()
self.param_trans = PARAM_TRANS()
self.log1 = LogHandler('param_extract', stream=True, file=False)
self.log2 = LogHandler('param_extract', stream=False, file=True)
self.bigdata_db = DBConn.db_bigdata()
self.param_trans = ParamTrans()
self.param_data = ParamData
self.log1 = LogHandler('value_task_info', stream=True, file=True)
self.log2 = LogHandler('value_task_detail', stream=False, file=True)
self.now = DateHandler.now_datetime()
self.empty_unit_id = 26
"""
加载数据
"""
def load_data(self, file_name, sheet):
self.log1.info("加载数据")
# 读取Excel数据
data = ExcelHandler.read_to_excel(file_name, sheet, 1)
data.pop(0)
# 遍历数据
for row in data:
# 参数id、二级分类id、基础参数名、猎芯参数名、函数名
param_id = row[0]
# 参数id、二级分类id、基础参数名、猎芯参数名
param_id = row[4]
second_classify_id = row[1]
basic_attr_name = row[2]
lx_attr_name = row[3]
basic_attr_name = row[5]
lx_attr_name = row[7]
# 函数名、参数id、单位id
fuc_name = self.param_trans.param_func[lx_attr_name]
class_id = self.param_data.get_class_id(second_classify_id, self.class_db)
unit_id = self.param_trans.param_unit[lx_attr_name]
self.class_list.append(str(int(second_classify_id)))
self.param_dict[str(int(param_id))] = fuc_name
ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db)
self.class_list = set(self.class_list)
"""
运行任务
"""
def run_task(self):
self.log1.info("运行任务")
ids = ",".join(self.class_list)
pams = ",".join([_ for _ in self.param_dict])
mapping = self.param_data.class_mapping(self.class_db)
unit_dict = self.param_data.get_unit_value(self.class_db)
for i in range(1, 10):
res = self.param_data.get_dgk_goods_by_ids(i, ids, pams, self.bigdata_db)
for row in res:
attr_name = row[0]
attr_value = row[1]
fields_attr_id = row[2]
cat_id = row[3]
if mapping[cat_id].get(attr_name):
attr_id = mapping[cat_id][attr_name]
value, unit = getattr(ParamTrans, self.param_dict[str(fields_attr_id)])(attr_value)
if value != 0:
unit_id = unit_dict[unit] if unit_dict.get(unit) else self.empty_unit_id
# redis去重
if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0:
self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now,
self.class_db)
print(self.class_list)
print(self.param_dict)
"""
运行任务
"""
def run(self):
self.load_data("C:\\data\\lx\\dgk_2.xlsx", "attr")
# self.run_task()
if __name__ == '__main__':
task = VALUE_TASK()
task.load_data("E:\\doc\\DGK参数.xlsx", "Sheet1")
task = ValueTask()
task.run()
param_task/value_task_all.py 0 → 100644
# -*- coding: utf-8 -*-
# !/usr/bin/env python
from utils.log_handler import LogHandler
from utils.excel_handler import ExcelHandler
from fun.param_data import ParamData
from fun.param_trans import ParamTrans
from config.db_config import DBConn
class ValueTaskAll:
def __init__(self):
self.param_data = ParamData
self.param_trans = ParamTrans()
self.class_db = DBConn.db_class()
self.log1 = LogHandler('value_task_all_info', stream=True, file=True)
"""
加载参数
"""
def load_attr(self):
self.log1.info("加载参数")
# 有效参数
effect_attrs = []
effect_1 = ExcelHandler.read_to_excel("C:\\data\\lx\\dgk_1.xlsx", "Sheet1", 1)
effect_1.pop(0)
[effect_attrs.append(int(row[0])) for row in effect_1]
effect_2 = ExcelHandler.read_to_excel("C:\\data\\lx\\dgk_2.xlsx", "Sheet1", 1)
effect_2.pop(0)
[effect_attrs.append(int(row[4])) for row in effect_2]
# 全部参数
all_data = ExcelHandler.read_to_excel("C:\\data\\lx\\dgk_all.xlsx", "Sheet1", 1)
all_data.pop(0)
for row in all_data:
# 参数id、二级分类id、基础参数名、猎芯参数名
param_id = row[0]
second_classify_id = row[3]
basic_attr_name = row[5]
is_important = row[6]
# 非重要参数 + 非有效参数
if is_important == 1 and param_id not in effect_attrs:
class_id = self.param_data.get_class_id(second_classify_id, self.class_db)
unit_id = self.param_trans.empty_unit_id
lx_attr_name = ''
ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db)
def run(self):
self.load_attr()
if __name__ == '__main__':
task = ValueTaskAll()
task.run()
utils/db_handler.py
......@@ -58,7 +58,7 @@ class DBHandler:
db.commit()
except:
db.rollback()
# traceback.print_exc()
traceback.print_exc()
# print(sql)
"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment