Skip to content

lzzzzl / param_data

Go to a project
Commit 51f9623d by lzzzzl
Options

更新代码

parent 15efb07d
Inline Side-by-side
Showing with 390 additions and 53 deletions
Test.py 0 → 100644
# _*_ coding:utf-8 _*_
from param.zy_handler import ZY_HANDLER
from config.db_config import DBConn
from utils.db_handler import DBHandler
import threading
dgk_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []}
sql = "SELECT goods_id FROM lie_goods_attrs"
res = DBHandler.read(DBConn.db_learning(), sql)
print(len(res))
def getSpuId(goods_id, conn):
sql = "SELECT spu_id FROM lie_goods WHERE goods_id = %d" % goods_id
# res = DBHandler.read(bigdata_db, sql)
res = DBHandler.read(conn, sql)
if len(res) > 0:
return res[0][0]
else:
return 0
def updateGoodsAttrs(goods_id,spu_id,conn):
sql = "UPDATE lie_goods_attrs SET spu_id=%d WHERE goods_id=%d" % (spu_id,goods_id)
print(sql)
# DBHandler.read(learning_db, sql)
DBHandler.read(conn, sql)
for row in res:
param_index = int(str(row[0])[-1])
dgk_dict[param_index].append({'goods_id': row[0]})
def updateData(data):
index = 0
bigdata_db = DBConn.db_bigdata()
learning_db = DBConn.db_learning()
for row in data:
index += 1
goods_id = row['goods_id']
spu_id = getSpuId(goods_id, bigdata_db)
if spu_id != 0:
updateGoodsAttrs(goods_id, spu_id, learning_db)
if index % 1000 == 0:
print(index)
thread_list = []
t0 = threading.Thread(target=updateData, args=(dgk_dict[0],))
t1 = threading.Thread(target=updateData, args=(dgk_dict[1],))
t2 = threading.Thread(target=updateData, args=(dgk_dict[2],))
t3 = threading.Thread(target=updateData, args=(dgk_dict[3],))
t4 = threading.Thread(target=updateData, args=(dgk_dict[4],))
t5 = threading.Thread(target=updateData, args=(dgk_dict[5],))
t6 = threading.Thread(target=updateData, args=(dgk_dict[6],))
t7 = threading.Thread(target=updateData, args=(dgk_dict[7],))
t8 = threading.Thread(target=updateData, args=(dgk_dict[8],))
t9 = threading.Thread(target=updateData, args=(dgk_dict[9],))
thread_list.append(t0)
thread_list.append(t1)
thread_list.append(t2)
thread_list.append(t3)
thread_list.append(t4)
thread_list.append(t5)
thread_list.append(t6)
thread_list.append(t7)
thread_list.append(t8)
thread_list.append(t9)
for t in thread_list:
t.setDaemon(True) # 设置为守护线程，不会因主线程结束而中断
t.start()
for t in thread_list:
t.join() # 子线程全部加入，主线程等所有子线程运行完毕
print('Mainthread %s ended.' % threading.current_thread().name)
config/__pycache__/db_config.cpython-37.pyc 0 → 100644
No preview for this file type
config/__pycache__/zy_config.cpython-37.pyc 0 → 100644
No preview for this file type
config/trans_config.py
...@@ -35,6 +35,7 @@ vol_map = { ...@@ -35,6 +35,7 @@ vol_map = {
} }
pow_map = { pow_map = {
'mW': 0.001,
'W': 1, 'W': 1,
'kW': 1000 'kW': 1000
} }
...@@ -78,5 +79,23 @@ brand_map = { ...@@ -78,5 +79,23 @@ brand_map = {
'Pulse': 15 'Pulse': 15
} }
brand_res_map = {
1: 'Yageo',
2: 'Murata',
3: 'Samsung',
4: 'TDK',
5: 'Taiyo Yuden',
6: 'Rohm',
7: 'KEMET',
8: 'Vishay',
9: 'AVX',
10: 'Panasonic',
11: 'KOA',
12: 'Bourns',
13: 'TE',
14: 'Wurth',
15: 'Pulse'
}
temp_map = ["C0G","NP0","COG","NPO","X7R","X5R","Y5V","X6S","X7S","X7T","SL","U2J","UJ","X7U","X8R","Z5U","C0H","COH","U2K","X6T","X8G","X8L","Y5R","Y5U","ZLM"] temp_map = ["C0G","NP0","COG","NPO","X7R","X5R","Y5V","X6S","X7S","X7T","SL","U2J","UJ","X7U","X8R","Z5U","C0H","COH","U2K","X6T","X8G","X8L","Y5R","Y5U","ZLM"]
sep_temp_map = ["C0G","NP0","COG","NPO"] sep_temp_map = ["C0G","NP0","COG","NPO"]
fun/__pycache__/__init__.cpython-37.pyc 0 → 100644
No preview for this file type
fun/__pycache__/param_conn.cpython-37.pyc 0 → 100644
No preview for this file type
fun/__pycache__/param_data.cpython-37.pyc 0 → 100644
No preview for this file type
fun/__pycache__/param_thread.cpython-37.pyc 0 → 100644
No preview for this file type
fun/__pycache__/param_trans.cpython-37.pyc 0 → 100644
No preview for this file type
fun/__pycache__/unit_trans.cpython-37.pyc 0 → 100644
No preview for this file type
fun/unit_trans.py
...@@ -25,61 +25,71 @@ class UnitTrans: ...@@ -25,61 +25,71 @@ class UnitTrans:
# 阻值（欧姆） # 阻值（欧姆）
def trans_resistance(self, key): def trans_resistance(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
key = str(key).split("(")[0] key = str(key).split("(")[0]
key = str(key).replace("Ohms", 'Ω') key = str(key).replace("Ohms", 'Ω')
obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]Ω|µΩ|Ω)$', key.replace(" ", "")) key = key.replace(" ", "")
obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]Ω|µΩ|Ω)$', key)
value = obj.group(1) if obj else '0' value = obj.group(1) if obj else '0'
unit = obj.group(2) if obj else 1 unit = obj.group(2) if obj else 1
if res_map.get(unit): if res_map.get(unit):
ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal(res_map[unit]))) ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal(res_map[unit])))
else: else:
ts_value = '' ts_value = ''
return key, ts_value
return {"key": key, "value": value, "ts_value": ts_value}
# 容值 # 容值
def trans_capacity(self, key): def trans_capacity(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
key = str(key).split("(")[0] key = str(key).split("(")[0]
obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]F|µF|F)$', key.replace(" ", "")) obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]F|µF|F)$', key.replace(" ", ""))
value = obj.group(1) if obj else '0' value = obj.group(1) if obj else '0'
unit = obj.group(2) if obj else 1 unit = obj.group(2) if obj else 1
return key, self.remove_exponent(Decimal(Decimal(value) * Decimal(cap_map[unit]))) if cap_map.get(unit) else '' ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal(cap_map[unit]))) if cap_map.get(unit) else ''
return {"key": key, "value": value, "ts_value": ts_value}
# 电感值 # 电感值
def trans_inductance(self, key): def trans_inductance(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
key = str(key).split("(")[0] key = str(key).split("(")[0]
obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]H|µH|H)$', key.replace(" ", "")) obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]H|µH|H)$', key.replace(" ", ""))
value = obj.group(1) if obj else '0' value = obj.group(1) if obj else '0'
unit = obj.group(2) if obj else 1 unit = obj.group(2) if obj else 1
return key, self.remove_exponent(Decimal(Decimal(value) * Decimal(ind_map[unit]))) if ind_map.get(unit) else '' ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal(ind_map[unit]))) if ind_map.get(unit) else ''
return {"key": key, "value": value, "ts_value": ts_value}
# 额定电压 # 额定电压
def trans_rated_voltage(self, key): def trans_rated_voltage(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
key = str(key).split("（")[0] key = str(key).split("（")[0]
obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]V|V)$', key.replace(" ", "")) obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]V|V)$', key.replace(" ", ""))
value = obj.group(1) if obj else '0' value = obj.group(1) if obj else '0'
unit = obj.group(2) if obj else 1 unit = obj.group(2) if obj else 1
return key, self.remove_exponent(Decimal(Decimal(value) * Decimal(vol_map[unit]))) if vol_map.get(unit) else '' ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal(vol_map[unit]))) if vol_map.get(unit) else ''
return {"key": key, "value": value, "ts_value": ts_value}
# 额定电流 # 额定电流
def trans_rated_current(self, key): def trans_rated_current(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]A|A)$', key.replace(" ", "")) obj = re.match(r'(\d+|\d+\.\d+)([a-zA-Z]A|A)$', key.replace(" ", ""))
value = obj.group(1) if obj else '0' value = obj.group(1) if obj else '0'
unit = obj.group(2) if obj else 1 unit = obj.group(2) if obj else 1
return key, self.remove_exponent(Decimal(Decimal(value) * Decimal(cur_map[unit]))) if cur_map.get(unit) else '' ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal(cur_map[unit]))) if cur_map.get(unit) else ''
return {"key": key, "value": value, "ts_value": ts_value}
# 功率 # 功率
def trans_power(self, key): def trans_power(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
key = str(key).split("，")[0] key = str(key).split("，")[0]
obj = re.match(r'(\d+|\d+\.\d+|\d+/\d+)([a-zA-Z]W|W|WS)$', key.replace(" ", "")) obj = re.match(r'(\d+|\d+\.\d+|\d+/\d+)([a-zA-Z]W|W|WS)$', key.replace(" ", ""))
value = obj.group(1) if obj else '0' value = obj.group(1) if obj else '0'
...@@ -88,44 +98,55 @@ class UnitTrans: ...@@ -88,44 +98,55 @@ class UnitTrans:
val_spl = value.split('/') val_spl = value.split('/')
value = Decimal((int(val_spl[0]) / int(val_spl[1]))).quantize(Decimal("0.000")) value = Decimal((int(val_spl[0]) / int(val_spl[1]))).quantize(Decimal("0.000"))
value = Decimal(value).quantize(Decimal("0.000")) value = Decimal(value).quantize(Decimal("0.000"))
return key, self.remove_exponent(Decimal(value * Decimal(pow_map[unit]))) if pow_map.get(unit) else '' ts_value = self.remove_exponent(Decimal(value * Decimal(pow_map[unit]))) if pow_map.get(unit) else ''
return {"key": key, "value": int(value), "ts_value": ts_value}
# 直流电阻 # 直流电阻
def trans_dc_resistance(self, key): def trans_dc_resistance(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
for ct in cast_map: for ct in cast_map:
if str(key).find(ct) != -1: if str(key).find(ct) != -1:
key = str(key).replace(ct, cast_map[ct]) key = str(key).replace(ct, cast_map[ct])
return self.trans_resistance(key) return self.trans_resistance(key)
# 精度 # 精度
def trans_accuracy(self, key): def trans_accuracy(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
obj = re.match(r'(±|)(\d+|\d+\.\d+)([a-zA-Z]+|%)$', key.replace(" ", "")) obj = re.match(r'(±|)(\d+|\d+\.\d+)([a-zA-Z]+|%)$', key.replace(" ", ""))
value = obj.group(2) if obj else '0' value = obj.group(2) if obj else '0'
unit = obj.group(3) if obj else 1 unit = obj.group(3) if obj else 1
return key, self.remove_exponent(Decimal(Decimal(value) * Decimal("0.01"))) ts_value = self.remove_exponent(Decimal(Decimal(value) * Decimal("0.01")))
return {"key": key, "value": value, "ts_value": ts_value}
# 封装 # 封装
def trans_encap(self, key): def trans_encap(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
key = str(key).split("，")[0] key = str(key).split("，")[0]
for encap in encap_list: for encap in encap_list:
if str(key).find(encap) != -1: if str(key).find(encap) != -1:
return key, encap return {"key": key, "value": encap, "ts_value": encap}
return key, ''
return {"key": "", "value": "", "ts_value": ""}
# 温度 # 温度
def trans_temp(self, key): def trans_temp(self, key):
if key == '-' or key == '': if key == '-' or key == '':
return key, '' return {"key": "", "value": "", "ts_value": ""}
for sep_temp in sep_temp_map: for sep_temp in sep_temp_map:
if str(key).find(sep_temp) != -1: if str(key).find(sep_temp) != -1:
return key, "C0G" return {"key": key, "value": "COG", "ts_value": "COG"}
for temp in temp_map: for temp in temp_map:
if str(key).find(temp) != -1: if str(key).find(temp) != -1:
return key, temp return {"key": key, "value": temp, "ts_value": temp}
return key, ''
return {"key": "", "value": "", "ts_value": ""}
param/__pycache__/spu_extract.cpython-37.pyc 0 → 100644
No preview for this file type
param/__pycache__/zy_handler.cpython-37.pyc 0 → 100644
No preview for this file type
param/param_class_redis.py
...@@ -21,18 +21,18 @@ class PARAM_CLASS_REDIS(object): ...@@ -21,18 +21,18 @@ class PARAM_CLASS_REDIS(object):
def __init__(self): def __init__(self):
self.db = DBConn self.db = DBConn
self.log = LogHandler('param_extract') self.log = LogHandler('param_extract')
self.log2 = LogHandler('param_class_redis', stream=False, file=True) self.log2 = LogHandler('param_class_redis', stream=True, file=True)
# self.redis = redis.StrictRedis(host="192.168.1.235", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8", encoding="UTF-8") self.redis = redis.StrictRedis(host="192.168.1.235", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8", encoding="UTF-8")
self.redis = redis.StrictRedis(host="172.18.137.38", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8", # self.redis = redis.StrictRedis(host="172.18.137.38", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8",
encoding="UTF-8") # encoding="UTF-8")
def search_sku(self, spu_id): def search_sku(self, spu_id):
for i in range(0, 10): for i in range(0, 10):
db = "liexin_sku_%d" % i db = "liexin_sku_%d" % i
# conn = pymysql.connect(config.local_spu['host'], config.local_spu['user'], config.local_spu['password'], db, conn = pymysql.connect(config.local_spu['host'], config.local_spu['user'], config.local_spu['password'], db,
# charset=config.local_spu['charset']) charset=config.local_spu['charset'])
conn = pymysql.connect("spu-slave.ichunt.db", "LxiCSpuR", "Mysx3Tyzlo00oxlmllyR", "liexin_sku_%d" % i, # conn = pymysql.connect("spu-slave.ichunt.db", "LxiCSpuR", "Mysx3Tyzlo00oxlmllyR", "liexin_sku_%d" % i,
charset="utf8") # charset="utf8")
for j in range(0, 10): for j in range(0, 10):
table = "lie_sku_%d" % j table = "lie_sku_%d" % j
sql = "SELECT spu_id,goods_id FROM %s WHERE supplier_id=7 AND spu_id=\'%d\'" % (table, spu_id) sql = "SELECT spu_id,goods_id FROM %s WHERE supplier_id=7 AND spu_id=\'%d\'" % (table, spu_id)
......
param/param_extract.py
...@@ -28,7 +28,8 @@ class PARAM_EXTRACT(object): ...@@ -28,7 +28,8 @@ class PARAM_EXTRACT(object):
for i in range(0, 10): for i in range(0, 10):
db = "liexin_sku_%d" % i db = "liexin_sku_%d" % i
# 获取sku 连接 # 获取sku 连接
conn = ParamConn.online_sku_conn(i) # conn = ParamConn.online_sku_conn(i)
conn = ParamConn.local_sku_conn(i)
for j in range(0, 10): for j in range(0, 10):
table = "lie_sku_%d" % j table = "lie_sku_%d" % j
sql = "SELECT spu_id,old_goods_id FROM %s WHERE supplier_id = 7" % table sql = "SELECT spu_id,old_goods_id FROM %s WHERE supplier_id = 7" % table
......
param/spu_extract.py
...@@ -4,14 +4,14 @@ import pymysql ...@@ -4,14 +4,14 @@ import pymysql
from param.class_extract import CLASS_EXTRACT from param.class_extract import CLASS_EXTRACT
from utils.db_handler import DBHandler from utils.db_handler import DBHandler
from utils.log_handler import LogHandler from utils.log_handler import LogHandler
from fun.param_data import PARAM_DATA from fun.param_data import ParamData
from fun.param_conn import PARAM_CONN from fun.param_conn import ParamConn
from config.db_config import DBConn from config.db_config import DBConn
import threading import threading
import json import json
import time import time
param_data = PARAM_DATA() param_data = ParamData()
class SPU_EXTRACT(object): class SPU_EXTRACT(object):
...@@ -33,7 +33,8 @@ class SPU_EXTRACT(object): ...@@ -33,7 +33,8 @@ class SPU_EXTRACT(object):
for i in range(0, 10): for i in range(0, 10):
self.log1.info("GET_SPU_DATA I-{i} : start".format(i=i)) self.log1.info("GET_SPU_DATA I-{i} : start".format(i=i))
# 获取sku连接 # 获取sku连接
conn = PARAM_CONN.online_sku_conn(i) # conn = PARAM_CONN.online_sku_conn(i)
conn = ParamConn.local_sku_conn(i)
for j in range(0, 10): for j in range(0, 10):
self.log1.info("GET_SPU_DATA J-{j} : start".format(j=j)) self.log1.info("GET_SPU_DATA J-{j} : start".format(j=j))
sql = "SELECT old_goods_id,spu_id,goods_id FROM lie_sku_%d WHERE supplier_id = 7" % j sql = "SELECT old_goods_id,spu_id,goods_id FROM lie_sku_%d WHERE supplier_id = 7" % j
......
param/zy_handler.py
...@@ -7,9 +7,7 @@ from elasticsearch.helpers import bulk ...@@ -7,9 +7,7 @@ from elasticsearch.helpers import bulk
import config.zy_config as zy_config import config.zy_config as zy_config
from config.db_config import DBConn from config.db_config import DBConn
from utils.date_handler import DateHandler from utils.date_handler import DateHandler
import pymysql
import json import json
import time
index = "goods_map" index = "goods_map"
...@@ -126,6 +124,7 @@ class ZY_HANDLER(BaseHandler): ...@@ -126,6 +124,7 @@ class ZY_HANDLER(BaseHandler):
if value != '': if value != '':
attrs.append({'attr_value': '%s€%s' % (key, value)}) attrs.append({'attr_value': '%s€%s' % (key, value)})
# FACE 10 COMPANY
ACTIONS.append( ACTIONS.append(
{ {
"_index": index, "_index": index,
......
param_match.py
# _*_ coding:utf-8 _*_ # _*_ coding:utf-8 _*_
from param.spu_extract import SPU_EXTRACT
from param.param_extract import PARAM_EXTRACT
from upload.param_mongo import ParamMongo from upload.param_mongo import ParamMongo
from param_task.db_attr_value_all import DbAttrValueAll from param_task.db_attr_value_all import DbAttrValueAll
from param.param_class_redis import PARAM_CLASS_REDIS
if __name__ == '__main__': if __name__ == '__main__':
# 更新 spu表-attr_values字段 # 更新 spu表-attr_values字段
# extract = SPU_EXTRACT() # extract = SPU_EXTRACT()
# extract.run() # extract.run()
# 更新 spu表-class_id字段 # # 更新 spu表-class_id字段
# param_extract = PARAM_EXTRACT() # param_extract = PARAM_EXTRACT()
# param_extract.run() # param_extract.run()
......
param_task/__pycache__/task.cpython-37.pyc 0 → 100644
No preview for this file type
param_task/db_attr_value_all.py
...@@ -16,7 +16,7 @@ class DbAttrValueAll(Task): ...@@ -16,7 +16,7 @@ class DbAttrValueAll(Task):
self.value_at = ValueTaskAll() self.value_at = ValueTaskAll()
self.unit_value = self.param_data.get_unit_value(self.class_db) self.unit_value = self.param_data.get_unit_value(self.class_db)
self.cls_map = self.param_data.class_mapping(self.class_db) self.cls_map = self.param_data.class_mapping(self.class_db)
self.class_list, self.param_dict = self.param_data.load_excel("/data2/param_data/dgk.xlsx") self.class_list, self.param_dict = self.param_data.load_excel("E://doc//dgk.xlsx")
self.dgk_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []} self.dgk_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []}
self.log1 = LogHandler('data_attr_value_summary', stream=True, file=True) self.log1 = LogHandler('data_attr_value_summary', stream=True, file=True)
self.log2 = LogHandler('data_attr_value', stream=False, file=True) self.log2 = LogHandler('data_attr_value', stream=False, file=True)
...@@ -30,7 +30,7 @@ class DbAttrValueAll(Task): ...@@ -30,7 +30,7 @@ class DbAttrValueAll(Task):
for i in range(0, 10): for i in range(0, 10):
self.log1.info("spu_data Database: {i}".format(i=i)) self.log1.info("spu_data Database: {i}".format(i=i))
# 获取sku连接 # 获取sku连接
conn = self.param_conn.online_sku_conn(i) conn = self.param_conn.local_sku_conn(i)
for j in range(0, 10): for j in range(0, 10):
data = self.param_data.get_sku_data(j, conn) data = self.param_data.get_sku_data(j, conn)
# self.log1.info("data Table: {table} length: {length}".format(table=j, length=len(data))) # self.log1.info("data Table: {table} length: {length}".format(table=j, length=len(data)))
...@@ -107,13 +107,15 @@ class DbAttrValueAll(Task): ...@@ -107,13 +107,15 @@ class DbAttrValueAll(Task):
break break
if len(dgk_list) > 0: if len(dgk_list) > 0:
time.sleep(0.1)
self.param_data.update_spu(int(spu_id), json.dumps(dgk_list), spu_db, self.log2) self.param_data.update_spu(int(spu_id), json.dumps(dgk_list), spu_db, self.log2)
if count % 3000 == 0: if count % 3000 == 0:
self.log1.info("INDEX: {INDEX}, COUNT: {COUNT}".format(INDEX=param_index, COUNT=count)) self.log1.info("INDEX: {INDEX}, COUNT: {COUNT}".format(INDEX=param_index, COUNT=count))
except: except:
self.log1.info("INDEX: {INDEX}, COUNT: {COUNT}".format(INDEX=param_index, COUNT=count)) self.log1.info("INDEX: {INDEX}, COUNT: {COUNT}".format(INDEX=param_index, COUNT=count))
"""
运行任务
"""
def run(self): def run(self):
self.init_data() self.init_data()
self.value_at.load_attr() self.value_at.load_attr()
......
param_task/value_task_all.py
...@@ -19,11 +19,11 @@ class ValueTaskAll(Task): ...@@ -19,11 +19,11 @@ class ValueTaskAll(Task):
def load_attr(self): def load_attr(self):
self.log1.info("加载参数") self.log1.info("加载参数")
# 有效参数 # 有效参数
effect = ExcelHandler.read_to_excel("/data2/param_data/dgk.xlsx", "Sheet1", 1) effect = ExcelHandler.read_to_excel("E://doc//dgk.xlsx", "Sheet1", 1)
effect.pop(0) effect.pop(0)
[self.attrs_id_effect.append(int(row[0])) for row in effect] [self.attrs_id_effect.append(int(row[0])) for row in effect]
# 全部参数 # 全部参数
all_data = ExcelHandler.read_to_excel("/data2/param_data/dgk_all.xlsx", "Sheet1", 1) all_data = ExcelHandler.read_to_excel("E://doc//dgk_all.xlsx", "Sheet1", 1)
all_data.pop(0) all_data.pop(0)
for row in all_data: for row in all_data:
# 参数id、二级分类id、基础参数名、猎芯参数名 # 参数id、二级分类id、基础参数名、猎芯参数名
......
upload/bom_mongo.py 0 → 100644
# _*_ coding:utf-8 _*_
from urllib import parse
import traceback
import pymongo
from fun.param_data import ParamData
from param_task.task import Task
from utils.db_handler import DBHandler
from utils.log_handler import LogHandler
from fun.unit_trans import UnitTrans
brand_res_map = {
1: 'Yageo',
2: 'Murata',
3: 'Samsung',
4: 'TDK',
5: 'Taiyo Yuden',
6: 'Rohm',
7: 'KEMET',
8: 'Vishay',
9: 'AVX',
10: 'Panasonic',
11: 'KOA',
12: 'Bourns',
13: 'TE',
14: 'Wurth',
15: 'Pulse'
}
# 转义用户名和密码
user = parse.quote_plus("ichunt")
pwd = parse.quote_plus("huntmon6699")
# pwd = parse.quote_plus("huntmon66499")
myclient = pymongo.MongoClient("mongodb://{0}:{1}@192.168.1.237:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
# myclient = pymongo.MongoClient("mongodb://{0}:{1}@172.18.137.23:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
mydb = myclient['ichunt']
mycol = mydb["spu_attrs"]
class BomMongo(Task):
def __init__(self):
super().__init__()
self.unit_trans = UnitTrans()
self.spu_conn = self.DBConn.db_spu()
self.learn_conn = self.DBConn.db_learning()
self.log1 = LogHandler('param_mongo', stream=True, file=True)
self.param_data = ParamData
self.data_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []}
def search_mongo(self, spu_id, data, brand_name):
res = mycol.find({"spu_id": spu_id}).limit(1)
res = list(res)
# if len(res) == 0:
# insert_data = {'spu_id': spu_id, 'attrs_format': data}
# mycol.insert_one(insert_data)
# else:
if len(res) > 0:
mg_dt = res[0]
mg_dt['attrs_format'] = data
mg_dt['brand_name'] = brand_name
mycol.save(mg_dt)
def getBomData(self):
sql = "SELECT spu_id,resistance,capacity,inductance,rated_voltage,rated_current,power,dc_resistance,accuracy,\
encap,temperature,brand_sort,brand_name FROM lie_goods_attrs WHERE spu_id != 0"
return DBHandler.read(self.learn_conn, sql)
def dealData(self, data):
print(len(data))
index = 0
for row in data:
try:
index += 1
attrs = []
spu_id = row[0]
resistance = row[1]
capacity = row[2]
inductance = row[3]
rated_voltage = row[4]
rated_current = row[5]
power = row[6]
dc_resistance = row[7]
accuracy = row[8]
encap = row[9]
temperature = row[10]
brand_name = brand_res_map[int(row[11])] if brand_res_map.get(int(row[11])) else row[12]
if resistance != '' and str(resistance).startswith(".") is not True:
ts_value = self.unit_trans.trans_resistance(resistance)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "阻值（欧姆）",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "Ω"})
if capacity != '' and str(capacity).startswith(".") is not True:
ts_value = self.unit_trans.trans_capacity(capacity)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "容值",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "pF"})
if inductance != '' and str(inductance).startswith(".") is not True:
ts_value = self.unit_trans.trans_inductance(inductance)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "电感值",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "pH"})
if rated_voltage != '' and str(rated_voltage).startswith(".") is not True:
ts_value = self.unit_trans.trans_rated_voltage(rated_voltage)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "额定电压",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "V"})
if rated_current != '' and str(rated_current).startswith(".") is not True:
ts_value = self.unit_trans.trans_rated_current(rated_current)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "额定电流",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "mA"})
if power != '' and str(power).startswith(".") is not True:
ts_value = self.unit_trans.trans_power(power)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "功率",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "W"})
if dc_resistance != '' and str(dc_resistance).startswith(".") is not True:
ts_value = self.unit_trans.trans_dc_resistance(dc_resistance)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "直流电阻（内阻）",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": "Ω"})
if accuracy != '' and str(accuracy).startswith(".") is not True:
ts_value = self.unit_trans.trans_accuracy(accuracy)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "精度",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": ""})
if encap != '' and str(encap).startswith(".") is not True:
ts_value = self.unit_trans.trans_encap(encap)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "封装",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": ""})
if temperature != '' and str(temperature).startswith(".") is not True:
ts_value = self.unit_trans.trans_temp(temperature)
if ts_value["key"] != "" and ts_value["ts_value"] != "":
attrs.append({"attr_name": "温漂系数（介质材料)",
"attr_origin": ts_value["key"],
"attr_format": ts_value["ts_value"],
"attr_unit": ""})
if len(attrs) > 0:
self.search_mongo(spu_id, attrs, brand_name)
if index % 1000 == 0:
print(index)
except:
traceback.print_exc()
def run(self):
data = self.getBomData()
self.dealData(data)
print(len(data))
if __name__ == '__main__':
bom = BomMongo()
bom.run()
upload/param_mongo.py
...@@ -6,16 +6,18 @@ from urllib import parse ...@@ -6,16 +6,18 @@ from urllib import parse
from utils.log_handler import LogHandler from utils.log_handler import LogHandler
from fun.param_thread import ParamThread from fun.param_thread import ParamThread
from fun.param_data import ParamData from fun.param_data import ParamData
from utils.db_handler import DBHandler
from fun.param_conn import ParamConn
from param_task.task import Task from param_task.task import Task
# 转义用户名和密码 # 转义用户名和密码
user = parse.quote_plus("ichunt") user = parse.quote_plus("ichunt")
# pwd = parse.quote_plus("huntmon6699") pwd = parse.quote_plus("huntmon6699")
pwd = parse.quote_plus("huntmon66499") # pwd = parse.quote_plus("huntmon66499")
# myclient = pymongo.MongoClient("mongodb://{0}:{1}@192.168.1.237:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd)) myclient = pymongo.MongoClient("mongodb://{0}:{1}@192.168.1.237:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
myclient = pymongo.MongoClient("mongodb://{0}:{1}@172.18.137.23:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd)) # myclient = pymongo.MongoClient("mongodb://{0}:{1}@172.18.137.23:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
mydb = myclient['ichunt'] mydb = myclient['ichunt']
mycol = mydb["spu_attrs"] mycol = mydb["spu_attrs"]
...@@ -29,6 +31,20 @@ class ParamMongo(Task): ...@@ -29,6 +31,20 @@ class ParamMongo(Task):
self.param_data = ParamData self.param_data = ParamData
self.data_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []} self.data_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []}
def search_sku_id(self, spu_id):
for i in range(0, 10):
# 获取sku 连接
# conn = ParamConn.online_sku_conn(i)
conn = ParamConn.local_sku_conn(i)
for j in range(0, 10):
table = "lie_sku_%d" % j
sql = "SELECT spu_id,goods_id FROM %s WHERE supplier_id = 7 and spu_id=%s" % (table, spu_id)
# print(sql)
db_data = DBHandler.read(conn, sql)
if len(db_data) > 0:
print("spu_id: %s sku_id: %s" % (spu_id, db_data[0][1]))
break
""" """
初始化数据 初始化数据
""" """
...@@ -39,8 +55,10 @@ class ParamMongo(Task): ...@@ -39,8 +55,10 @@ class ParamMongo(Task):
self.data_dict[i] = self.param_data.get_spu(i, self.spu_conn) self.data_dict[i] = self.param_data.get_spu(i, self.spu_conn)
def search_mongo(self, spu_id, data): def search_mongo(self, spu_id, data):
self.search_sku_id(spu_id)
res = mycol.find({"spu_id": spu_id}).limit(1) res = mycol.find({"spu_id": spu_id}).limit(1)
res = list(res) res = list(res)
print(data)
if len(res) == 0: if len(res) == 0:
insert_data = {'spu_id': spu_id, 'attrs_extend': data} insert_data = {'spu_id': spu_id, 'attrs_extend': data}
mycol.insert_one(insert_data) mycol.insert_one(insert_data)
...@@ -53,6 +71,7 @@ class ParamMongo(Task): ...@@ -53,6 +71,7 @@ class ParamMongo(Task):
count = 0 count = 0
self.log1.info("update_mongo : start") self.log1.info("update_mongo : start")
class_db = self.DBConn.db_class() class_db = self.DBConn.db_class()
print(len(data))
for row in data: for row in data:
try: try:
count += 1 count += 1
...@@ -65,7 +84,7 @@ class ParamMongo(Task): ...@@ -65,7 +84,7 @@ class ParamMongo(Task):
attr_name = self.param_data.get_attr_name(val[0], class_db) attr_name = self.param_data.get_attr_name(val[0], class_db)
attr_value = self.param_data.get_attr_value(val[1], class_db) attr_value = self.param_data.get_attr_value(val[1], class_db)
attr_unit = self.param_data.get_attr_unit(val[2], class_db) attr_unit = self.param_data.get_attr_unit(val[2], class_db)
if attr_name and attr_value and attr_unit: if attr_name and attr_value:
new_av_list.append({'attr_name': attr_name, 'attr_value': attr_value, 'attr_unit': attr_unit}) new_av_list.append({'attr_name': attr_name, 'attr_value': attr_value, 'attr_unit': attr_unit})
self.search_mongo(spu_id, new_av_list) self.search_mongo(spu_id, new_av_list)
...@@ -76,7 +95,9 @@ class ParamMongo(Task): ...@@ -76,7 +95,9 @@ class ParamMongo(Task):
def run(self): def run(self):
self.init_data() self.init_data()
ParamThread.thread_func_ten(self.update_mongo, self.data_dict) for i in self.data_dict:
self.update_mongo(self.data_dict[i])
# ParamThread.thread_func_ten(self.update_mongo, self.data_dict)
if __name__ == '__main__': if __name__ == '__main__':
......
utils/__pycache__/db_handler.cpython-37.pyc 0 → 100644
No preview for this file type
utils/__pycache__/log_handler.cpython-37.pyc 0 → 100644
No preview for this file type
utils/db_handler.py
...@@ -58,7 +58,7 @@ class DBHandler: ...@@ -58,7 +58,7 @@ class DBHandler:
db.commit() db.commit()
except: except:
db.rollback() db.rollback()
traceback.print_exc() # traceback.print_exc()
# print(sql) # print(sql)
""" """
......
utils/mysql_handler.py
...@@ -67,7 +67,7 @@ class MysqlHandler(object): ...@@ -67,7 +67,7 @@ class MysqlHandler(object):
# SELECT spu_id,attr_values FROM lie_spu_{0} WHERE attr_values != '' AND attr_values != '[]' # SELECT spu_id,attr_values FROM lie_spu_{0} WHERE attr_values != '' AND attr_values != '[]'
# """.format(index) # """.format(index)
sql = "SELECT spu_id,attr_values FROM lie_spu_%d WHERE class_id1 != 0 AND class_id2 != 0 \ sql = "SELECT spu_id,attr_values FROM lie_spu_%d WHERE class_id1 != 0 AND class_id2 != 0 \
AND attr_values != '' ORDER BY spu_id LIMIT 100 " % index AND attr_values != '' LIMIT 1000" % index
return DBHandler.read(self.db, sql) return DBHandler.read(self.db, sql)
def get_attr_name(self, attr_id): def get_attr_name(self, attr_id):
...@@ -77,6 +77,7 @@ class MysqlHandler(object): ...@@ -77,6 +77,7 @@ class MysqlHandler(object):
FROM lie_class_attr FROM lie_class_attr
WHERE attr_id=%s WHERE attr_id=%s
""" """
print(sql)
cursor.execute(sql, attr_id) cursor.execute(sql, attr_id)
result = cursor.fetchone() result = cursor.fetchone()
return result[0] return result[0]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment