Skip to content

lzzzzl / param_data

Go to a project
Commit 04a07679 by lzzzzl
Options

添加处理全部参数逻辑

parent 5f74c23a
Inline Side-by-side
Showing with 74 additions and 31 deletions
fun/param_data.py
...@@ -154,6 +154,12 @@ class ParamData(object): ...@@ -154,6 +154,12 @@ class ParamData(object):
attr_id IN (%s) AND attr_value != ''" % (i, ids, pams) attr_id IN (%s) AND attr_value != ''" % (i, ids, pams)
return DBHandler.read(bg_conn, sql) return DBHandler.read(bg_conn, sql)
@staticmethod
def get_dgk_goods_by_index(idx, bg_conn):
sql = "SELECT attr_name,attr_value,attr_id,cat_id FROM lie_goods_attr_fields%d WHERE \
attr_value != '' AND attr_value != '-' LIMIT 10000" % (idx)
return DBHandler.read(bg_conn, sql)
""" """
写入attr value 写入attr value
""" """
......
param/param_class_redis.py
...@@ -5,10 +5,9 @@ import json ...@@ -5,10 +5,9 @@ import json
from config.db_config import DBConn from config.db_config import DBConn
from utils.log_handler import LogHandler from utils.log_handler import LogHandler
from utils.db_handler import DBHandler from utils.db_handler import DBHandler
import config.db_config as config
class PARAM_CLASS_REDIS(object): class ParamClassRedis(object):
""" """
参数提取 参数提取
...@@ -63,5 +62,5 @@ class PARAM_CLASS_REDIS(object): ...@@ -63,5 +62,5 @@ class PARAM_CLASS_REDIS(object):
if __name__ == "__main__": if __name__ == "__main__":
param = PARAM_CLASS_REDIS() param = ParamClassRedis()
param.spu_data() param.spu_data()
\ No newline at end of file
param/param_extract.py
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
from config.db_config import DBConn from config.db_config import DBConn
from utils.log_handler import LogHandler from utils.log_handler import LogHandler
from utils.db_handler import DBHandler from utils.db_handler import DBHandler
from fun.param_conn import PARAM_CONN from fun.param_conn import ParamConn
from fun.param_data import PARAM_DATA from fun.param_data import ParamData
import time import time
...@@ -28,7 +28,7 @@ class PARAM_EXTRACT(object): ...@@ -28,7 +28,7 @@ class PARAM_EXTRACT(object):
for i in range(0, 10): for i in range(0, 10):
db = "liexin_sku_%d" % i db = "liexin_sku_%d" % i
# 获取sku 连接 # 获取sku 连接
conn = PARAM_CONN.online_sku_conn(i) conn = ParamConn.online_sku_conn(i)
for j in range(0, 10): for j in range(0, 10):
table = "lie_sku_%d" % j table = "lie_sku_%d" % j
sql = "SELECT spu_id,old_goods_id FROM %s WHERE supplier_id = 7" % table sql = "SELECT spu_id,old_goods_id FROM %s WHERE supplier_id = 7" % table
...@@ -52,23 +52,22 @@ class PARAM_EXTRACT(object): ...@@ -52,23 +52,22 @@ class PARAM_EXTRACT(object):
bd_conn = DBConn.db_bigdata() bd_conn = DBConn.db_bigdata()
# 数据 # 数据
class_data = PARAM_DATA.class_data(class_conn) class_data = ParamData.class_data(class_conn)
# dgk数据遍历 # dgk数据遍历
index = 0 index = 0
for i in self.dgk_dict: for i in self.dgk_dict:
dgk_data = self.dgk_dict[i] dgk_data = self.dgk_dict[i]
if i == 5: for spu_id, old_goods_id in dgk_data:
for spu_id, old_goods_id in dgk_data: cat_id, sku_id = ParamData.search_cat_id(old_goods_id, bd_conn)
cat_id, sku_id = PARAM_DATA.search_cat_id(old_goods_id, bd_conn) if cat_id != 0:
if cat_id != 0: parent_id = class_data[cat_id]['parent_id']
parent_id = class_data[cat_id]['parent_id'] class_id = class_data[cat_id]['class_id']
class_id = class_data[cat_id]['class_id'] time.sleep(0.01)
time.sleep(0.01) ParamData.update_spu_classId(int(spu_id), int(class_id), int(parent_id), spu_conn, self.log2)
PARAM_DATA.update_spu_classId(int(spu_id), int(class_id), int(parent_id), spu_conn, self.log2) index += 1
index += 1 if index % 10000 == 0:
if index % 10000 == 0: self.log1.info("run - index: {index}".format(index=index))
self.log1.info("run - index: {index}".format(index=index))
""" """
运行 运行
...@@ -76,4 +75,3 @@ class PARAM_EXTRACT(object): ...@@ -76,4 +75,3 @@ class PARAM_EXTRACT(object):
def run(self): def run(self):
self.dgk_data() self.dgk_data()
self.update_classId() self.update_classId()
# PARAM_THREAD.thread_func_ten(self.update_classId, self.dgk_dict)
\ No newline at end of file
param_task/task.py 0 → 100644
# -*- coding: utf-8 -*-
# !/usr/bin/env python
from utils.date_handler import DateHandler
from fun.param_data import ParamData
from fun.param_trans import ParamTrans
from fun.param_conn import ParamConn
from config.db_config import DBConn
class Task:
def __init__(self):
self.now = DateHandler.now_datetime()
self.param_data = ParamData
self.param_trans = ParamTrans()
self.class_db = DBConn.db_class()
self.bigdata_db = DBConn.db_bigdata()
self.redis = ParamConn.local_redis_conn()
\ No newline at end of file
param_task/value_task.py
...@@ -25,7 +25,6 @@ class ValueTask: ...@@ -25,7 +25,6 @@ class ValueTask:
self.log1 = LogHandler('value_task_info', stream=True, file=True) self.log1 = LogHandler('value_task_info', stream=True, file=True)
self.log2 = LogHandler('value_task_detail', stream=False, file=True) self.log2 = LogHandler('value_task_detail', stream=False, file=True)
self.now = DateHandler.now_datetime() self.now = DateHandler.now_datetime()
self.empty_unit_id = 26
""" """
加载数据 加载数据
...@@ -66,7 +65,7 @@ class ValueTask: ...@@ -66,7 +65,7 @@ class ValueTask:
mapping = self.param_data.class_mapping(self.class_db) mapping = self.param_data.class_mapping(self.class_db)
unit_dict = self.param_data.get_unit_value(self.class_db) unit_dict = self.param_data.get_unit_value(self.class_db)
for i in range(1, 10): for i in range(0, 10):
res = self.param_data.get_dgk_goods_by_ids(i, ids, pams, self.bigdata_db) res = self.param_data.get_dgk_goods_by_ids(i, ids, pams, self.bigdata_db)
for row in res: for row in res:
attr_name = row[0] attr_name = row[0]
...@@ -77,7 +76,7 @@ class ValueTask: ...@@ -77,7 +76,7 @@ class ValueTask:
attr_id = mapping[cat_id][attr_name] attr_id = mapping[cat_id][attr_name]
value, unit = getattr(ParamTrans, self.param_dict[str(fields_attr_id)])(attr_value) value, unit = getattr(ParamTrans, self.param_dict[str(fields_attr_id)])(attr_value)
if value != 0: if value != 0:
unit_id = unit_dict[unit] if unit_dict.get(unit) else self.empty_unit_id unit_id = unit_dict[unit] if unit_dict.get(unit) else self.param_trans.empty_unit_id
# redis去重 # redis去重
if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0: if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0:
self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now, self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now,
......
param_task/value_task_all.py
...@@ -2,17 +2,14 @@ ...@@ -2,17 +2,14 @@
# !/usr/bin/env python # !/usr/bin/env python
from utils.log_handler import LogHandler from utils.log_handler import LogHandler
from utils.excel_handler import ExcelHandler from utils.excel_handler import ExcelHandler
from fun.param_data import ParamData from param_task.task import Task
from fun.param_trans import ParamTrans
from config.db_config import DBConn
class ValueTaskAll: class ValueTaskAll(Task):
def __init__(self): def __init__(self):
self.param_data = ParamData super().__init__()
self.param_trans = ParamTrans() self.attrs_id = []
self.class_db = DBConn.db_class()
self.log1 = LogHandler('value_task_all_info', stream=True, file=True) self.log1 = LogHandler('value_task_all_info', stream=True, file=True)
""" """
...@@ -41,11 +38,36 @@ class ValueTaskAll: ...@@ -41,11 +38,36 @@ class ValueTaskAll:
if is_important == 1 and param_id not in effect_attrs: if is_important == 1 and param_id not in effect_attrs:
class_id = self.param_data.get_class_id(second_classify_id, self.class_db) class_id = self.param_data.get_class_id(second_classify_id, self.class_db)
unit_id = self.param_trans.empty_unit_id unit_id = self.param_trans.empty_unit_id
self.attrs_id.append(param_id)
lx_attr_name = '' lx_attr_name = ''
ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db) # ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db)
"""
写入所有数据
"""
def write_all(self):
self.log1.info("运行任务")
mapping = self.param_data.class_mapping(self.class_db)
print(len(mapping))
for i in range(0, 10):
res = self.param_data.get_dgk_goods_by_index(i, self.bigdata_db)
for row in res:
attr_name = row[0]
attr_value = row[1]
fields_attr_id = row[2]
cat_id = row[3]
if mapping[cat_id].get(attr_name) and fields_attr_id in self.attrs_id:
attr_id = mapping[cat_id][attr_name]
value = attr_value
unit_id = self.param_trans.empty_unit_id
print(attr_id, value, unit_id)
# if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0:
# self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now,
# self.class_db)
def run(self): def run(self):
self.load_attr() self.load_attr()
self.write_all()
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment