添加处理全部参数逻辑

fun/param_data.py

@@ -154,6 +154,12 @@ class ParamData(object):
                 attr_id IN (%s) AND attr_value != ''" % (i, ids, pams)
         return DBHandler.read(bg_conn, sql)
 
+    @staticmethod
+    def get_dgk_goods_by_index(idx, bg_conn):
+        sql = "SELECT attr_name,attr_value,attr_id,cat_id FROM lie_goods_attr_fields%d WHERE \
+                attr_value != '' AND attr_value != '-' LIMIT 10000" % (idx)
+        return DBHandler.read(bg_conn, sql)
+
     """
     写入attr value
     """

param/param_class_redis.py

@@ -5,10 +5,9 @@ import json
 from config.db_config import DBConn
 from utils.log_handler import LogHandler
 from utils.db_handler import DBHandler
-import config.db_config as config
 
 
-class PARAM_CLASS_REDIS(object):
+class ParamClassRedis(object):
 
     """
     参数提取
@@ -63,5 +62,5 @@ class PARAM_CLASS_REDIS(object):
 
 
 if __name__ == "__main__":
-    param = PARAM_CLASS_REDIS()
+    param = ParamClassRedis()
     param.spu_data()
\ No newline at end of file

param/param_extract.py

@@ -2,8 +2,8 @@
 from config.db_config import DBConn
 from utils.log_handler import LogHandler
 from utils.db_handler import DBHandler
-from fun.param_conn import PARAM_CONN
-from fun.param_data import PARAM_DATA
+from fun.param_conn import ParamConn
+from fun.param_data import ParamData
 import time
 
 
@@ -28,7 +28,7 @@ class PARAM_EXTRACT(object):
         for i in range(0, 10):
             db = "liexin_sku_%d" % i
             # 获取sku 连接
-            conn = PARAM_CONN.online_sku_conn(i)
+            conn = ParamConn.online_sku_conn(i)
             for j in range(0, 10):
                 table = "lie_sku_%d" % j
                 sql = "SELECT spu_id,old_goods_id FROM %s WHERE supplier_id = 7" % table
@@ -52,23 +52,22 @@ class PARAM_EXTRACT(object):
         bd_conn = DBConn.db_bigdata()
 
         # 数据
-        class_data = PARAM_DATA.class_data(class_conn)
+        class_data = ParamData.class_data(class_conn)
 
         # dgk数据遍历
         index = 0
         for i in self.dgk_dict:
             dgk_data = self.dgk_dict[i]
-            if i == 5:
-                for spu_id, old_goods_id in dgk_data:
-                    cat_id, sku_id = PARAM_DATA.search_cat_id(old_goods_id, bd_conn)
-                    if cat_id != 0:
-                        parent_id = class_data[cat_id]['parent_id']
-                        class_id = class_data[cat_id]['class_id']
-                        time.sleep(0.01)
-                        PARAM_DATA.update_spu_classId(int(spu_id), int(class_id), int(parent_id), spu_conn, self.log2)
-                        index += 1
-                        if index % 10000 == 0:
-                            self.log1.info("run - index: {index}".format(index=index))
+            for spu_id, old_goods_id in dgk_data:
+                cat_id, sku_id = ParamData.search_cat_id(old_goods_id, bd_conn)
+                if cat_id != 0:
+                    parent_id = class_data[cat_id]['parent_id']
+                    class_id = class_data[cat_id]['class_id']
+                    time.sleep(0.01)
+                    ParamData.update_spu_classId(int(spu_id), int(class_id), int(parent_id), spu_conn, self.log2)
+                    index += 1
+                    if index % 10000 == 0:
+                        self.log1.info("run - index: {index}".format(index=index))
 
     """
     运行
@@ -76,4 +75,3 @@ class PARAM_EXTRACT(object):
     def run(self):
         self.dgk_data()
         self.update_classId()
-        # PARAM_THREAD.thread_func_ten(self.update_classId, self.dgk_dict)
\ No newline at end of file

param_task/task.py

+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+from utils.date_handler import DateHandler
+from fun.param_data import ParamData
+from fun.param_trans import ParamTrans
+from fun.param_conn import ParamConn
+from config.db_config import DBConn
+
+
+class Task:
+
+    def __init__(self):
+        self.now = DateHandler.now_datetime()
+        self.param_data = ParamData
+        self.param_trans = ParamTrans()
+        self.class_db = DBConn.db_class()
+        self.bigdata_db = DBConn.db_bigdata()
+        self.redis = ParamConn.local_redis_conn()
\ No newline at end of file

param_task/value_task.py

@@ -25,7 +25,6 @@ class ValueTask:
         self.log1 = LogHandler('value_task_info', stream=True, file=True)
         self.log2 = LogHandler('value_task_detail', stream=False, file=True)
         self.now = DateHandler.now_datetime()
-        self.empty_unit_id = 26
 
     """
     加载数据
@@ -66,7 +65,7 @@ class ValueTask:
         mapping = self.param_data.class_mapping(self.class_db)
         unit_dict = self.param_data.get_unit_value(self.class_db)
 
-        for i in range(1, 10):
+        for i in range(0, 10):
             res = self.param_data.get_dgk_goods_by_ids(i, ids, pams, self.bigdata_db)
             for row in res:
                 attr_name = row[0]
@@ -77,7 +76,7 @@ class ValueTask:
                     attr_id = mapping[cat_id][attr_name]
                     value, unit = getattr(ParamTrans, self.param_dict[str(fields_attr_id)])(attr_value)
                     if value != 0:
-                        unit_id = unit_dict[unit] if unit_dict.get(unit) else self.empty_unit_id
+                        unit_id = unit_dict[unit] if unit_dict.get(unit) else self.param_trans.empty_unit_id
                         # redis去重
                         if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0:
                             self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now,

param_task/value_task_all.py

@@ -2,17 +2,14 @@
 # !/usr/bin/env python
 from utils.log_handler import LogHandler
 from utils.excel_handler import ExcelHandler
-from fun.param_data import ParamData
-from fun.param_trans import ParamTrans
-from config.db_config import DBConn
+from param_task.task import Task
 
 
-class ValueTaskAll:
+class ValueTaskAll(Task):
 
     def __init__(self):
-        self.param_data = ParamData
-        self.param_trans = ParamTrans()
-        self.class_db = DBConn.db_class()
+        super().__init__()
+        self.attrs_id = []
         self.log1 = LogHandler('value_task_all_info', stream=True, file=True)
 
     """
@@ -41,11 +38,36 @@ class ValueTaskAll:
             if is_important == 1 and param_id not in effect_attrs:
                 class_id = self.param_data.get_class_id(second_classify_id, self.class_db)
                 unit_id = self.param_trans.empty_unit_id
+                self.attrs_id.append(param_id)
                 lx_attr_name = ''
-                ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db)
+                # ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db)
+
+    """
+    写入所有数据
+    """
+    def write_all(self):
+        self.log1.info("运行任务")
+        mapping = self.param_data.class_mapping(self.class_db)
+        print(len(mapping))
+        for i in range(0, 10):
+            res = self.param_data.get_dgk_goods_by_index(i, self.bigdata_db)
+            for row in res:
+                attr_name = row[0]
+                attr_value = row[1]
+                fields_attr_id = row[2]
+                cat_id = row[3]
+                if mapping[cat_id].get(attr_name) and fields_attr_id in self.attrs_id:
+                    attr_id = mapping[cat_id][attr_name]
+                    value = attr_value
+                    unit_id = self.param_trans.empty_unit_id
+                    print(attr_id, value, unit_id)
+                    # if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0:
+                    #     self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now,
+                    #                                      self.class_db)
 
     def run(self):
         self.load_attr()
+        self.write_all()
 
 
 if __name__ == '__main__':