修改redis处理

param/param_class_redis.py

@@ -2,22 +2,29 @@
 import redis
 import pymysql
 import json
+import re
 from config.db_config import DBConn
 from utils.log_handler import LogHandler
 from utils.db_handler import DBHandler
+import traceback
+import codecs
 
+import config.db_config as config
 
-class ParamClassRedis(object):
 
+class PARAM_CLASS_REDIS(object):
     """
     参数提取
     处理class_id
     """
+
     def __init__(self):
         self.db = DBConn
         self.log = LogHandler('param_extract')
+        self.log2 = LogHandler('param_class_redis', stream=False, file=True)
         # self.redis = redis.StrictRedis(host="192.168.1.235", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8", encoding="UTF-8")
-        self.redis = redis.StrictRedis(host="172.18.137.38", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8", encoding="UTF-8")
+        self.redis = redis.StrictRedis(host="172.18.137.38", port=6379, db=0, password="icDb29mLy2s", charset="UTF-8",
+                                       encoding="UTF-8")
 
     def search_sku(self, spu_id):
         for i in range(0, 10):
@@ -34,33 +41,51 @@ class ParamClassRedis(object):
                     print("spu_id: {spu_id}, goods_id: {goods_id}".format(spu_id=spu_id, goods_id=res[0][1]))
 
     def spu_data(self):
-        for i in range(0, 10):
+        for i in range(1, 10):
             # TODO 临时变更
             # sql = "SELECT spu_id,class_id1,class_id2 FROM lie_spu_%d WHERE class_id1 != 0 AND class_id2 != 0 AND attr_values != ''" % i
-            sql = "SELECT spu_id,class_id1,class_id2 FROM lie_spu_%d WHERE class_id1 != 0 AND class_id2 != 0 \
-                    AND attr_values != '' ORDER BY spu_id LIMIT 100 " % i
+            sql = "SELECT spu_id,class_id1,class_id2,images_l,pdf,spu_brief FROM lie_spu_%d WHERE class_id1 != 0 AND class_id2 != 0" % i
             res = DBHandler.read(self.db.db_spu(), sql)
             print(len(res))
+            index = 0
             for row in res:
                 spu_id = row[0]
                 class_id1 = row[1]
                 class_id2 = row[2]
+                # images_l = str(row[3]).replace(r"/", r"\/") if "\\/" not in str(row[3]) else str(row[3])
+                # pdf = str(row[4]).replace(r"/", r"\/") if "\\/" not in str(row[4]) else str(row[4])
+                images_l = row[3]
+                pdf = row[4]
+                spu_brief = row[5]
                 spu_res = self.redis.hget("spu", str(spu_id))
                 if spu_res:
-                    spu_res = eval(spu_res)
-                    spu_res['class_id1'] = class_id1
-                    spu_res['class_id2'] = class_id2
-                    # print(spu_res)
-                    self.redis.hset("spu", str(spu_id), json.dumps(spu_res))
+                    try:
+                        index += 1
+                        spu_res = eval(spu_res)
+                        spu_res['class_id1'] = class_id1
+                        spu_res['class_id2'] = class_id2
+                        spu_res['images_l'] = images_l
+                        spu_res['pdf'] = pdf
+                        spu_res['spu_brief'] = spu_brief
+                        spu_res = json.dumps(spu_res)
+                        spu_res = spu_res.replace("\\\\/", "\\/").replace("\\\\\\/", "\\/")
+                        self.redis.hset("spu", str(spu_id), spu_res)
+                        self.log2.info("spu_id: {spu_id}, spu_res: {spu_res}".format(spu_id=spu_id, spu_res=spu_res))
+                    except:
+                        # traceback.print_exc()
+                        pass
                     # self.log.info("spu_id: {spu_id}".format(spu_id=spu_id))
-                    self.search_sku(spu_id)
+                    # self.search_sku(spu_id)
                 else:
                     continue
 
+                if index % 10000 == 0:
+                    self.log.info("run - index: {index}".format(index=index))
+
     def run(self):
         self.spu_data()
 
 
 if __name__ == "__main__":
-    param = ParamClassRedis()
-    param.spu_data()
\ No newline at end of file
+    param = PARAM_CLASS_REDIS()
+    param.spu_data()