Commit dc6bfb23 by lzzzzl

更新attr_values mongo处理

parent 20f6fa41
Showing with 18 additions and 11 deletions
# _*_ coding:utf-8 _*_
from upload.param_mongo import ParamMongo
from param_task.db_attr_value_all import DbAttrValueAll
if __name__ == '__main__':
......@@ -16,8 +17,8 @@ if __name__ == '__main__':
# param_redis.run()
# 更新mongo
# param_mongo = PARAM_MONGO()
# param_mongo.run()
param_mongo = ParamMongo()
param_mongo.run()
db_value = DbAttrValueAll()
db_value.run()
\ No newline at end of file
# db_value = DbAttrValueAll()
# db_value.run()
\ No newline at end of file
......@@ -3,6 +3,7 @@ import json
import traceback
import pymongo
from urllib import parse
from utils.log_handler import LogHandler
from fun.param_thread import ParamThread
from fun.param_data import ParamData
from param_task.task import Task
......@@ -10,11 +11,11 @@ from param_task.task import Task
# 转义用户名和密码
user = parse.quote_plus("ichunt")
pwd = parse.quote_plus("huntmon6699")
# pwd = parse.quote_plus("huntmon66499")
# pwd = parse.quote_plus("huntmon6699")
pwd = parse.quote_plus("huntmon66499")
myclient = pymongo.MongoClient("mongodb://{0}:{1}@192.168.1.237:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
# myclient = pymongo.MongoClient("mongodb://{0}:{1}@172.18.137.23:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
# myclient = pymongo.MongoClient("mongodb://{0}:{1}@192.168.1.237:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
myclient = pymongo.MongoClient("mongodb://{0}:{1}@172.18.137.23:27017/?authSource=ichunt&authMechanism=SCRAM-SHA-1".format(user, pwd))
mydb = myclient['ichunt']
mycol = mydb["spu_attrs"]
......@@ -24,6 +25,7 @@ class ParamMongo(Task):
def __init__(self):
super().__init__()
self.spu_conn = self.DBConn.db_spu()
self.log1 = LogHandler('param_mongo', stream=True, file=True)
self.param_data = ParamData
self.data_dict = {0: [], 1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: []}
......@@ -31,6 +33,7 @@ class ParamMongo(Task):
初始化数据
"""
def init_data(self):
self.log1.info("init_data : start")
# 遍历SKU数据
for i in range(0, 10):
self.data_dict[i] = self.param_data.get_spu(i, self.spu_conn)
......@@ -48,12 +51,14 @@ class ParamMongo(Task):
def update_mongo(self, data):
count = 0
self.log1.info("update_mongo : start")
class_db = self.DBConn.db_class()
for row in data:
try:
count += 1
spu_id = row[0]
attr_values = json.loads(row[1])
param_index = int(str(spu_id)[-1])
new_av_list = list()
for vals in attr_values:
val = str(vals).split(",")
......@@ -63,8 +68,9 @@ class ParamMongo(Task):
if attr_name and attr_value and attr_unit:
new_av_list.append({'attr_name': attr_name, 'attr_value': attr_value, 'attr_unit': attr_unit})
self.search_mongo(spu_id, new_av_list)
if count % 10000 == 0:
print(count)
if count % 3000 == 0:
self.log1.info("INDEX: {INDEX}, COUNT: {COUNT}".format(INDEX=param_index, COUNT=count))
except:
traceback.print_exc()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment