Skip to content

岳巧源 / jd_data_process

Go to a project
Commit 0c8c6ae3 by larosa
Options

add py script

parent 6ebfed49
Inline Side-by-side
Showing with 96 additions and 56 deletions
push_sku_to_nplm.py
# TODO push the data of liexin to go_nplm_server. # # TODO push the data of liexin to go_nplm_server.
# 首先从234的redis中获取 supplier_id, brand_id的相关信息 # # 首先从234的redis中获取 supplier_id, brand_id的相关信息
# 根据获取到的信息去sku数据库中筛选 符合条件的数据信息（包含spu_id） # # 根据获取到的信息去sku数据库中筛选 符合条件的数据信息（包含spu_id）
# 在redis中根据spu_id 获取到相关的分类信息等等。 # # 在redis中根据spu_id 获取到相关的分类信息等等。
# 映射组装后推送到 go_nplm_server处理 # # 映射组装后推送到 go_nplm_server处理
import json # import json
#
import pymysql # import pymysql
import redis # import redis
#
db = redis.Redis(host='192.168.1.235', port=6379, password='icDb29mLy2s', decode_responses=True) # db = redis.Redis(host='192.168.1.235', port=6379, password='icDb29mLy2s', decode_responses=True)
db_234 = redis.Redis(host='192.168.1.234', port=6379, password='icDb29mLy1s', decode_responses=True) # db_234 = redis.Redis(host='192.168.1.234', port=6379, password='icDb29mLy1s', decode_responses=True)
#
def get_channel_info(): # def get_channel_info():
"""return supplier_id 或者 brand_id 等等""" # """return supplier_id 或者 brand_id 等等"""
data = {} # data = {}
infos = db.hgetall('shop_distribution') # infos = db.hgetall('shop_distribution')
for key in infos: # for key in infos:
info_value = json.loads(infos[key]) # info_value = json.loads(infos[key])
# 暂时只推京东数据 # # 暂时只推京东数据
if info_value["shop_id"] != 1 or info_value["platform"] != 1: # if info_value["shop_id"] != 1 or info_value["platform"] != 1:
continue # continue
supplier_id_list = info_value["supplier_id_list"] # supplier_id_list = info_value["supplier_id_list"]
brand_id_list = info_value["brand_id_list"] # brand_id_list = info_value["brand_id_list"]
platform = info_value["platform"] # platform = info_value["platform"]
data["supplier_id_list"] = supplier_id_list # data["supplier_id_list"] = supplier_id_list
data["brand_id_list"] = brand_id_list # data["brand_id_list"] = brand_id_list
data["platform"] = platform # data["platform"] = platform
return data # return data
#
def period_get_data(db_name, table_name, condition) -> list: # def period_get_data(db_name, table_name, condition) -> list:
"""指定数据库 指定表中获取数据""" # """指定数据库 指定表中获取数据"""
db = pymysql.connect(host='192.168.1.238', port=3306, user='spu', password='spu', database=db_name) # db = pymysql.connect(host='192.168.1.238', port=3306, user='spu', password='spu', database=db_name)
supplier_ids = condition["supplier_id_list"] # supplier_ids = condition["supplier_id_list"]
sql = "select spu_id from " + table_name + " where supplier_id in %s" # sql = "select spu_id from " + table_name + " where supplier_id in %s"
cursor = db.cursor() # cursor = db.cursor()
cursor.execute(sql, args=(supplier_ids,)) # cursor.execute(sql, args=(supplier_ids,))
results = cursor.fetchall() # results = cursor.fetchall()
data = [] # data = []
for i in range(len(results)): # for i in range(len(results)):
data.append(results[i][0]) # data.append(results[i][0])
return data # return data
#
def get_class_from_rds(spu_id): # def get_class_from_rds(spu_id):
"""根据spu_id 在redis中获取到对应的品牌和分类""" # """根据spu_id 在redis中获取到对应的品牌和分类"""
infos = db_234.hget("spu", str(spu_id)) # infos = db_234.hget("spu", str(spu_id))
return json.loads(infos) # return json.loads(infos)
#
#
if __name__ == '__main__': # if __name__ == '__main__':
condition = get_channel_info() # condition = get_channel_info()
spu_ids = period_get_data("liexin_sku_0", "lie_sku_0", condition) # spu_ids = period_get_data("liexin_sku_0", "lie_sku_0", condition)
print(spu_ids) # print(spu_ids)
infos = get_class_from_rds(spu_ids[0]) # infos = get_class_from_rds(spu_ids[0])
print(infos) # print(infos)
\ No newline at end of file \ No newline at end of file
update_jd_attr.py
# 主要用于更新db中对应分类下的参数属性 # 主要用于更新db中对应分类下的参数属性
# 分类数据来源于lie_shop_class 数据库中class_id 字段或 parent_id 字段 # 分类数据来源于lie_shop_class 数据库中class_id 字段或 parent_id 字段
import json
import sys import sys
import pymysql
from common import request_to_jd_vc
config_dev = { config_dev = {
"host": "192.168.1.238", "host": "192.168.1.238",
...@@ -15,13 +18,50 @@ config_prod = { ...@@ -15,13 +18,50 @@ config_prod = {
} }
def update_jd_attr_by_class(conf: dict):
"""
基于lie_shop_attr中的class_id来更新属性
缺少的属性直接添加，改变的属性则进行update
"""
host = conf["host"]
port = conf["port"]
user = conf["user"]
password = conf["password"]
database = conf["database"]
db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
select_sql = "select distinct class_id from lie_shop_attr"
cursor = db.cursor()
cursor.execute(query=select_sql)
results = cursor.fetchall()
class_ids = []
for i in range(len(results)):
class_ids.append(results[i][0])
# 请求京东平台 获取分类下的属性（vc接口）
method_name = "jingdong.vc.item.props.find"
token = "845ce8478b074103b9e78a769d5fa4831y2u"
app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
app_secret = "c92691b2379c48de87e699c4c2f7fb32"
for i in range(len(class_ids)):
if i != 0:
break
param = {
"category_leaf_id": class_ids[i]
}
ans = request_to_jd_vc(method_name=method_name, token=token, app_key=app_key, app_secret=app_secret, param=param)
if __name__ == '__main__': if __name__ == '__main__':
option = {"dev", "prod"} option = {"dev", "prod"}
if len(sys.argv) >= 2 and sys.argv[1] in option: if len(sys.argv) >= 2 and sys.argv[1] in option:
environment = sys.argv[1] environment = sys.argv[1]
else: else:
sys.exit(1) sys.exit(1)
config = config_dev
if environment == "dev": if environment == "dev":
config = config_dev config = config_dev
elif environment == "prod": elif environment == "prod":
config = config_prod config = config_prod
update_jd_attr_by_class(conf=config)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment