Skip to content

岳巧源 / jd_data_process

Go to a project
Commit 09472ff9 by larosa
Options

modify project

parent 3d8b9935
Expand all
Inline Side-by-side
Showing with 9 additions and 315 deletions
config.json
......@@ -10,5 +10,12 @@
"app_key": "CCE6563827D13EDF4769C01E7429378D",
"app_secret": "98c72374c5b045529879786fc4e5c20e"
},
"json_process_website": "https://www.jyshare.com/front-end/53/"
"json_process_website": "https://www.jyshare.com/front-end/53/",
"mysql_dev": {
"host": "192.168.1.238",
"port": 3306,
"user": "liexin_data_distribution",
"password": "liexin_data_distribution#zsyM",
"database": "liexin_data_distribution"
}
}
\ No newline at end of file
move_copy_to_db.py deleted 100644 → 0
import pymysql
# 主要用于将copy表移动到原表。
if __name__ == '__main__':
db = pymysql.connect(host='192.168.1.238', port=3306, user='liexin_data_distribution',
password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
query_sql = "select id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time from lie_shop_attr_copy1"
cursor = db.cursor()
cursor.execute(query=query_sql)
results = cursor.fetchall()
for i in range(len(results)):
id = results[i][0]
class_id = results[i][1]
attr_id = results[i][2]
attr_name = results[i][3]
unit = results[i][4]
is_enum = results[i][5]
enum_value = results[i][6]
input_type = results[i][7]
is_required = results[i][8]
status = results[i][9]
platform = results[i][10]
create_time = results[i][11]
update_time = results[i][12]
insert_sql = "insert into lie_shop_attr (id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s,%s)"
cursor.execute(insert_sql, args=(id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time,))
db.commit()
print(insert_sql % (id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time))
\ No newline at end of file
tmp.py deleted 100644 → 0
import json
import time
import pymysql
from common import request_to_jd_vc
def get_db_connect(host, port, user, password, database):
db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
cursor = db.cursor()
return db, cursor
def get_attr():
pass
def get_ext_attr():
pass
if __name__ == '__main__':
# 临时处理 京东参数问题 之后会删除
db, cursor = get_db_connect(host='192.168.1.238', port=3306, user='liexin_data_distribution', password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
query_sql = "select distinct class_id from lie_shop_attr_copy1"
cursor.execute(query=query_sql)
results = cursor.fetchall()
with open('config.json', 'r', encoding='utf-8') as f:
json_data = json.load(f)
token = json_data["shop_vc"]["token"]
app_key = json_data["shop_vc"]["app_key"]
app_secret = json_data["shop_vc"]["app_secret"]
for i in range(len(results)):
class_id = results[i][0]
ans = request_to_jd_vc(method_name="jingdong.vc.item.props.find", token=token, app_key=app_key, app_secret=app_secret, param={"category_leaf_id": class_id})
# print(json.dumps(ans, ensure_ascii=False))
jd_results = ans["jingdong_vc_item_props_find_responce"]["jos_result_dto"]["result"]
for j in range(len(jd_results)):
props = jd_results[j]["props"]
for k in range(len(props)):
attr_id = props[k]["id"]
attr_name = props[k]["name"]
unit = props[k]["val_unit"]
jd_value = props[k]["values"]
if len(jd_value) == 0:
enum_value = "[]"
else:
enum_value = json.dumps(jd_value, ensure_ascii=False)
input_type = props[k]["input_type"]
flag = props[k]["required"]
if flag:
is_required = 1
else:
is_required = 0
create_time = int(time.time())
check_sql = "select id from lie_shop_attr_copy1 where class_id = %s and attr_id = %s"
cursor.execute(query=check_sql, args=(class_id, attr_id,))
check_results = cursor.fetchall()
if len(check_results) != 0:
#说明此class_id 下的此属性id是已经存在的
continue
else:
insert_sql = "insert into lie_shop_attr_copy1 (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, 1, 1, create_time))
db.commit()
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, 1, 1, create_time))
print("==================================================================")
tmp2.py deleted 100644 → 0
import json
import time
import pymysql
from common import request_to_jd_vc
def get_db_connect(host, port, user, password, database):
db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
cursor = db.cursor()
return db, cursor
def get_attr():
pass
def get_ext_attr():
pass
if __name__ == '__main__':
# 处理拓展参数问题
db, cursor = get_db_connect(host='192.168.1.238', port=3306, user='liexin_data_distribution',
password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
query_sql = "select distinct class_id from lie_shop_attr_copy1"
cursor.execute(query=query_sql)
results = cursor.fetchall()
# results = [[33607]]
with open('config.json', 'r', encoding='utf-8') as f:
json_data = json.load(f)
token = json_data["shop_vc"]["token"]
app_key = json_data["shop_vc"]["app_key"]
app_secret = json_data["shop_vc"]["app_secret"]
for i in range(len(results)):
class_id = results[i][0]
ans = request_to_jd_vc(method_name="jingdong.vc.item.extProps.find", token=token, app_key=app_key,
app_secret=app_secret, param={"category_leaf_id": class_id})
# print(json.dumps(ans, ensure_ascii=False))
if "result" not in ans["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]:
continue
jd_results = ans["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]["result"]
for j in range(len(jd_results)):
attr_id = jd_results[j]["att_id"]
attr_name = jd_results[j]["name"]
unit = jd_results[j]["val_unit"]
tmp_value = jd_results[j]["ext_prop_value"]
vals = []
for k in range(len(tmp_value)):
value_id = tmp_value[k]["value_id"]
value_name = tmp_value[k]["value_name"]
item = {
"attr_value_id": value_id,
"attr_value_name": value_name,
}
vals.append(item)
enum_value = json.dumps(vals, ensure_ascii=False)
input_type = jd_results[j]["input_type"]
flag = jd_results[j]["is_required"]
if flag == 1:
is_required = 1
else:
is_required = 0
status = 1
platform = 1
create_time = int(time.time())
check_sql = "select id from lie_shop_attr_copy1 where class_id = %s and attr_id = %s"
cursor.execute(check_sql, args=(class_id, attr_id,))
db_ans = cursor.fetchall()
if len(db_ans) != 0:
continue
insert_sql = "insert into lie_shop_attr_copy1 (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
# cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
update_jd_attr.py
import json
import time
import pymysql
from common import request_to_jd_vc
db = pymysql.connect(host='192.168.1.238', port=3306, user='liexin_data_distribution',
password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
cursor = db.cursor()
# 主要用于从 lie_shop_class 表中获取到class_id 后，将对应的属性和拓展属性 写入lie_shop_attr表(或者它的copy表)。
def get_class_ids():
query_sql = "select distinct class_id from lie_shop_class"
cursor.execute(query=query_sql)
results = cursor.fetchall()
data = []
for i in range(len(results)):
data.append(results[i][0])
return data
def check_class_and_attr(class_id: int, attr_id: int) -> bool:
check_sql = "select id from lie_shop_attr where class_id = %s and attr_id = %s"
cursor.execute(query=check_sql, args=(class_id, attr_id,))
results = cursor.fetchall()
if len(results) != 0:
return True
return False
def insert_class_of_attr(class_id: int):
"""加入新的分类属性"""
method_name = "jingdong.vc.item.props.find"
token = "845ce8478b074103b9e78a769d5fa4831y2u"
app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
app_secret = "c92691b2379c48de87e699c4c2f7fb32"
data = request_to_jd_vc(method_name=method_name,
token=token,
app_key=app_key,
app_secret=app_secret,
param={"category_leaf_id": class_id}
)
print(json.dumps(data, ensure_ascii=False))
if "result" not in data["jingdong_vc_item_props_find_responce"]["jos_result_dto"]:
return
results = data["jingdong_vc_item_props_find_responce"]["jos_result_dto"]["result"]
for i in range(len(results)):
props = results[i]["props"]
for j in range(len(props)):
class_id = class_id
attr_id = props[j]["id"]
ex = check_class_and_attr(class_id, attr_id)
if ex:
continue
attr_name = props[j]["name"]
unit = props[j]["val_unit"]
input_type = props[j]["input_type"]
enum_value = "[]"
if input_type == 1 or input_type == 2:
# input 为1 或 2 则获取 值id和值名称。
lists = []
for k in range(len(props[j]["values"])):
item = dict()
item["attr_value_id"] = props[j]["values"][k]["id"]
item["attr_value_name"] = props[j]["values"][k]["name"]
lists.append(item)
enum_value = json.dumps(lists, ensure_ascii=False)
if input_type == 10:
# input_type 为10 则获取单位
lists = []
for k in range(len(props[j]["values"])):
if "units" in props[j]["values"][k]:
for x in range(len(props[j]["values"][k]["units"])):
lists.append(props[j]["values"][k]["units"][x])
unit = json.dumps(lists, ensure_ascii=False)
if props[j]["required"] == True:
is_required = 1
else:
is_required = 0
status = 1
platform = 1
create_time = int(time.time())
exists = check_class_and_attr(class_id, attr_id)
if not exists:
insert_sql = "insert into lie_shop_attr (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time,))
db.commit()
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
def insert_class_of_ext_attr(class_id: int):
"""加入新的分类拓展属性（拓展属性）"""
method_name = "jingdong.vc.item.extProps.find"
token = "845ce8478b074103b9e78a769d5fa4831y2u"
app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
app_secret = "c92691b2379c48de87e699c4c2f7fb32"
data = request_to_jd_vc(method_name=method_name,
token=token,
app_key=app_key,
app_secret=app_secret,
param={"category_leaf_id": class_id}
)
# print(json.dumps(data,ensure_ascii=False))
if "result" in data["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]:
results = data["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]["result"]
for i in range(len(results)):
attr_id = results[i]["att_id"]
exists = check_class_and_attr(class_id, attr_id)
if not exists:
attr_name = results[i]["name"]
unit = results[i]["val_unit"]
ext_prop_value = results[i]["ext_prop_value"]
lists = []
for j in range(len(ext_prop_value)):
item = dict()
item["attr_value_id"] = ext_prop_value[j]["value_id"]
item["attr_value_name"] = ext_prop_value[j]["value_name"]
lists.append(item)
enum_value = json.dumps(lists, ensure_ascii=False)
input_type = results[i]["input_type"]
if results[i]["is_required"] == 1:
is_required = 1
else:
is_required = 0
status = 1
platform = 1
create_time = int(time.time())
insert_sql = "insert into lie_shop_attr (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time,))
db.commit()
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
if __name__ == '__main__':
# class_ids = get_class_ids()
# for i in range(len(class_ids)):
# class_id = class_ids[i]
# insert_class_of_attr(class_id)
# insert_class_of_ext_attr(class_id)
insert_class_of_attr(15818)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment