Skip to content

岳巧源 / jd_data_process

Go to a project
Commit 09472ff9 by larosa
Options

modify project

parent 3d8b9935
Inline Side-by-side
Showing with 9 additions and 689 deletions
config.json
......@@ -10,5 +10,12 @@
"app_key": "CCE6563827D13EDF4769C01E7429378D",
"app_secret": "98c72374c5b045529879786fc4e5c20e"
},
"json_process_website": "https://www.jyshare.com/front-end/53/"
"json_process_website": "https://www.jyshare.com/front-end/53/",
"mysql_dev": {
"host": "192.168.1.238",
"port": 3306,
"user": "liexin_data_distribution",
"password": "liexin_data_distribution#zsyM",
"database": "liexin_data_distribution"
}
}
\ No newline at end of file
move_copy_to_db.py deleted 100644 → 0
import pymysql
# 主要用于将copy表移动到原表。
if __name__ == '__main__':
db = pymysql.connect(host='192.168.1.238', port=3306, user='liexin_data_distribution',
password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
query_sql = "select id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time from lie_shop_attr_copy1"
cursor = db.cursor()
cursor.execute(query=query_sql)
results = cursor.fetchall()
for i in range(len(results)):
id = results[i][0]
class_id = results[i][1]
attr_id = results[i][2]
attr_name = results[i][3]
unit = results[i][4]
is_enum = results[i][5]
enum_value = results[i][6]
input_type = results[i][7]
is_required = results[i][8]
status = results[i][9]
platform = results[i][10]
create_time = results[i][11]
update_time = results[i][12]
insert_sql = "insert into lie_shop_attr (id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time) values (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s,%s)"
cursor.execute(insert_sql, args=(id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time,))
db.commit()
print(insert_sql % (id, class_id, attr_id, attr_name, unit, is_enum, enum_value, input_type, is_required, status, platform, create_time, update_time))
\ No newline at end of file
tmp.json deleted 100644 → 0
{
"jingdong_vc_item_props_find_responce": {
"code": "0",
"request_id": "161812.39587.17275790342353654",
"jos_result_dto": {
"result": [
{
"order_sort": 0,
"name": "主体参数",
"id": 39805,
"props": [
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235747,
"order_sort": 1669,
"level": 0,
"inputAreaLevel": "3",
"name": "温度特性",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235745,
"order_sort": 1668,
"level": 0,
"inputAreaLevel": "3",
"name": "生命周期",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235717,
"order_sort": 1661,
"level": 0,
"inputAreaLevel": "3",
"name": "额定电压(v)",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235749,
"order_sort": 1670,
"level": 0,
"inputAreaLevel": "3",
"name": "应用级别",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 246020,
"order_sort": 101,
"level": 0,
"inputAreaLevel": "3",
"name": "RoHS",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235721,
"order_sort": 1662,
"level": 0,
"inputAreaLevel": "3",
"name": "封装规格",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235726,
"order_sort": 1663,
"level": 0,
"inputAreaLevel": "3",
"name": "工作温度范围(℃)",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235730,
"order_sort": 1664,
"level": 0,
"inputAreaLevel": "3",
"name": "规格型号(MPN)",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 246007,
"order_sort": 98,
"level": 0,
"inputAreaLevel": "3",
"name": "ECCN",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235734,
"order_sort": 1665,
"level": 0,
"inputAreaLevel": "3",
"name": "精度(%)",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235737,
"order_sort": 1666,
"level": 0,
"inputAreaLevel": "3",
"name": "容值(F)",
"val_unit": ""
},
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 235741,
"order_sort": 1667,
"level": 0,
"inputAreaLevel": "3",
"name": "介电材料",
"val_unit": ""
}
]
},
{
"order_sort": 0,
"name": "公共参数",
"id": 42975,
"props": [
{
"attr_alias": "",
"values": [],
"maintain_remark": "",
"required": false,
"cascadeGroupLevel": 2147483647,
"input_type": 3,
"id": 270352,
"order_sort": 0,
"level": 0,
"inputAreaLevel": "3",
"name": "生产批次",
"val_unit": ""
},
{
"attr_alias": "0",
"values": [],
"maintain_remark": "",
"choose_purchase": 0,
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 7,
"id": 270353,
"order_sort": 0,
"level": 0,
"inputAreaLevel": "3",
"name": "包装规格(MPQ)",
"val_unit": "psc"
},
{
"attr_alias": "0",
"values": [
{
"order_sort": 0,
"name": "包",
"id": 1639264,
"selected": false
},
{
"order_sort": 1,
"name": "颗",
"id": 1639265,
"selected": false
},
{
"order_sort": 2,
"name": "张",
"id": 1639266,
"selected": false
},
{
"order_sort": 3,
"name": "个",
"id": 1639267,
"selected": false
},
{
"order_sort": 4,
"name": "支",
"id": 1639268,
"selected": false
},
{
"order_sort": 5,
"name": "本",
"id": 1639269,
"selected": false
},
{
"order_sort": 6,
"name": "箱",
"id": 1639270,
"selected": false
},
{
"order_sort": 7,
"name": "根",
"id": 1639271,
"selected": false
},
{
"order_sort": 8,
"name": "片",
"id": 1639272,
"selected": false
},
{
"order_sort": 9,
"name": "盒",
"id": 1639273,
"selected": false
},
{
"order_sort": 10,
"name": "套",
"id": 1639274,
"selected": false
},
{
"order_sort": 11,
"name": "打",
"id": 1639275,
"selected": false
},
{
"order_sort": 12,
"name": "盘",
"id": 1639276,
"selected": false
},
{
"order_sort": 13,
"name": "条",
"id": 1639277,
"selected": false
},
{
"order_sort": 14,
"name": "部",
"id": 1639278,
"selected": false
},
{
"order_sort": 15,
"name": "百个",
"id": 1639279,
"selected": false
},
{
"order_sort": 16,
"name": "只",
"id": 1639280,
"selected": false
},
{
"order_sort": 17,
"name": "米",
"id": 1639281,
"selected": false
},
{
"order_sort": 18,
"name": "件",
"id": 1639282,
"selected": false
},
{
"order_sort": 19,
"name": "卷",
"id": 1639283,
"selected": false
},
{
"order_sort": 20,
"name": "台",
"id": 1639284,
"selected": false
},
{
"order_sort": 21,
"name": "板",
"id": 1639285,
"selected": false
}
],
"maintain_remark": "",
"choose_purchase": 0,
"required": true,
"cascadeGroupLevel": 2147483647,
"input_type": 1,
"id": 270354,
"order_sort": 0,
"level": 0,
"inputAreaLevel": "3",
"name": "销售单位",
"val_unit": ""
}
]
}
],
"return_code": "100",
"is_success": true,
"count": 2,
"return_message": "执行成功"
}
}
}
\ No newline at end of file
tmp.py deleted 100644 → 0
import json
import time
import pymysql
from common import request_to_jd_vc
def get_db_connect(host, port, user, password, database):
db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
cursor = db.cursor()
return db, cursor
def get_attr():
pass
def get_ext_attr():
pass
if __name__ == '__main__':
# 临时处理 京东参数问题 之后会删除
db, cursor = get_db_connect(host='192.168.1.238', port=3306, user='liexin_data_distribution', password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
query_sql = "select distinct class_id from lie_shop_attr_copy1"
cursor.execute(query=query_sql)
results = cursor.fetchall()
with open('config.json', 'r', encoding='utf-8') as f:
json_data = json.load(f)
token = json_data["shop_vc"]["token"]
app_key = json_data["shop_vc"]["app_key"]
app_secret = json_data["shop_vc"]["app_secret"]
for i in range(len(results)):
class_id = results[i][0]
ans = request_to_jd_vc(method_name="jingdong.vc.item.props.find", token=token, app_key=app_key, app_secret=app_secret, param={"category_leaf_id": class_id})
# print(json.dumps(ans, ensure_ascii=False))
jd_results = ans["jingdong_vc_item_props_find_responce"]["jos_result_dto"]["result"]
for j in range(len(jd_results)):
props = jd_results[j]["props"]
for k in range(len(props)):
attr_id = props[k]["id"]
attr_name = props[k]["name"]
unit = props[k]["val_unit"]
jd_value = props[k]["values"]
if len(jd_value) == 0:
enum_value = "[]"
else:
enum_value = json.dumps(jd_value, ensure_ascii=False)
input_type = props[k]["input_type"]
flag = props[k]["required"]
if flag:
is_required = 1
else:
is_required = 0
create_time = int(time.time())
check_sql = "select id from lie_shop_attr_copy1 where class_id = %s and attr_id = %s"
cursor.execute(query=check_sql, args=(class_id, attr_id,))
check_results = cursor.fetchall()
if len(check_results) != 0:
#说明此class_id 下的此属性id是已经存在的
continue
else:
insert_sql = "insert into lie_shop_attr_copy1 (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, 1, 1, create_time))
db.commit()
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, 1, 1, create_time))
print("==================================================================")
tmp2.py deleted 100644 → 0
import json
import time
import pymysql
from common import request_to_jd_vc
def get_db_connect(host, port, user, password, database):
db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
cursor = db.cursor()
return db, cursor
def get_attr():
pass
def get_ext_attr():
pass
if __name__ == '__main__':
# 处理拓展参数问题
db, cursor = get_db_connect(host='192.168.1.238', port=3306, user='liexin_data_distribution',
password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
query_sql = "select distinct class_id from lie_shop_attr_copy1"
cursor.execute(query=query_sql)
results = cursor.fetchall()
# results = [[33607]]
with open('config.json', 'r', encoding='utf-8') as f:
json_data = json.load(f)
token = json_data["shop_vc"]["token"]
app_key = json_data["shop_vc"]["app_key"]
app_secret = json_data["shop_vc"]["app_secret"]
for i in range(len(results)):
class_id = results[i][0]
ans = request_to_jd_vc(method_name="jingdong.vc.item.extProps.find", token=token, app_key=app_key,
app_secret=app_secret, param={"category_leaf_id": class_id})
# print(json.dumps(ans, ensure_ascii=False))
if "result" not in ans["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]:
continue
jd_results = ans["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]["result"]
for j in range(len(jd_results)):
attr_id = jd_results[j]["att_id"]
attr_name = jd_results[j]["name"]
unit = jd_results[j]["val_unit"]
tmp_value = jd_results[j]["ext_prop_value"]
vals = []
for k in range(len(tmp_value)):
value_id = tmp_value[k]["value_id"]
value_name = tmp_value[k]["value_name"]
item = {
"attr_value_id": value_id,
"attr_value_name": value_name,
}
vals.append(item)
enum_value = json.dumps(vals, ensure_ascii=False)
input_type = jd_results[j]["input_type"]
flag = jd_results[j]["is_required"]
if flag == 1:
is_required = 1
else:
is_required = 0
status = 1
platform = 1
create_time = int(time.time())
check_sql = "select id from lie_shop_attr_copy1 where class_id = %s and attr_id = %s"
cursor.execute(check_sql, args=(class_id, attr_id,))
db_ans = cursor.fetchall()
if len(db_ans) != 0:
continue
insert_sql = "insert into lie_shop_attr_copy1 (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
# cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
update_jd_attr.py
import json
import time
import pymysql
from common import request_to_jd_vc
db = pymysql.connect(host='192.168.1.238', port=3306, user='liexin_data_distribution',
password='liexin_data_distribution#zsyM', database='liexin_data_distribution')
cursor = db.cursor()
# 主要用于从 lie_shop_class 表中获取到class_id 后，将对应的属性和拓展属性 写入lie_shop_attr表(或者它的copy表)。
def get_class_ids():
query_sql = "select distinct class_id from lie_shop_class"
cursor.execute(query=query_sql)
results = cursor.fetchall()
data = []
for i in range(len(results)):
data.append(results[i][0])
return data
def check_class_and_attr(class_id: int, attr_id: int) -> bool:
check_sql = "select id from lie_shop_attr where class_id = %s and attr_id = %s"
cursor.execute(query=check_sql, args=(class_id, attr_id,))
results = cursor.fetchall()
if len(results) != 0:
return True
return False
def insert_class_of_attr(class_id: int):
"""加入新的分类属性"""
method_name = "jingdong.vc.item.props.find"
token = "845ce8478b074103b9e78a769d5fa4831y2u"
app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
app_secret = "c92691b2379c48de87e699c4c2f7fb32"
data = request_to_jd_vc(method_name=method_name,
token=token,
app_key=app_key,
app_secret=app_secret,
param={"category_leaf_id": class_id}
)
print(json.dumps(data, ensure_ascii=False))
if "result" not in data["jingdong_vc_item_props_find_responce"]["jos_result_dto"]:
return
results = data["jingdong_vc_item_props_find_responce"]["jos_result_dto"]["result"]
for i in range(len(results)):
props = results[i]["props"]
for j in range(len(props)):
class_id = class_id
attr_id = props[j]["id"]
ex = check_class_and_attr(class_id, attr_id)
if ex:
continue
attr_name = props[j]["name"]
unit = props[j]["val_unit"]
input_type = props[j]["input_type"]
enum_value = "[]"
if input_type == 1 or input_type == 2:
# input 为1 或 2 则获取 值id和值名称。
lists = []
for k in range(len(props[j]["values"])):
item = dict()
item["attr_value_id"] = props[j]["values"][k]["id"]
item["attr_value_name"] = props[j]["values"][k]["name"]
lists.append(item)
enum_value = json.dumps(lists, ensure_ascii=False)
if input_type == 10:
# input_type 为10 则获取单位
lists = []
for k in range(len(props[j]["values"])):
if "units" in props[j]["values"][k]:
for x in range(len(props[j]["values"][k]["units"])):
lists.append(props[j]["values"][k]["units"][x])
unit = json.dumps(lists, ensure_ascii=False)
if props[j]["required"] == True:
is_required = 1
else:
is_required = 0
status = 1
platform = 1
create_time = int(time.time())
exists = check_class_and_attr(class_id, attr_id)
if not exists:
insert_sql = "insert into lie_shop_attr (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time,))
db.commit()
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
def insert_class_of_ext_attr(class_id: int):
"""加入新的分类拓展属性（拓展属性）"""
method_name = "jingdong.vc.item.extProps.find"
token = "845ce8478b074103b9e78a769d5fa4831y2u"
app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
app_secret = "c92691b2379c48de87e699c4c2f7fb32"
data = request_to_jd_vc(method_name=method_name,
token=token,
app_key=app_key,
app_secret=app_secret,
param={"category_leaf_id": class_id}
)
# print(json.dumps(data,ensure_ascii=False))
if "result" in data["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]:
results = data["jingdong_vc_item_extProps_find_responce"]["jos_result_dto"]["result"]
for i in range(len(results)):
attr_id = results[i]["att_id"]
exists = check_class_and_attr(class_id, attr_id)
if not exists:
attr_name = results[i]["name"]
unit = results[i]["val_unit"]
ext_prop_value = results[i]["ext_prop_value"]
lists = []
for j in range(len(ext_prop_value)):
item = dict()
item["attr_value_id"] = ext_prop_value[j]["value_id"]
item["attr_value_name"] = ext_prop_value[j]["value_name"]
lists.append(item)
enum_value = json.dumps(lists, ensure_ascii=False)
input_type = results[i]["input_type"]
if results[i]["is_required"] == 1:
is_required = 1
else:
is_required = 0
status = 1
platform = 1
create_time = int(time.time())
insert_sql = "insert into lie_shop_attr (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time,))
db.commit()
print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
if __name__ == '__main__':
# class_ids = get_class_ids()
# for i in range(len(class_ids)):
# class_id = class_ids[i]
# insert_class_of_attr(class_id)
# insert_class_of_ext_attr(class_id)
insert_class_of_attr(15818)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment