update script

crontab.py

+"""
+此脚本用于拉取并更新最新的vc数据
+遵循无数据就 insert, 有数据就update
+"""
+from common import request_to_jd_vc
+import sys
+import pymysql
+import time
+import json
+
+config_dev = {
+    "mysql": {
+        "host": "192.168.1.238",
+        "port": 3306,
+        "user": "liexin_data_distribution",
+        "password": "liexin_data_distribution#zsyM",
+        "database": "liexin_data_distribution"
+    },
+    "redis": {
+        "host": "192.168.1.235",
+        "port": 6379,
+        "password": "icDb29mLy2s"
+    }
+}
+
+config_prod = {
+    "mysql": {
+        "host": "bigdata2.ichunt.db",
+        "port": 3306,
+        "user": "DsbuUx",
+        "password": "sfj09JjsfpQx0",
+        "database": "liexin_data_distribution"
+    },
+    "redis": {
+        "host": "172.18.137.38",
+        "port": 6379,
+        "password": "icDb29mLy2s"
+    }
+}
+
+def update_vc_class(conf: dict):
+    """
+    查找分类id 存在则不插入。
+    """
+    method_name = "jingdong.vc.item.categories.find"
+    token = "845ce8478b074103b9e78a769d5fa4831y2u"
+    app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
+    app_secret = "c92691b2379c48de87e699c4c2f7fb32"
+    param = {}
+    ans = request_to_jd_vc(method_name, token, app_key, app_secret, param)
+    results = ans["jingdong_vc_item_categories_find_responce"]["jos_result_dto"]["result"]
+    host = conf["mysql"]["host"]
+    port = conf["mysql"]["port"]
+    user = conf["mysql"]["user"]
+    password = conf["mysql"]["password"]
+    database = conf["mysql"]["database"]
+    db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
+    cursor = db.cursor()
+    for i in range(len(results)):
+        parent_id = results[i]["cid3"]
+        class_id = results[i]["id"]
+        class_name = results[i]["name"]
+        depth = results[i]["depth"]
+        status = 1
+        platform = 1
+        create_time = int(time.time())
+        # check if already exist.
+        check_sql = "select class_id from lie_shop_class where class_id = %s"
+        cursor.execute(query=check_sql, args=(class_id,))
+        check_result = cursor.fetchall()
+        if len(check_result) > 0:
+            continue
+        insert_sql = "insert into lie_shop_class (parent_id, class_id, class_name, depth, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s)"
+        cursor.execute(query=insert_sql, args=(parent_id, class_id, class_name, depth, status, platform, create_time,))
+        db.commit()
+        print(insert_sql % (parent_id, class_id, class_name, depth, status, platform, create_time))
+
+
+
+
+def update_vc_attr(conf: dict):
+    """lie_shop_class中拿vc class_id 插入属性"""
+    select_sql = "select class_id from lie_shop_class where platform = 1"
+    host = conf["mysql"]["host"]
+    port = conf["mysql"]["port"]
+    user = conf["mysql"]["user"]
+    password = conf["mysql"]["password"]
+    database = conf["mysql"]["database"]
+    method_name = "jingdong.vc.item.props.find"
+    token = "845ce8478b074103b9e78a769d5fa4831y2u"
+    app_key = "CA52430E90209F51D8F5D7B615DDE9AD"
+    app_secret = "c92691b2379c48de87e699c4c2f7fb32"
+    db = pymysql.connect(host=host, port=port, user=user, password=password, database=database)
+    cursor = db.cursor()
+    cursor.execute(query=select_sql)
+    db_results = cursor.fetchall()
+    for i in range(len(db_results)):
+        class_id = db_results[i][0]
+        param = {
+            "category_leaf_id": class_id
+        }
+        ans = request_to_jd_vc(method_name=method_name, token=token, app_key=app_key, app_secret=app_secret,
+                               param=param)
+        print(json.dumps(ans, ensure_ascii=False))
+        if "result" not in ans["jingdong_vc_item_props_find_responce"]["jos_result_dto"]:
+            continue
+        jd_results = ans["jingdong_vc_item_props_find_responce"]["jos_result_dto"]["result"]
+
+        """
+        先预先把此分类的全部属性查询出来
+        """
+        check_sql = "select id, class_id, attr_id, attr_name, unit, enum_value, input_type, is_required from lie_shop_attr where platform = 1 and class_id = %s"
+        cursor.execute(query=check_sql, args=(class_id, ))
+        check_result = cursor.fetchall()
+        for j in range(len(jd_results)):
+            result = jd_results[j]
+            props = result["props"]
+            for k in range(len(props)):
+                prop = props[k]
+                attr_id = prop["id"]
+                attr_name = prop["name"]
+                unit = prop["val_unit"]
+                input_type = prop["input_type"]
+                platform = 1
+                status = 1
+                if not prop["required"]:
+                    is_required = 0
+                else:
+                    is_required = 1
+                enum_value = "[]"
+                if input_type == 1 or input_type == 2:
+                    # 单选或者多选 把选项取出来
+                    tmp_value_list = []
+                    values = prop["values"]
+                    for index in range(len(values)):
+                        attr_value_id = values[index]["id"]
+                        attr_value_name = values[index]["name"]
+                        item_map = {
+                            "attr_value_id": attr_value_id,
+                            "attr_value_name": attr_value_name,
+                        }
+                        tmp_value_list.append(item_map)
+                    enum_value = json.dumps(tmp_value_list, ensure_ascii=False)
+                elif input_type == 3:
+                    # 文本类型
+                    # 文本类型没有可选值
+                        pass
+                elif input_type == 7:
+                    # 数值类型
+                    # 数值类型没有可选值
+                        pass
+                elif input_type == 10:
+                    # 多单位多解析的方式
+                    # 多单位多解析 需要特殊处理单位字段
+                    tmp_unit_list = []
+                    values = prop["values"]
+                    for index in range(len(values)):
+                        if "units" in values[index]:
+                            unit_list = values[index]["units"]
+                            for x in range(len(unit_list)):
+                                tmp_unit_list.append(unit_list[x])
+                    unit = json.dumps(tmp_unit_list, ensure_ascii=False)
+                else:
+                    print("此input_type类型暂未兼容处理！请单独处理！")
+                create_time = int(time.time())
+
+                """
+                检查属性需不需要更新等等。。。
+                """
+                exists_attr = False
+                now_attr_map = {
+                    "attr_id": attr_id,
+                    "attr_name": attr_name,
+                    "unit": unit,
+                    "enum_value": enum_value,
+                    "input_type": input_type,
+                    "is_required": is_required,
+                }
+                for z in range(len(check_result)):
+                    primary_id = check_result[z][0]
+                    origin_class_id = check_result[z][1]
+                    origin_attr_id = check_result[z][2]
+                    origin_attr_name = check_result[z][3]
+                    origin_unit = check_result[z][4]
+                    origin_enum_value = check_result[z][5]
+                    origin_input_type = check_result[z][6]
+                    origin_is_required = check_result[z][7]
+                    """
+                    原来数据库中的属性map
+                    """
+                    origin_attr_map = {
+                        "attr_id": origin_attr_id,
+                        "attr_name": origin_attr_name,
+                        "unit": origin_unit,
+                        "enum_value": origin_enum_value,
+                        "input_type": origin_input_type,
+                        "is_required": origin_is_required,
+                    }
+                    flag1 = (origin_attr_id == attr_id) and (origin_attr_name == attr_name)
+                    flag2 = (origin_attr_id == attr_id) and (origin_attr_name != attr_name)
+                    flag3 = (origin_attr_id != attr_id) and (origin_attr_name == attr_name)
+                    if flag1 or flag2 or flag3:
+                        """
+                        检查其他项是否需要更新,有一个属性不一样就要执行update
+                        """
+                        exists_attr = True
+                        for key in origin_attr_map:
+                            if origin_attr_map[key] != now_attr_map[key]:
+                                update_sql1 = "update lie_shop_attr set unit = %s, enum_value = %s, input_type = %s, is_required = %s where id = %s"
+                                cursor.execute(query=update_sql1, args=(unit, enum_value, input_type, is_required, primary_id))
+                                db.commit()
+                                print(update_sql1 % (unit, enum_value, input_type, is_required, primary_id))
+                                break
+                        break
+                    else:
+                        continue
+                if not exists_attr:
+                    insert_sql = "insert into lie_shop_attr (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
+                    cursor.execute(query=insert_sql, args=(class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time,))
+                    db.commit()
+                    print(insert_sql % (class_id, attr_id, attr_name, unit, enum_value, input_type, is_required, status, platform, create_time))
+
+
+
+
+"""
+执行命令 
+线上：python3 crontab.py prod
+测试：python3 crontab.py dev 
+"""
+if __name__ == '__main__':
+    option = {"dev", "prod"}
+    if len(sys.argv) >= 2 and sys.argv[1] in option:
+        environment = sys.argv[1]
+    else:
+        sys.exit(1)
+    config = config_dev
+    if environment == "dev":
+        config = config_dev
+    elif environment == "prod":
+        config = config_prod
+    update_vc_class(config)
+    print("========================> 分类更新完毕")
+    update_vc_attr(config)
+    print("========================> 参数更新完毕")
\ No newline at end of file