add proxy

plat/hc360/cate_stock.py

+from utils.base import Module_Base
+kw_base = 'https://s.hc360.com/seller/search.html?a=1&w={}'
+
+headers={
+"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+"Accept-Encoding":"gzip, deflate, br",
+"Accept-Language":"zh-CN,zh;q=0.9",
+"Cache-Control":"max-age=0",
+"Connection":"keep-alive",
+"Host":"www.hc360.com",
+"Upgrade-Insecure-Requests":"1",
+"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
+}
+a = Module_Base()
+url = 'https://www.hc360.com/'
+resp = a.requests.get(url,headers=headers).text
+dom = a.etree.HTML(resp)
+level1_name_doms=dom.xpath('//ul[@class="Bside_Uwrap"]//li')
+level1_content_doms=dom.xpath('//div[@id="category"]/div')
+level1_doms=zip(level1_name_doms,level1_content_doms)
+levels=[]
+for level1_dom in level1_doms:
+    level1=dict()
+    level1['cat_name'] = level1_dom[0].xpath('string(.)')
+    level1['url'] = level1_dom[0].xpath('.//div[@class="rigNav_cont"]//a/@href')[0]
+    level1['level'] = 1
+    level1['islast'] = 0
+    level1['parent_id'] = 0
+    level1['level2_list'] = []
+    level2_doms=level1_dom[1].xpath('.//div[@class="sideBarLeft"]//ul//li//a')
+    # print('一级分类',level1)
+    for level2_dom in level2_doms:
+        level2 = {}
+        level2['cat_name'] = level2_dom.xpath('./text()')[0]
+        level2['url'] = kw_base.format(level2['cat_name'])
+        level2['level'] = 2
+        level2['islast'] = 1
+        level1['level2_list'].append(level2)
+        # print('二级分类',level2)
+    levels.append(level1)
+# print(levels)
\ No newline at end of file

update_scrip.py

@@ -8,4 +8,7 @@ if __name__ == '__main__':
     module = importlib.import_module(module_name)
 
     updater = Updater(plat_name)
+    if plat_name in ['hc360']:
+        updater.up_once(module.levels)
+    else:
         updater.up(module.levels)

updateopera.py

 from utils.mysqlopera import MySqlOperator
 
+
 class Updater():
-    def __init__(self, db_name):
-        self.operator = MySqlOperator(db_name)
+    def __init__(self, pn_name):
+        self.operator = MySqlOperator(pn_name)
+        self.table_name = 'lie_' + pn_name + '_' + 'category' if pn_name in ['hc360'] else 'lie_category'
 
     def clean_db(self):
         with self.operator.db.cursor() as cursor:
-            sql = "update lie_category set is_show = 0"
+            sql = "update {} set is_show = 0".format(self.table_name)
             cursor.execute(sql)
 
     def get_cat_id(self, cat_name):
         with self.operator.db.cursor() as cursor:
-            sql = 'select cat_id from lie_category where cat_name =%s'
+            sql = 'select cat_id from {} where cat_name =%s'.format(self.table_name)
             cursor.execute(sql, cat_name)
             result = cursor.fetchone()
             result = result[0] if result else None
             return result
 
-    def update(self, cat_id, url, islast, level, parent_id=0,is_show=1):
+    def update(self, cat_id, url, islast, level, parent_id=0, is_show=1):
         with self.operator.db.cursor() as cursor:
-            sql = 'UPDATE lie_category SET url=%s,islast=%s,level=%s,is_show=%s, parent_id=%s where cat_id=%s'
-            data=(url,islast, level, is_show,parent_id,cat_id)
+            sql = 'UPDATE {} SET url=%s,islast=%s,level=%s,is_show=%s, parent_id=%s where cat_id=%s'.format(
+                self.table_name)
+            data = (url, islast, level, is_show, parent_id, cat_id)
             cursor.execute(sql, data)
 
-    def insert(self, cat_name, url, islast, level, parent_id=0, is_show=1  , sort_order=50, page_count=1):
+    def insert(self, cat_name, url, islast, level, parent_id=0, is_show=1, sort_order=50, page_count=1):
         with self.operator.db.cursor() as cursor:
-
-            sql = 'INSERT into lie_category(cat_name,parent_id,sort_order,is_show,url,islast,level,page_count) values' \
-                  '(%s,%s,%s,%s,%s,%s,%s,%s)'
+            sql = 'INSERT into {}(cat_name,parent_id,sort_order,is_show,url,islast,level,page_count) values' \
+                  '(%s,%s,%s,%s,%s,%s,%s,%s)'.format(self.table_name)
             data = (cat_name, parent_id, sort_order, is_show, url, islast, level, page_count)
             cursor.execute(sql, data)
-            cursor.execute("select max(cat_id) from lie_category")
-            cat_id=cursor.fetchone()[0]
+            cursor.execute("select max(cat_id) from {}".format(self.table_name))
+            cat_id = cursor.fetchone()[0]
             return cat_id
-    def up(self,levels):
+
+    def up(self, levels):
         self.clean_db()
         for level1 in levels:
-            cat1_id=self.get_cat_id(level1['cat_name'])
+            cat1_id = self.get_cat_id(level1['cat_name'])
             if cat1_id:
-                self.update(cat1_id,level1['url'],level1['islast'],level1['level'])
+                self.update(cat1_id, level1['url'], level1['islast'], level1['level'])
             else:
-                cat1_id = self.insert(level1['cat_name'], level1['url'], level1['islast'], level1['level'], level1['parent_id'])
-            print('一级分类',level1['cat_name'])
+                cat1_id = self.insert(level1['cat_name'], level1['url'], level1['islast'], level1['level'],
+                                      level1['parent_id'])
+            print('一级分类', level1['cat_name'])
             for level2 in level1.get('level2_list'):
                 if level2['islast']:
-                    cat2_id=self.get_cat_id(level2['cat_name'])
+                    cat2_id = self.get_cat_id(level2['cat_name'])
                     if cat2_id:
-                        self.update(cat2_id,level2['url'],level2['islast'],level2['level'],cat1_id)
+                        self.update(cat2_id, level2['url'], level2['islast'], level2['level'], cat1_id)
                     else:
-                        self.insert(level2['cat_name'],level2['url'],level2['islast'],level2['level'],cat1_id)
-                    print('二级分类',level2['cat_name'])
+                        self.insert(level2['cat_name'], level2['url'], level2['islast'], level2['level'], cat1_id)
+                    print('二级分类', level2['cat_name'])
                 else:
-                    pass #TODO 增加三级分类
+                    pass  # TODO 增加三级分类
+
+        self.operator.db.commit()
+
+    def insert_once(self, cat_name, url, islast, level, parent_id=0, page_count=1):
+        with self.operator.db.cursor() as cursor:
+            sql = 'INSERT into {}(cat_name,parent_id,url,islast,level,page_count) values' \
+                  '(%s,%s,%s,%s,%s,%s)'.format(self.table_name)
+            data = (cat_name, parent_id, url, islast, level, page_count)
+            cursor.execute(sql, data)
+            cursor.execute("select max(cat_id) from {}".format(self.table_name))
+            cat_id = cursor.fetchone()[0]
+            return cat_id
+
+    def up_once(self, levels):
+        for level1 in levels:
+            cat1_id = self.insert_once(level1['cat_name'], level1['url'], level1['islast'], level1['level'],
+                                  level1['parent_id'])
+            print('一级分类', level1['cat_name'])
+            for level2 in level1.get('level2_list'):
+                self.insert_once(level2['cat_name'], level2['url'], level2['islast'], level2['level'], cat1_id)
+                print('二级分类', level2['cat_name'])
 
         self.operator.db.commit()
 

utils/mysqlopera.py

@@ -17,6 +17,7 @@ ENV = get_env()
 
 HOST_SET = {
     'test': '192.168.1.232',
+    'test234': '192.168.1.234',
     'produce': '172.18.137.37'
 }
 UP_SET = {
@@ -27,6 +28,9 @@ UP_SET = {
 
 def get_mysql_conf(db):
     host = HOST_SET[ENV]
+    if ENV == 'test' and db in ['hc360']:
+        host = HOST_SET['test234']
+        db = 'bigdata'
     up = UP_SET[ENV]
     conf = {
         'host': host,
@@ -38,10 +42,12 @@ def get_mysql_conf(db):
     }
     return conf
 
+
 class MySqlOperator(object):
     def __init__(self, db_key_name):
         config = get_mysql_conf(db_key_name)
         self.db = pymysql.connect(**config)
+
     def re_connect(self):
         try:
             self.db.ping()
@@ -413,6 +419,30 @@ class MySqlOperator(object):
             result = result[0] if result else None
             return result
 
+    def get_goods_id_by_name_brand(self, name, brand):
+        with self.db.cursor() as cursor:
+            sql = "SELECT goods_id FROM lie_goods WHERE goods_name=%s AND provider_name=%s"
+            cursor.execute(sql, (name, brand))
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
+    def get_goods_id_by_name(self, name):
+        with self.db.cursor() as cursor:
+            sql = "SELECT goods_id FROM lie_goods WHERE goods_name=%s"
+            cursor.execute(sql, name)
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
+    def get_goods_id_by_name_and_style(self, name, style):
+        with self.db.cursor() as cursor:
+            sql = "SELECT goods_id FROM lie_goods WHERE goods_name=%s and goods_name_style=%s"
+            cursor.execute(sql, (name, style))
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
     def get_brand_id_by_brand_name(self, bn):
         with self.db.cursor() as cursor:
             sql = "SELECT brand_id FROM lie_brand WHERE brand_name=%s"
@@ -435,6 +465,24 @@ class MySqlOperator(object):
             cursor.execute(sql, goods)
             self.db.commit()
 
+    def add_szlc_goods(self, goods):
+        with self.db.cursor() as cursor:
+            goods['add_time'] = self.get_ts()
+            sql = """
+                INSERT INTO lie_goods(cat_id, goods_sn, goods_name, goods_name_style, provider_name, brand_id,
+                goods_number, min_buynum, goods_brief, goods_desc, goods_thumb, goods_img, site_url, pdf_url, product_id, increment,
+                Encap, Package, goods_unit, encap_sulation, encap_sulation_number, sale_count, sale_unit, goods_weight,
+                add_time) VALUES
+                (%(cat_id)s, %(goods_sn)s, %(goods_name)s, %(goods_name_style)s, %(provider_name)s, %(brand_id)s,
+                 %(goods_number)s, %(min_buynum)s, %(goods_brief)s, %(goods_desc)s, %(goods_thumb)s, %(goods_img)s,
+                 %(site_url)s, %(pdf_url)s, %(goods_id)s, %(increment)s,
+                %(Encap)s, %(Package)s, %(goods_unit)s, %(encap_sulation)s, %(encap_sulation_number)s, %(sale_count)s, %(sale_unit)s, %(goods_weight)s, 
+                %(add_time)s)
+
+            """
+            cursor.execute(sql, goods)
+            self.db.commit()
+
     def add_goods_company(self, goods):
         with self.db.cursor() as cursor:
             goods['add_time'] = self.get_ts()
@@ -538,13 +586,12 @@ class MySqlOperator(object):
             cursor.execute(sql, (cost, gid))
             self.db.commit()
 
-
     # pc
     def insert_pc(self, data):
         with self.db.cursor() as cursor:
             sql = """
                 INSERT INTO lie_price_relations(goods_id, goods_name, goods_sn, provider_name, add_time, prices, 
-                prices_ori, avg_disparity_usd, avg_disparity_rmb, plat, compare) VALUES 
+                prices_ori, avg_disparity_usd, avg_disparity_rmb, platform, compare) VALUES 
                 (%(goods_id)s, %(goods_name)s, %(goods_sn)s, %(provider_name)s, %(batch_time)s, %(prices)s, 
                 %(prices_ori)s, %(avg_disparity_usd)s, %(avg_disparity_rmb)s, %(pn)s, %(cp)s)
             """
@@ -565,4 +612,92 @@ class MySqlOperator(object):
             cursor.execute(sql)
             self.db.commit()
 
+    # dzsc
+    def get_supplier_id_by_name(self, name):
+        with self.db.cursor() as cursor:
+            # cursor = self.db.cursor()
+            sql = "SELECT supplier_id FROM lie_dzsc_supplier WHERE supplier_name = %s"
+            cursor.execute(sql, name)
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
+    def insert_supplier_dzsc(self, data):
+        with self.db.cursor() as cursor:
+            sql = """
+            INSERT INTO lie_dzsc_supplier(supplier_name, qq, mobile, tel, address, site_url, contacts) 
+            VALUES (%(supplier_name)s, %(QQ)s, %(mobile)s, %(tel)s, %(address)s, %(supplier_url)s, %(contacts)s)
+            """
+            cursor.execute(sql, data)
+            self.db.commit()
+
+    # ic37
+    def get_supplier_id_by_name_ic37(self, name):
+        with self.db.cursor() as cursor:
+            # cursor = self.db.cursor()
+            sql = "SELECT supplier_id FROM lie_ic37_supplier WHERE supplier_name = %s"
+            cursor.execute(sql, name)
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
+    def insert_supplier_ic37(self, data):
+        with self.db.cursor() as cursor:
+            sql = """
+            INSERT INTO lie_ic37_supplier(supplier_name, qq, tel, address, site_url, contacts) 
+            VALUES (%(supplier_name)s, %(QQ)s, %(tel)s, %(address)s, %(supplier_url)s, %(contacts)s)
+            """
+            cursor.execute(sql, data)
+            self.db.commit()
+
+    def get_goods_id_by_name_brand_dzsc(self, name, brand):
+        with self.db.cursor() as cursor:
+            # cursor = self.db.cursor()
+            sql = "SELECT goods_id FROM lie_dzsc_goods WHERE goods_name = %s AND brand_name= %s"
+            cursor.execute(sql, (name, brand))
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
+    def insert_goods_dzsc(self, data):
+        with self.db.cursor() as cursor:
+            sql = """
+            INSERT INTO lie_dzsc_goods(goods_name, brand_name) 
+            VALUES (%(goods_name)s, %(brand_name)s)
+            """
+            cursor.execute(sql, data)
+            self.db.commit()
+
+    def get_iid_by_sid_and_gid(self, sid, gid):
+        with self.db.cursor() as cursor:
+            # cursor = self.db.cursor()
+            sql = "SELECT item_id FROM lie_dzsc_goods_items WHERE goods_id = %s AND supplier_id= %s"
+            cursor.execute(sql, (gid, sid))
+            result = cursor.fetchone()
+            result = result[0] if result else None
+            return result
+
+    def insert_item(self, sid, gid, data):
+        with self.db.cursor() as cursor:
+            sql = """
+            INSERT INTO lie_dzsc_goods_items(goods_id, supplier_id, encap, batch_sn) 
+            VALUES (%s, %s, %s, %s)
+            """
+            cursor.execute(sql, (gid, sid, data['encap'], data['batch_sn']))
+            self.db.commit()
+
 
+if __name__ == '__main__':
+    # pass
+    d = MySqlOperator('arrow')
+    print(d.get_ext_id_by_goods_id_attr_name(18000500790, 'Product Category'))
+    # data = (('SO180219037', '蔡**', '19**62A', 1519055525, '2018-02-19 23:52:05', 100, '已付3', '好好', '待取货3', 1519055525,
+    #          2519055525),
+    #         ('SO180219035', '蔡**', '19**62A', 1519055525, '2018-02-19 23:52:05', 100, '衣付3', '好好', '待取货3', 1519055525,
+    #          2519055525))
+    # d.insert_lcsz_order_list(data)
+    # data1 = (
+    #     ('SO180219034_3', 'SO180219037', '91383', '4.7nH ±0.3nH 编带', '高频电感', '0402', 500, 0.019, 0.0118, 1519055525),
+    #     ('SO180219034_4', 'SO180219037', '91383', '4.7nH ±0.3nH 编带', '高频电感', '0402', 500, 0.019, 0.0118, 1519055525),
+    # )
+    # d.insert_lcsz_order_details(data1)