Skip to content

lichenggang / update_cate

Go to a project
Commit d3030da7 by lichenggang
Options

add proxy

parent bcbdc6cf
Inline Side-by-side
Showing with 232 additions and 28 deletions
plat/hc360/__init__.py 0 → 100644
File mode changed
plat/hc360/cate_stock.py 0 → 100644
from utils.base import Module_Base
kw_base = 'https://s.hc360.com/seller/search.html?a=1&w={}'
headers={
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding":"gzip, deflate, br",
"Accept-Language":"zh-CN,zh;q=0.9",
"Cache-Control":"max-age=0",
"Connection":"keep-alive",
"Host":"www.hc360.com",
"Upgrade-Insecure-Requests":"1",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
}
a = Module_Base()
url = 'https://www.hc360.com/'
resp = a.requests.get(url,headers=headers).text
dom = a.etree.HTML(resp)
level1_name_doms=dom.xpath('//ul[@class="Bside_Uwrap"]//li')
level1_content_doms=dom.xpath('//div[@id="category"]/div')
level1_doms=zip(level1_name_doms,level1_content_doms)
levels=[]
for level1_dom in level1_doms:
level1=dict()
level1['cat_name'] = level1_dom[0].xpath('string(.)')
level1['url'] = level1_dom[0].xpath('.//div[@class="rigNav_cont"]//a/@href')[0]
level1['level'] = 1
level1['islast'] = 0
level1['parent_id'] = 0
level1['level2_list'] = []
level2_doms=level1_dom[1].xpath('.//div[@class="sideBarLeft"]//ul//li//a')
# print('一级分类',level1)
for level2_dom in level2_doms:
level2 = {}
level2['cat_name'] = level2_dom.xpath('./text()')[0]
level2['url'] = kw_base.format(level2['cat_name'])
level2['level'] = 2
level2['islast'] = 1
level1['level2_list'].append(level2)
# print('二级分类',level2)
levels.append(level1)
# print(levels)
\ No newline at end of file
update_scrip.py
...@@ -8,4 +8,7 @@ if __name__ == '__main__': ...@@ -8,4 +8,7 @@ if __name__ == '__main__':
module = importlib.import_module(module_name) module = importlib.import_module(module_name)
updater = Updater(plat_name) updater = Updater(plat_name)
updater.up(module.levels) if plat_name in ['hc360']:
updater.up_once(module.levels)
else:
updater.up(module.levels)
updateopera.py
from utils.mysqlopera import MySqlOperator from utils.mysqlopera import MySqlOperator
class Updater(): class Updater():
def __init__(self, db_name): def __init__(self, pn_name):
self.operator = MySqlOperator(db_name) self.operator = MySqlOperator(pn_name)
self.table_name = 'lie_' + pn_name + '_' + 'category' if pn_name in ['hc360'] else 'lie_category'
def clean_db(self): def clean_db(self):
with self.operator.db.cursor() as cursor: with self.operator.db.cursor() as cursor:
sql = "update lie_category set is_show = 0" sql = "update {} set is_show = 0".format(self.table_name)
cursor.execute(sql) cursor.execute(sql)
def get_cat_id(self, cat_name): def get_cat_id(self, cat_name):
with self.operator.db.cursor() as cursor: with self.operator.db.cursor() as cursor:
sql = 'select cat_id from lie_category where cat_name =%s' sql = 'select cat_id from {} where cat_name =%s'.format(self.table_name)
cursor.execute(sql, cat_name) cursor.execute(sql, cat_name)
result = cursor.fetchone() result = cursor.fetchone()
result = result[0] if result else None result = result[0] if result else None
return result return result
def update(self, cat_id, url, islast, level, parent_id=0,is_show=1): def update(self, cat_id, url, islast, level, parent_id=0, is_show=1):
with self.operator.db.cursor() as cursor: with self.operator.db.cursor() as cursor:
sql = 'UPDATE lie_category SET url=%s,islast=%s,level=%s,is_show=%s, parent_id=%s where cat_id=%s' sql = 'UPDATE {} SET url=%s,islast=%s,level=%s,is_show=%s, parent_id=%s where cat_id=%s'.format(
data=(url,islast, level, is_show,parent_id,cat_id) self.table_name)
data = (url, islast, level, is_show, parent_id, cat_id)
cursor.execute(sql, data) cursor.execute(sql, data)
def insert(self, cat_name, url, islast, level, parent_id=0, is_show=1 , sort_order=50, page_count=1): def insert(self, cat_name, url, islast, level, parent_id=0, is_show=1, sort_order=50, page_count=1):
with self.operator.db.cursor() as cursor: with self.operator.db.cursor() as cursor:
sql = 'INSERT into {}(cat_name,parent_id,sort_order,is_show,url,islast,level,page_count) values' \
sql = 'INSERT into lie_category(cat_name,parent_id,sort_order,is_show,url,islast,level,page_count) values' \ '(%s,%s,%s,%s,%s,%s,%s,%s)'.format(self.table_name)
'(%s,%s,%s,%s,%s,%s,%s,%s)'
data = (cat_name, parent_id, sort_order, is_show, url, islast, level, page_count) data = (cat_name, parent_id, sort_order, is_show, url, islast, level, page_count)
cursor.execute(sql, data) cursor.execute(sql, data)
cursor.execute("select max(cat_id) from lie_category") cursor.execute("select max(cat_id) from {}".format(self.table_name))
cat_id=cursor.fetchone()[0] cat_id = cursor.fetchone()[0]
return cat_id return cat_id
def up(self,levels):
def up(self, levels):
self.clean_db() self.clean_db()
for level1 in levels: for level1 in levels:
cat1_id=self.get_cat_id(level1['cat_name']) cat1_id = self.get_cat_id(level1['cat_name'])
if cat1_id: if cat1_id:
self.update(cat1_id,level1['url'],level1['islast'],level1['level']) self.update(cat1_id, level1['url'], level1['islast'], level1['level'])
else: else:
cat1_id = self.insert(level1['cat_name'], level1['url'], level1['islast'], level1['level'], level1['parent_id']) cat1_id = self.insert(level1['cat_name'], level1['url'], level1['islast'], level1['level'],
print('一级分类',level1['cat_name']) level1['parent_id'])
print('一级分类', level1['cat_name'])
for level2 in level1.get('level2_list'): for level2 in level1.get('level2_list'):
if level2['islast']: if level2['islast']:
cat2_id=self.get_cat_id(level2['cat_name']) cat2_id = self.get_cat_id(level2['cat_name'])
if cat2_id: if cat2_id:
self.update(cat2_id,level2['url'],level2['islast'],level2['level'],cat1_id) self.update(cat2_id, level2['url'], level2['islast'], level2['level'], cat1_id)
else: else:
self.insert(level2['cat_name'],level2['url'],level2['islast'],level2['level'],cat1_id) self.insert(level2['cat_name'], level2['url'], level2['islast'], level2['level'], cat1_id)
print('二级分类',level2['cat_name']) print('二级分类', level2['cat_name'])
else: else:
pass #TODO 增加三级分类 pass # TODO 增加三级分类
self.operator.db.commit()
def insert_once(self, cat_name, url, islast, level, parent_id=0, page_count=1):
with self.operator.db.cursor() as cursor:
sql = 'INSERT into {}(cat_name,parent_id,url,islast,level,page_count) values' \
'(%s,%s,%s,%s,%s,%s)'.format(self.table_name)
data = (cat_name, parent_id, url, islast, level, page_count)
cursor.execute(sql, data)
cursor.execute("select max(cat_id) from {}".format(self.table_name))
cat_id = cursor.fetchone()[0]
return cat_id
def up_once(self, levels):
for level1 in levels:
cat1_id = self.insert_once(level1['cat_name'], level1['url'], level1['islast'], level1['level'],
level1['parent_id'])
print('一级分类', level1['cat_name'])
for level2 in level1.get('level2_list'):
self.insert_once(level2['cat_name'], level2['url'], level2['islast'], level2['level'], cat1_id)
print('二级分类', level2['cat_name'])
self.operator.db.commit() self.operator.db.commit()
...@@ -61,4 +86,4 @@ class Updater(): ...@@ -61,4 +86,4 @@ class Updater():
# with uper.db.cursor() as cursor: # with uper.db.cursor() as cursor:
# sql = 'UPDATE lie_category SET url=%s,islast=%s,level=%s,is_show=%s, parent_id=%s where cat_id=%s' # sql = 'UPDATE lie_category SET url=%s,islast=%s,level=%s,is_show=%s, parent_id=%s where cat_id=%s'
# cursor.execute(sql,('https://hk.element14.com/c/test-measurement/spectrometers',1,2,1,2379,'2391')) # cursor.execute(sql,('https://hk.element14.com/c/test-measurement/spectrometers',1,2,1,2379,'2391'))
# uper.db.commit() # uper.db.commit()
\ No newline at end of file
utils/mysqlopera.py
...@@ -17,6 +17,7 @@ ENV = get_env() ...@@ -17,6 +17,7 @@ ENV = get_env()
HOST_SET = { HOST_SET = {
'test': '192.168.1.232', 'test': '192.168.1.232',
'test234': '192.168.1.234',
'produce': '172.18.137.37' 'produce': '172.18.137.37'
} }
UP_SET = { UP_SET = {
...@@ -27,6 +28,9 @@ UP_SET = { ...@@ -27,6 +28,9 @@ UP_SET = {
def get_mysql_conf(db): def get_mysql_conf(db):
host = HOST_SET[ENV] host = HOST_SET[ENV]
if ENV == 'test' and db in ['hc360']:
host = HOST_SET['test234']
db = 'bigdata'
up = UP_SET[ENV] up = UP_SET[ENV]
conf = { conf = {
'host': host, 'host': host,
...@@ -38,10 +42,12 @@ def get_mysql_conf(db): ...@@ -38,10 +42,12 @@ def get_mysql_conf(db):
} }
return conf return conf
class MySqlOperator(object): class MySqlOperator(object):
def __init__(self, db_key_name): def __init__(self, db_key_name):
config = get_mysql_conf(db_key_name) config = get_mysql_conf(db_key_name)
self.db = pymysql.connect(**config) self.db = pymysql.connect(**config)
def re_connect(self): def re_connect(self):
try: try:
self.db.ping() self.db.ping()
...@@ -413,6 +419,30 @@ class MySqlOperator(object): ...@@ -413,6 +419,30 @@ class MySqlOperator(object):
result = result[0] if result else None result = result[0] if result else None
return result return result
def get_goods_id_by_name_brand(self, name, brand):
with self.db.cursor() as cursor:
sql = "SELECT goods_id FROM lie_goods WHERE goods_name=%s AND provider_name=%s"
cursor.execute(sql, (name, brand))
result = cursor.fetchone()
result = result[0] if result else None
return result
def get_goods_id_by_name(self, name):
with self.db.cursor() as cursor:
sql = "SELECT goods_id FROM lie_goods WHERE goods_name=%s"
cursor.execute(sql, name)
result = cursor.fetchone()
result = result[0] if result else None
return result
def get_goods_id_by_name_and_style(self, name, style):
with self.db.cursor() as cursor:
sql = "SELECT goods_id FROM lie_goods WHERE goods_name=%s and goods_name_style=%s"
cursor.execute(sql, (name, style))
result = cursor.fetchone()
result = result[0] if result else None
return result
def get_brand_id_by_brand_name(self, bn): def get_brand_id_by_brand_name(self, bn):
with self.db.cursor() as cursor: with self.db.cursor() as cursor:
sql = "SELECT brand_id FROM lie_brand WHERE brand_name=%s" sql = "SELECT brand_id FROM lie_brand WHERE brand_name=%s"
...@@ -435,6 +465,24 @@ class MySqlOperator(object): ...@@ -435,6 +465,24 @@ class MySqlOperator(object):
cursor.execute(sql, goods) cursor.execute(sql, goods)
self.db.commit() self.db.commit()
def add_szlc_goods(self, goods):
with self.db.cursor() as cursor:
goods['add_time'] = self.get_ts()
sql = """
INSERT INTO lie_goods(cat_id, goods_sn, goods_name, goods_name_style, provider_name, brand_id,
goods_number, min_buynum, goods_brief, goods_desc, goods_thumb, goods_img, site_url, pdf_url, product_id, increment,
Encap, Package, goods_unit, encap_sulation, encap_sulation_number, sale_count, sale_unit, goods_weight,
add_time) VALUES
(%(cat_id)s, %(goods_sn)s, %(goods_name)s, %(goods_name_style)s, %(provider_name)s, %(brand_id)s,
%(goods_number)s, %(min_buynum)s, %(goods_brief)s, %(goods_desc)s, %(goods_thumb)s, %(goods_img)s,
%(site_url)s, %(pdf_url)s, %(goods_id)s, %(increment)s,
%(Encap)s, %(Package)s, %(goods_unit)s, %(encap_sulation)s, %(encap_sulation_number)s, %(sale_count)s, %(sale_unit)s, %(goods_weight)s,
%(add_time)s)
"""
cursor.execute(sql, goods)
self.db.commit()
def add_goods_company(self, goods): def add_goods_company(self, goods):
with self.db.cursor() as cursor: with self.db.cursor() as cursor:
goods['add_time'] = self.get_ts() goods['add_time'] = self.get_ts()
...@@ -538,13 +586,12 @@ class MySqlOperator(object): ...@@ -538,13 +586,12 @@ class MySqlOperator(object):
cursor.execute(sql, (cost, gid)) cursor.execute(sql, (cost, gid))
self.db.commit() self.db.commit()
# pc # pc
def insert_pc(self, data): def insert_pc(self, data):
with self.db.cursor() as cursor: with self.db.cursor() as cursor:
sql = """ sql = """
INSERT INTO lie_price_relations(goods_id, goods_name, goods_sn, provider_name, add_time, prices, INSERT INTO lie_price_relations(goods_id, goods_name, goods_sn, provider_name, add_time, prices,
prices_ori, avg_disparity_usd, avg_disparity_rmb, plat, compare) VALUES prices_ori, avg_disparity_usd, avg_disparity_rmb, platform, compare) VALUES
(%(goods_id)s, %(goods_name)s, %(goods_sn)s, %(provider_name)s, %(batch_time)s, %(prices)s, (%(goods_id)s, %(goods_name)s, %(goods_sn)s, %(provider_name)s, %(batch_time)s, %(prices)s,
%(prices_ori)s, %(avg_disparity_usd)s, %(avg_disparity_rmb)s, %(pn)s, %(cp)s) %(prices_ori)s, %(avg_disparity_usd)s, %(avg_disparity_rmb)s, %(pn)s, %(cp)s)
""" """
...@@ -565,4 +612,92 @@ class MySqlOperator(object): ...@@ -565,4 +612,92 @@ class MySqlOperator(object):
cursor.execute(sql) cursor.execute(sql)
self.db.commit() self.db.commit()
# dzsc
def get_supplier_id_by_name(self, name):
with self.db.cursor() as cursor:
# cursor = self.db.cursor()
sql = "SELECT supplier_id FROM lie_dzsc_supplier WHERE supplier_name = %s"
cursor.execute(sql, name)
result = cursor.fetchone()
result = result[0] if result else None
return result
def insert_supplier_dzsc(self, data):
with self.db.cursor() as cursor:
sql = """
INSERT INTO lie_dzsc_supplier(supplier_name, qq, mobile, tel, address, site_url, contacts)
VALUES (%(supplier_name)s, %(QQ)s, %(mobile)s, %(tel)s, %(address)s, %(supplier_url)s, %(contacts)s)
"""
cursor.execute(sql, data)
self.db.commit()
# ic37
def get_supplier_id_by_name_ic37(self, name):
with self.db.cursor() as cursor:
# cursor = self.db.cursor()
sql = "SELECT supplier_id FROM lie_ic37_supplier WHERE supplier_name = %s"
cursor.execute(sql, name)
result = cursor.fetchone()
result = result[0] if result else None
return result
def insert_supplier_ic37(self, data):
with self.db.cursor() as cursor:
sql = """
INSERT INTO lie_ic37_supplier(supplier_name, qq, tel, address, site_url, contacts)
VALUES (%(supplier_name)s, %(QQ)s, %(tel)s, %(address)s, %(supplier_url)s, %(contacts)s)
"""
cursor.execute(sql, data)
self.db.commit()
def get_goods_id_by_name_brand_dzsc(self, name, brand):
with self.db.cursor() as cursor:
# cursor = self.db.cursor()
sql = "SELECT goods_id FROM lie_dzsc_goods WHERE goods_name = %s AND brand_name= %s"
cursor.execute(sql, (name, brand))
result = cursor.fetchone()
result = result[0] if result else None
return result
def insert_goods_dzsc(self, data):
with self.db.cursor() as cursor:
sql = """
INSERT INTO lie_dzsc_goods(goods_name, brand_name)
VALUES (%(goods_name)s, %(brand_name)s)
"""
cursor.execute(sql, data)
self.db.commit()
def get_iid_by_sid_and_gid(self, sid, gid):
with self.db.cursor() as cursor:
# cursor = self.db.cursor()
sql = "SELECT item_id FROM lie_dzsc_goods_items WHERE goods_id = %s AND supplier_id= %s"
cursor.execute(sql, (gid, sid))
result = cursor.fetchone()
result = result[0] if result else None
return result
def insert_item(self, sid, gid, data):
with self.db.cursor() as cursor:
sql = """
INSERT INTO lie_dzsc_goods_items(goods_id, supplier_id, encap, batch_sn)
VALUES (%s, %s, %s, %s)
"""
cursor.execute(sql, (gid, sid, data['encap'], data['batch_sn']))
self.db.commit()
if __name__ == '__main__':
# pass
d = MySqlOperator('arrow')
print(d.get_ext_id_by_goods_id_attr_name(18000500790, 'Product Category'))
# data = (('SO180219037', '蔡**', '19**62A', 1519055525, '2018-02-19 23:52:05', 100, '已付3', '好好', '待取货3', 1519055525,
# 2519055525),
# ('SO180219035', '蔡**', '19**62A', 1519055525, '2018-02-19 23:52:05', 100, '衣付3', '好好', '待取货3', 1519055525,
# 2519055525))
# d.insert_lcsz_order_list(data)
# data1 = (
# ('SO180219034_3', 'SO180219037', '91383', '4.7nH ±0.3nH 编带', '高频电感', '0402', 500, 0.019, 0.0118, 1519055525),
# ('SO180219034_4', 'SO180219037', '91383', '4.7nH ±0.3nH 编带', '高频电感', '0402', 500, 0.019, 0.0118, 1519055525),
# )
# d.insert_lcsz_order_details(data1)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment