dcap["phantomjs.page.settings.userAgent"]=("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")#设置user-agent请求头
dcap["phantomjs.page.settings.userAgent"]=("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36")#设置user-agent请求头
dcap["phantomjs.page.settings.loadImages"]=False
dcap["phantomjs.page.settings.loadImages"]=False
defgen_table_doms(url,cat_id):
defgen_table_doms(url,cat_id):
pid=os.getpid()
pid=os.getpid()
...
@@ -158,10 +158,11 @@ if __name__ == '__main__':
...
@@ -158,10 +158,11 @@ if __name__ == '__main__':
p=Pool(3)
p=Pool(3)
forreqinget_url():
forreqinget_url():
# for req in [('https://www.chip1stop.com/HKG/zh/view/searchResult/SearchResultTop?classCd=081015&classLv=3&subWinSearchFlg=false&searchType=2&dispAllFlg=true&searchFlg=false&direct_Flg=true',0)]:
url=req[0]
url=req[0]
cat_id=req[1]
cat_id=req[1]
# for url in ['https://www.chip1stop.com/CHN/zh/view/searchResult/SearchResultWithClassCd?classCd=010233&classLv=3&dispAllFlg=true&UseSession=false']: