Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
lichenggang
/
update_cate
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Snippets
Settings
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit
52b0ecd0
authored
Apr 22, 2019
by
lichenggang
Browse files
Options
_('Browse Files')
Download
Email Patches
Plain Diff
add china
parent
44d8054e
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
49 additions
and
4 deletions
plat/china/__init__.py
plat/china/cate_stock.py
plat/liebiao/cate_stock.py
utils/mysqlopera.py
plat/china/__init__.py
0 → 100644
View file @
52b0ecd0
File mode changed
plat/china/cate_stock.py
0 → 100644
View file @
52b0ecd0
from
utils.base
import
Module_Base
headers
=
{
"User-Agent"
:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"
,
}
a
=
Module_Base
()
url
=
'https://cn.china.cn/dianzi/'
resp
=
a
.
requests
.
get
(
url
,
headers
=
headers
)
resp
.
encoding
=
'gbk'
text
=
resp
.
text
dom
=
a
.
etree
.
HTML
(
text
)
level1_name_doms
=
dom
.
xpath
(
'//ul[@class="cate-list"]/li[position()<8]/div[@class="item"]//dt//a'
)
level1_content_doms
=
[]
for
i
in
range
(
1
,
8
):
level1_content1_doms1
=
dom
.
xpath
(
'//ul[@class="cate-list"]/li[position()={}]/div[@class="item"]//dd'
.
format
(
i
))
level1_content1_doms2
=
dom
.
xpath
(
'//ul[@class="cate-list"]/li[position()={}]/div[@class="cate-extra"]//dl[not(dt)]//dd'
.
format
(
i
))
level1_content1_doms
=
level1_content1_doms1
+
level1_content1_doms2
level1_content2_doms
=
dom
.
xpath
(
'//ul[@class="cate-list"]/li[position()={}]/div[@class="cate-extra"]//dl[dt]//dd'
.
format
(
i
))
level1_content_doms
.
append
(
level1_content1_doms
)
level1_content_doms
+=
level1_content2_doms
level1_doms
=
zip
(
level1_name_doms
,
level1_content_doms
)
levels
=
[]
for
level1_dom
in
level1_doms
:
level1
=
dict
()
level1
[
'cat_name'
]
=
level1_dom
[
0
]
.
xpath
(
'./text()'
)[
0
]
level1
[
'url'
]
=
level1_dom
[
0
]
.
xpath
(
'./@href'
)[
0
]
level1
[
'level'
]
=
1
level1
[
'islast'
]
=
0
level1
[
'parent_id'
]
=
0
level1
[
'level2_list'
]
=
[]
# print('一级分类',level1)
level2_doms
=
[]
for
i
in
level1_dom
[
1
]:
level2_doms
+=
i
.
xpath
(
'.//a'
)
for
level2_dom
in
level2_doms
:
level2
=
{}
level2
[
'cat_name'
]
=
level2_dom
.
xpath
(
'./text()'
)[
0
]
level2
[
'url'
]
=
level2_dom
.
xpath
(
'./@href'
)[
0
]
level2
[
'level'
]
=
2
level2
[
'islast'
]
=
1
level1
[
'level2_list'
]
.
append
(
level2
)
# print('二级分类',level2)
levels
.
append
(
level1
)
# print(levels)
\ No newline at end of file
plat/liebiao/cate_stock.py
View file @
52b0ecd0
from
utils.base
import
Module_Base
from
utils.base
import
Module_Base
import
asyncio
from
aiohttp
import
ClientSession
headers
=
{
headers
=
{
"User-Agent"
:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"
,
"User-Agent"
:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"
,
...
...
utils/mysqlopera.py
View file @
52b0ecd0
...
@@ -5,7 +5,7 @@ import time
...
@@ -5,7 +5,7 @@ import time
import
pymysql
import
pymysql
COMPANY_PN_LIST
=
[
'hc360'
,
'huangye88'
,
'pangwo'
,
'coovee'
,
'liebiao'
]
COMPANY_PN_LIST
=
[
'hc360'
,
'huangye88'
,
'pangwo'
,
'coovee'
,
'liebiao'
,
'china'
]
def
get_env
():
def
get_env
():
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment