Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
lzzzzl
/
param_data
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Snippets
Settings
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit
04a07679
authored
Nov 07, 2020
by
lzzzzl
Browse files
Options
_('Browse Files')
Download
Email Patches
Plain Diff
添加处理全部参数逻辑
parent
5f74c23a
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
74 additions
and
31 deletions
fun/param_data.py
param/param_class_redis.py
param/param_extract.py
param_task/task.py
param_task/value_task.py
param_task/value_task_all.py
fun/param_data.py
View file @
04a07679
...
...
@@ -154,6 +154,12 @@ class ParamData(object):
attr_id IN (
%
s) AND attr_value != ''"
%
(
i
,
ids
,
pams
)
return
DBHandler
.
read
(
bg_conn
,
sql
)
@staticmethod
def
get_dgk_goods_by_index
(
idx
,
bg_conn
):
sql
=
"SELECT attr_name,attr_value,attr_id,cat_id FROM lie_goods_attr_fields
%
d WHERE
\
attr_value != '' AND attr_value != '-' LIMIT 10000"
%
(
idx
)
return
DBHandler
.
read
(
bg_conn
,
sql
)
"""
写入attr value
"""
...
...
param/param_class_redis.py
View file @
04a07679
...
...
@@ -5,10 +5,9 @@ import json
from
config.db_config
import
DBConn
from
utils.log_handler
import
LogHandler
from
utils.db_handler
import
DBHandler
import
config.db_config
as
config
class
P
ARAM_CLASS_REDIS
(
object
):
class
P
aramClassRedis
(
object
):
"""
参数提取
...
...
@@ -63,5 +62,5 @@ class PARAM_CLASS_REDIS(object):
if
__name__
==
"__main__"
:
param
=
P
ARAM_CLASS_REDIS
()
param
=
P
aramClassRedis
()
param
.
spu_data
()
\ No newline at end of file
param/param_extract.py
View file @
04a07679
...
...
@@ -2,8 +2,8 @@
from
config.db_config
import
DBConn
from
utils.log_handler
import
LogHandler
from
utils.db_handler
import
DBHandler
from
fun.param_conn
import
P
ARAM_CONN
from
fun.param_data
import
P
ARAM_DATA
from
fun.param_conn
import
P
aramConn
from
fun.param_data
import
P
aramData
import
time
...
...
@@ -28,7 +28,7 @@ class PARAM_EXTRACT(object):
for
i
in
range
(
0
,
10
):
db
=
"liexin_sku_
%
d"
%
i
# 获取sku 连接
conn
=
P
ARAM_CONN
.
online_sku_conn
(
i
)
conn
=
P
aramConn
.
online_sku_conn
(
i
)
for
j
in
range
(
0
,
10
):
table
=
"lie_sku_
%
d"
%
j
sql
=
"SELECT spu_id,old_goods_id FROM
%
s WHERE supplier_id = 7"
%
table
...
...
@@ -52,23 +52,22 @@ class PARAM_EXTRACT(object):
bd_conn
=
DBConn
.
db_bigdata
()
# 数据
class_data
=
P
ARAM_DATA
.
class_data
(
class_conn
)
class_data
=
P
aramData
.
class_data
(
class_conn
)
# dgk数据遍历
index
=
0
for
i
in
self
.
dgk_dict
:
dgk_data
=
self
.
dgk_dict
[
i
]
if
i
==
5
:
for
spu_id
,
old_goods_id
in
dgk_data
:
cat_id
,
sku_id
=
PARAM_DATA
.
search_cat_id
(
old_goods_id
,
bd_conn
)
if
cat_id
!=
0
:
parent_id
=
class_data
[
cat_id
][
'parent_id'
]
class_id
=
class_data
[
cat_id
][
'class_id'
]
time
.
sleep
(
0.01
)
PARAM_DATA
.
update_spu_classId
(
int
(
spu_id
),
int
(
class_id
),
int
(
parent_id
),
spu_conn
,
self
.
log2
)
index
+=
1
if
index
%
10000
==
0
:
self
.
log1
.
info
(
"run - index: {index}"
.
format
(
index
=
index
))
for
spu_id
,
old_goods_id
in
dgk_data
:
cat_id
,
sku_id
=
ParamData
.
search_cat_id
(
old_goods_id
,
bd_conn
)
if
cat_id
!=
0
:
parent_id
=
class_data
[
cat_id
][
'parent_id'
]
class_id
=
class_data
[
cat_id
][
'class_id'
]
time
.
sleep
(
0.01
)
ParamData
.
update_spu_classId
(
int
(
spu_id
),
int
(
class_id
),
int
(
parent_id
),
spu_conn
,
self
.
log2
)
index
+=
1
if
index
%
10000
==
0
:
self
.
log1
.
info
(
"run - index: {index}"
.
format
(
index
=
index
))
"""
运行
...
...
@@ -76,4 +75,3 @@ class PARAM_EXTRACT(object):
def
run
(
self
):
self
.
dgk_data
()
self
.
update_classId
()
# PARAM_THREAD.thread_func_ten(self.update_classId, self.dgk_dict)
\ No newline at end of file
param_task/task.py
0 → 100644
View file @
04a07679
# -*- coding: utf-8 -*-
# !/usr/bin/env python
from
utils.date_handler
import
DateHandler
from
fun.param_data
import
ParamData
from
fun.param_trans
import
ParamTrans
from
fun.param_conn
import
ParamConn
from
config.db_config
import
DBConn
class
Task
:
def
__init__
(
self
):
self
.
now
=
DateHandler
.
now_datetime
()
self
.
param_data
=
ParamData
self
.
param_trans
=
ParamTrans
()
self
.
class_db
=
DBConn
.
db_class
()
self
.
bigdata_db
=
DBConn
.
db_bigdata
()
self
.
redis
=
ParamConn
.
local_redis_conn
()
\ No newline at end of file
param_task/value_task.py
View file @
04a07679
...
...
@@ -25,7 +25,6 @@ class ValueTask:
self
.
log1
=
LogHandler
(
'value_task_info'
,
stream
=
True
,
file
=
True
)
self
.
log2
=
LogHandler
(
'value_task_detail'
,
stream
=
False
,
file
=
True
)
self
.
now
=
DateHandler
.
now_datetime
()
self
.
empty_unit_id
=
26
"""
加载数据
...
...
@@ -66,7 +65,7 @@ class ValueTask:
mapping
=
self
.
param_data
.
class_mapping
(
self
.
class_db
)
unit_dict
=
self
.
param_data
.
get_unit_value
(
self
.
class_db
)
for
i
in
range
(
1
,
10
):
for
i
in
range
(
0
,
10
):
res
=
self
.
param_data
.
get_dgk_goods_by_ids
(
i
,
ids
,
pams
,
self
.
bigdata_db
)
for
row
in
res
:
attr_name
=
row
[
0
]
...
...
@@ -77,7 +76,7 @@ class ValueTask:
attr_id
=
mapping
[
cat_id
][
attr_name
]
value
,
unit
=
getattr
(
ParamTrans
,
self
.
param_dict
[
str
(
fields_attr_id
)])(
attr_value
)
if
value
!=
0
:
unit_id
=
unit_dict
[
unit
]
if
unit_dict
.
get
(
unit
)
else
self
.
empty_unit_id
unit_id
=
unit_dict
[
unit
]
if
unit_dict
.
get
(
unit
)
else
self
.
param_trans
.
empty_unit_id
# redis去重
if
self
.
redis
.
sadd
(
"attr_value"
,
str
(
attr_id
)
+
'-'
+
value
+
'-'
+
str
(
unit_id
))
!=
0
:
self
.
param_data
.
write_attr_value
(
attr_id
,
value
,
unit_id
,
"1"
,
""
,
self
.
now
,
self
.
now
,
...
...
param_task/value_task_all.py
View file @
04a07679
...
...
@@ -2,17 +2,14 @@
# !/usr/bin/env python
from
utils.log_handler
import
LogHandler
from
utils.excel_handler
import
ExcelHandler
from
fun.param_data
import
ParamData
from
fun.param_trans
import
ParamTrans
from
config.db_config
import
DBConn
from
param_task.task
import
Task
class
ValueTaskAll
:
class
ValueTaskAll
(
Task
)
:
def
__init__
(
self
):
self
.
param_data
=
ParamData
self
.
param_trans
=
ParamTrans
()
self
.
class_db
=
DBConn
.
db_class
()
super
()
.
__init__
()
self
.
attrs_id
=
[]
self
.
log1
=
LogHandler
(
'value_task_all_info'
,
stream
=
True
,
file
=
True
)
"""
...
...
@@ -41,11 +38,36 @@ class ValueTaskAll:
if
is_important
==
1
and
param_id
not
in
effect_attrs
:
class_id
=
self
.
param_data
.
get_class_id
(
second_classify_id
,
self
.
class_db
)
unit_id
=
self
.
param_trans
.
empty_unit_id
self
.
attrs_id
.
append
(
param_id
)
lx_attr_name
=
''
ParamData
.
write_class_attr
(
class_id
,
basic_attr_name
,
lx_attr_name
,
unit_id
,
param_id
,
self
.
class_db
)
# ParamData.write_class_attr(class_id, basic_attr_name, lx_attr_name, unit_id, param_id, self.class_db)
"""
写入所有数据
"""
def
write_all
(
self
):
self
.
log1
.
info
(
"运行任务"
)
mapping
=
self
.
param_data
.
class_mapping
(
self
.
class_db
)
print
(
len
(
mapping
))
for
i
in
range
(
0
,
10
):
res
=
self
.
param_data
.
get_dgk_goods_by_index
(
i
,
self
.
bigdata_db
)
for
row
in
res
:
attr_name
=
row
[
0
]
attr_value
=
row
[
1
]
fields_attr_id
=
row
[
2
]
cat_id
=
row
[
3
]
if
mapping
[
cat_id
]
.
get
(
attr_name
)
and
fields_attr_id
in
self
.
attrs_id
:
attr_id
=
mapping
[
cat_id
][
attr_name
]
value
=
attr_value
unit_id
=
self
.
param_trans
.
empty_unit_id
print
(
attr_id
,
value
,
unit_id
)
# if self.redis.sadd("attr_value", str(attr_id) + '-' + value + '-' + str(unit_id)) != 0:
# self.param_data.write_attr_value(attr_id, value, unit_id, "1", "", self.now, self.now,
# self.class_db)
def
run
(
self
):
self
.
load_attr
()
self
.
write_all
()
if
__name__
==
'__main__'
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment