Skip to content

杨树贤 / search_server

Go to a project
Commit 6dd5e4e8 by huangchengyi
Options

1.0

parent e9e31500
Inline Side-by-side
Showing with 18 additions and 406 deletions
.gitignore
...@@ -9,3 +9,4 @@ cmd.exe~ ...@@ -9,3 +9,4 @@ cmd.exe~
/cmd/*.exe~ /cmd/*.exe~
/cmd/logs /cmd/logs
/bat/logs/ /bat/logs/
/conf
conf/config_dev.ini deleted 100644 → 0
[web]
port = 9000
mode = debug
cors_domain = http://bom.liexin.com
[message]
api_domain = http://api.ichunt.com/msg/sendMessageByAuto
api_md5_str = fh6y5t4rr351d2c3bryi
[goods]
api_url = http://192.168.2.232:60004
[es]
url = http://192.168.2.232:9200
urls = http://192.168.2.232:9200,http://192.168.2.232:9200
index_name = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,zhuanmai,peigenesis,powell,rs,buerklin,liexin_ziying
search_supplier = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,peigenesis,powell,rs,buerklin,zhuanmai
hk_delivery_type_supplier = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,peigenesis,powell,rs,buerklin
attr_index = goods_map
[database]
user_name = root
password = root
host = 192.168.2.239
database = test
table_prefix =
type = mysql
[redis]
write_host = 192.168.1.235:6379
read_host = 192.168.1.235:6379
write_password = icDb29mLy2s
read_password = icDb29mLy2s
max_idle = 10
max_active = 10
idle_timeout = 20
[mongo]
host = 127.0.0.1:27017
database = liexin_config
[brand]
#是否开启排除品牌
IS_NOT_TI = 1
TI_LY_BRAND_IDS = 23,45069,7754,17484,43215,13421,5220,7636,44534,43556,44543,9429,758,9470,10430,10653,10707,11227,11276,11819,12608,12965,12997,13327,13675,13680,13752
ti_zy_brand_id = 23
\ No newline at end of file
conf/config_production.ini deleted 100644 → 0
[web]
port = 20001
mode = release
cors_domain = bom.ichunt.com
[message]
api_domain = http://api.ichunt.com/msg/sendMessageByAuto
api_md5_str = fh6y5t4rr351d2c3bryi
[goods]
api_url = http://47.106.60.211:60004
[es]
url = http://172.18.137.29:9211
urls = http://172.18.137.29:9211,http://172.18.137.30:9211
index_name = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,zhuanmai,peigenesis,powell,rs,buerklin,liexin_ziying
search_supplier = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,peigenesis,powell,rs,buerklin,zhuanmai
hk_delivery_type_supplier = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,peigenesis,powell,rs,buerklin
attr_index = goods_map
[redis]
write_host = 172.18.137.38:6379
read_host = 172.18.137.39:6379
write_password = icDb29mLy2s
read_password = icDb29mLy2s
max_idle = 10
max_active = 10
idle_timeout = 20
conf/database.ini deleted 100644 → 0
[gorm]
mode = debug
[spu]
user_name = spu
password = spu
host = 192.168.1.234
database = liexin_spu
table_prefix =lie_
type = mysql
#线上地址
#SupDbUser:Supssy2@@!!@$#yxy@172.18.137.21:3306/liexin_supp
[supp]
user_name = liexin_ass
password = `liexin_ass#zsyM`
host = 192.168.2.232
database = liexin_ass
table_prefix =lie_
type = mysql
[cms]
user_name = ichuntcms
password = `ichuntcms#zsyM`
host = 192.168.2.232
database = ichuntcms
table_prefix =
type = mysql
conf/message.ini deleted 100644 → 0
[DINGDING]
SEARCH_API_MONITOR = 6d0fa85e01a02c39347d011ae973fd21b76c6c7ce582d3ea470c6b65a318848d
\ No newline at end of file
conf/rabmq_key.ini deleted 100644 → 0
[rabmq]
url = amqp://guest:guest@192.168.2.232:5672/
;存放本系统所有的队列名称
[rabmq_all]
; bom任务id
MQ_BOM_ITEMS_LIST=bom_items_list
; bom更新数据队列
MQ_BOM_SKU_LIST=bom_sku_list
;联营数据更新-推送入队列-》go后台任务消费（斌哥脚本）
SEARCH_SKU_UPDATE_LIST=search_sku_update_list
;联营数据新增-推送入队列-》go后台任务消费（斌哥脚本）
SEARCH_SKU_LIST=search_sku_list
; bom任务id
ZIYING_MQ_BOM_ITEMS_LIST=ziying_bom_items_list
; bom更新数据队列
ZIYING_MQ_BOM_SKU_LIST=ziying_bom_sku_list
\ No newline at end of file
conf/redis_config.ini deleted 100644 → 0
[default_redis_read]
host = 192.168.1.235:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
[default_redis_write]
host = 192.168.1.235:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
[api_redis]
host = 192.168.1.235:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
\ No newline at end of file
conf/redis_key.ini deleted 100644 → 0
;存放本系统所有的redis键值对和解析
[redis_all]
;供应商redis的首key
SUPPLIER_REDIS_PRE=SUPPLIER_REDIS_INFO_
;需要展示的供应商列表
SUPPLIER_REDIS_LIST_PRE=SUPPLIER_REDIS_LIST_INFO_
;新增sku时判读sku是否存在
SKU_UNIQUE_JUDGE=sku_unique_judge
;新增sku时判读sku是否存在
SPU_UNIQUE_JUDGE=sku_unique_judge
;品牌信息判断,根据md5后的品牌名称获取id
BRAND_NAME_ALL=brand_name_all
;所有联营sku缓存
SKU=sku
;所有联营spu缓存
SPU=spu
;品牌id，获取品牌名称
BRAND=brand
;供应商品牌信息判断，存在多对一的情况,根据md5后的品牌名称获取标准品牌id
SUPPLIER_BRAND_ALL=supplier_brand_all
;统计调用外联接口的次数的key的前缀 后面接mouser、verical、arrow、master
SEARCH_API_TOTAL_PRE=search_api_total_
;统计调用外联接口失败的次数的key的前缀 后面接mouser、verical、arrow、master
SEARCH_API_ERORR_PRE=search_api_overtime_
;曝光时间记录哈希集合
SEARCH_SHOW_SKU_TIME=search_show_sku_time
;获取关税信息，根据goods_name,brand_name获取
TAX_CUSTOMS_INFO=tax_customs_info
;存放联营外链原始映射
SKU_RAW_MAP=sku_raw_map
[redis_ziyin]
AGG_TABLE=sz_ziying_class_aggs
FIRST_LISTS_TABLE=ziying_first_lists
ZIYING_CLASS_KEY=ziying_class_string
TAX_CUSTOMS_INFO=tax_customs_info
;存放联营所有sku
SKU=sku
;存放联营所有spu
SPU=SPU
;存放digikey 型号映射
SKU_RAW_MAP=sku_raw_map
POOL_ALIKE_GOODS=pool_alike_goods
conf/search.ini deleted 100644 → 0
[SUPPLIER_ALL]
1 = future
2 = powerandsignal
3 = rochester
4 = tme
5 = verical
6 = element14
7 = digikey
8 = chip1stop
9 = aipco
10 = arrow
11 = bisco
12 = alliedelec
13 = avnet
14 = mouser
#16 = liexin_lianying1
17 = zhuanmai
18 = liexin_ti
19 = peigenesis
20 = powell
21 = rs
1676 = buerklin
[SEARCH_SUPPLIER_MERGE]
17 = zhuanmai
1 = future
3 = rochester
6 = element14
7 = digikey
8 = chip1stop
9 = aipco
12 = alliedelec
13 = avnet
19 = peigenesis
20 = powell
21 = rs
[SEARCH_SUPPLIER]
1 = future
#2 = powerandsignal
3 = rochester
4 = tme
5 = verical
6 = element14
7 = digikey
8 = chip1stop
9 = aipco
10 = arrow
#11 = bisco
12 = alliedelec
13 = avnet
14 = mouser
#15 = company
#16 = liexin_lianying1
17 = zhuanmai
#18 = liexin_ti
19 = peigenesis
20 = powell
21 = rs
#22 = liexin_sell
#100 = ziying
#101 = liexin_jingdiao
1676 = buerklin
#相似物料供应商
[ALIKE_SUPPLIER]
1 = ti
[common]
MOUSER_API = 'http://footstone.liexin.net/webapi/handle_mouser'
[BRAND]
IS_NOT_TI = 0
TI_LY_BRAND_IDS = 23,45069,7754,17484,43215,13421,5220,7636,44534,43556,44543,9429,758,9470,10430,10653,10707,11227,11276,11819,12608,12965,12997,13327,13675,13680,13752
TI_ZY_BRAND_IDS = 23
[supplier]
# 供应商redis的首key
SUPPLIER_REDIS_PRE = SUPPLIER_REDIS_INFO_
[ZIYING_BRAND_REPLACE]
FH = FH(风华)
BL = BL(上海贝岭)
TE = TE(泰科)
FM = FM(富满)
CJ = CJ(长电)
ST = ST(意法)
SK = SK(时科)
ON = ON(安森美)
TM = TM(天微)
TP = TP(拓微)
TI = TI(德州仪器)
MD = MD(明达)
EG = EG(屹晶微)
HF = HF(宏发)
DY = DY(迪一)
MK = MK(米客方德)
国民 = Nationz(国民技术)
中芯 = SMIC(中芯国际)
富士 = Fuji Electric(富士电机)
国星 = Nationstar(国星光电)
贝岭 = BL(上海贝岭)
[ZIYING_CONFIG]
Index = liexin_ziying
[SEARCH_KEYWORD_MAPPING]
`^(.* )?([\d\.]+)(欧|欧姆|R|r)( .*)?$` = ${1}${2}Ω${4}
`^(.* )?([\d\.]+)(U|u|μ)(.?)( .*)?$` = ${1}${2}Μ${4}${5}
`COG` = C0G
`NPO` = NP0
`华科|华新科技|华新科` = 华新
[mathExactGoodsName]
0=M7
1=M1
2=M4
3=M2
4=A7
5=F7
6=X2
7=G4
8=X4
9=m7
10=m1
11=m4
12=m2
13=a7
14=f7
15=x2
16=g4
17=x4
;自营以某些单位进行搜索时，进行特殊处理
[ZIYING_MEASURE]
0=kΩ
1=uf
2=v
3=kg
4=MHz
5=pf
6=pin
7=ppm
8=kw
[OFFICIAL_WEBSITE]
1 = https://www.futureelectronics.cn/search/?text=liexin
2 = http://www.powerandsignal.com/Products/Search?searchBox=liexin
3 = https://www.rocelec.com/search?q=liexin
4 = https://www.tme.eu/zh/katalog/#search=liexin
5 = https://www.verical.com/s/liexin/
6 = https://cn.element14.com/search?st=liexin
7 = https://www.digikey.com/products/en?keywords=liexin
8 = https://www.chip1stop.com/CHN/zh/view/searchResult/SearchResultTop?keyword=liexin
9 = http://www.aipco.com/search.php?pn=liexin
10 = https://www.arrow.com/en/products/search?cat=&q=liexin
11 = https://www.biscoind.com?SEARCHTEXT=liexin
12 = https://www.alliedelec.com/view/search?keyword=liexin
13 = https://www.avnet.com/shop/apac/search/liexin
14 = https://www.mouser.com/Search/Refine?Keyword=liexin
15 =
16 =
18 =
19 = https://www.peigenesis.cn/cn/shop/f/liexin.html
20 = https://www.powell.com/e2wItemQuickSearch.aspx?searchText=liexin
21 = https://uk.rs-online.com/web/c/semiconductors/amplifiers-comparators/op-amps/?searchTerm=liexin
22 =
1672 = https://www.masterelectronics.com/parts.aspx?text=liexin
1673 = https://www.rutronik24.com/search-result/qs:liexin
1675 = https://www.zaikostore.com/zaikostore/stockList?productName_forFind=liexin
1676 = https://www.buerklin.com/en/search?text=liexin
1677 = https://www.microchipdirect.com/product/search/all/liexin
1678 = https://estore.heilindasia.com/search.asp?p=liexin
1680 = http://shop.wpgam.com/WPGAStore/index.php?route=product/search&filter_description=true&filter_name=liexin
[AUTH]
SUPER_AUTH_KEY = fh6y5t4rr351d2c3bryi
SEARCH_TOKEN_EXPIRE_TIME = 30
[UNIT_SUPPLIER_LOG_CODE]
333333 = arrow
444444 = master
555555 = mouser
666666 = verical
777777 = tme
888888 = buerklin
999999 = zhuanmai
[SEARCH_API_LOG]
SEARCH_API_ERROR_PRE = search_api_overtime_
[LY_UPDATE_LIMIT]
;联营是否开启立刻更新：0否 1是
isUpdate = 0
controller/ly_mouser_controller.go
...@@ -34,6 +34,7 @@ func GetMouserData(c *gin.Context) { ...@@ -34,6 +34,7 @@ func GetMouserData(c *gin.Context) {
errCode = 1 errCode = 1
} }
common.PrintDebugHtml("-----执行结束----") //debug
c.JSON(200, common.BomResponse{ c.JSON(200, common.BomResponse{
ErrCode: errCode, ErrCode: errCode,
ErrMsg: "查询成功", ErrMsg: "查询成功",
......
go.mod
...@@ -40,6 +40,7 @@ require ( ...@@ -40,6 +40,7 @@ require (
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
gopkg.in/olivere/elastic.v5 v5.0.85 gopkg.in/olivere/elastic.v5 v5.0.85
sigs.k8s.io/yaml v1.2.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect
xorm.io/core v0.7.2-0.20190928055935-90aeac8d08eb
) )
replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
service/ly/common.go
...@@ -40,7 +40,6 @@ func (t *CommonLyService) LyServerRun(){ ...@@ -40,7 +40,6 @@ func (t *CommonLyService) LyServerRun(){
} }
} }
/* /*
联营入口 联营入口
联营大类，包括修改新增 sku es redis ，推送数据到队列让其他go服务更新数据到mysql 联营大类，包括修改新增 sku es redis ，推送数据到队列让其他go服务更新数据到mysql
...@@ -67,10 +66,6 @@ func (t *CommonLyService) GetSkuByGoodsSn(goodsList map[string]*model.LyClearGoo ...@@ -67,10 +66,6 @@ func (t *CommonLyService) GetSkuByGoodsSn(goodsList map[string]*model.LyClearGoo
common.PrintDebugHtml(goodsList) common.PrintDebugHtml(goodsList)
for goodsSn, info := range goodsList { for goodsSn, info := range goodsList {
common.PrintDebugHtml(goodsSn+"_"+supplierIdStr)
snSku := php2go.Md5(strings.ToLower(goodsSn+"_"+supplierIdStr)) snSku := php2go.Md5(strings.ToLower(goodsSn+"_"+supplierIdStr))
skuId,_ := gredis.String(redisWrite.Do("HGET",config.Get("redis_all.SKU_UNIQUE_JUDGE").String(),snSku)) //查询唯一值，反查sku_id skuId,_ := gredis.String(redisWrite.Do("HGET",config.Get("redis_all.SKU_UNIQUE_JUDGE").String(),snSku)) //查询唯一值，反查sku_id
...@@ -78,7 +73,7 @@ func (t *CommonLyService) GetSkuByGoodsSn(goodsList map[string]*model.LyClearGoo ...@@ -78,7 +73,7 @@ func (t *CommonLyService) GetSkuByGoodsSn(goodsList map[string]*model.LyClearGoo
/**********新增sku**************/ /**********新增sku**************/
skuFlag := false; //是否新增或者更新db+redis,为true则新增 skuFlag := false; //是否新增或者更新db+redis,为true则新增
//skuId = ""; //debug skuId = ""; //debug
if skuId == "" { //redis没有此sku，创建 if skuId == "" { //redis没有此sku，创建
lock_key := "searchapi_"+snSku; lock_key := "searchapi_"+snSku;
flag,_ := redis.Bool(redisWrite.Do("SETNX", lock_key,php2go.Time()+2)) flag,_ := redis.Bool(redisWrite.Do("SETNX", lock_key,php2go.Time()+2))
...@@ -183,10 +178,9 @@ func (t *CommonLyService) writeSkuInfo(skuInfo *model.LyClearGoodsList,skuId str ...@@ -183,10 +178,9 @@ func (t *CommonLyService) writeSkuInfo(skuInfo *model.LyClearGoodsList,skuId str
defer redisWrite.Close() defer redisWrite.Close()
spuKey := common.GetKey(skuInfo.GoodsName+"_"+skuInfo.BrandName) spuKey := common.GetKey(skuInfo.GoodsName+"_"+skuInfo.BrandName)
spuId,_ := gredis.String(redisWrite.Do("HGET",config.Get("redis_all.SKU_UNIQUE_JUDGE").String(),spuKey)) spuId,_ := gredis.String(redisWrite.Do("HGET",config.Get("redis_all.SPU_UNIQUE_JUDGE").String(),spuKey))
common.PrintDebugHtml(spuKey) common.PrintDebugHtml("spu唯一值---spu_unique_judge："+spuKey + " spu_id:"+spuId)
common.PrintDebugHtml(spuId)
if spuId == "" { //不存在spu,创建新的spu if spuId == "" { //不存在spu,创建新的spu
spuId = t.writeSpuInfo(skuInfo) spuId = t.writeSpuInfo(skuInfo)
...@@ -243,9 +237,13 @@ func (t *CommonLyService) writeSpuInfo(skuInfo *model.LyClearGoodsList) string { ...@@ -243,9 +237,13 @@ func (t *CommonLyService) writeSpuInfo(skuInfo *model.LyClearGoodsList) string {
brand_key := common.GetKey(strings.ToLower(skuInfo.BrandName)) brand_key := common.GetKey(strings.ToLower(skuInfo.BrandName))
brandId,_ := redis.Int64(redisWrite.Do("HGET", config.Get("redis_all.BRAND_NAME_ALL").String(),brand_key)) brandId,_ := redis.Int64(redisWrite.Do("HGET", config.Get("redis_all.BRAND_NAME_ALL").String(),brand_key))
common.PrintDebugHtml("品牌更新："+strings.ToLower(skuInfo.BrandName)+" brand_key: "+brand_key + " 查询得到brand_id:"+common.MyInt64ToStr(brandId))
if brandId == 0 { if brandId == 0 {
brandId,_ = redis.Int64(redisWrite.Do("HGET",config.Get("redis_all.SUPPLIER_BRAND_ALL").String(),brand_key)) brandId,_ = redis.Int64(redisWrite.Do("HGET",config.Get("redis_all.SUPPLIER_BRAND_ALL").String(),brand_key))
if brandId == 0 { if brandId == 0 {
common.PrintDebugHtml("不存在品牌，新增：")
brandId = t.writeBrandInfo(skuInfo) brandId = t.writeBrandInfo(skuInfo)
} }
} }
...@@ -268,8 +266,7 @@ func (t *CommonLyService) writeSpuInfo(skuInfo *model.LyClearGoodsList) string { ...@@ -268,8 +266,7 @@ func (t *CommonLyService) writeSpuInfo(skuInfo *model.LyClearGoodsList) string {
jsonStr, _ := json.Marshal(Spuinfo) jsonStr, _ := json.Marshal(Spuinfo)
redisWrite.Do("HSET","spu",spuId,jsonStr) redisWrite.Do("HSET","spu",spuId,jsonStr)
common.PrintDebugHtml("新增spu:") common.PrintDebugHtml("新增spu:"+string(jsonStr))
common.PrintDebugHtml(jsonStr)
spuKey := common.GetKey(skuInfo.GoodsName+"_"+skuInfo.BrandName) spuKey := common.GetKey(skuInfo.GoodsName+"_"+skuInfo.BrandName)
redisWrite.Do("HSET",config.Get("redis_all.SPU_UNIQUE_JUDGE").String(),spuKey,spuId) redisWrite.Do("HSET",config.Get("redis_all.SPU_UNIQUE_JUDGE").String(),spuKey,spuId)
...@@ -289,14 +286,14 @@ func (t *CommonLyService) writeBrandInfo(skuInfo *model.LyClearGoodsList) int64 ...@@ -289,14 +286,14 @@ func (t *CommonLyService) writeBrandInfo(skuInfo *model.LyClearGoodsList) int64
defer redisWrite.Close() defer redisWrite.Close()
has, err := dbSpu.Table("brand").Where("brand_name = ?", skuInfo.BrandName).Cols("id").Get(&brandId) has, err := dbSpu.Table("lie_brand").Where("brand_name = '"+skuInfo.BrandName+"' ").Cols("brand_id").Get(&brandId)
if err != nil { if err != nil {
fmt.Println(err) //查询错误 fmt.Println(err) //查询错误
return 0 return 0
} }
if has {//不存在品牌，则创建品牌
sql :="insert into brand(brand_name,create_time,insert_type) values (?, ?, ?)" if !has {//不存在品牌，则创建品牌
sql :="insert into lie_brand(brand_name,create_time,insert_type) values (?, ?, ?)"
results, err := dbSpu.Exec(sql,skuInfo.BrandName, php2go.Time(),1) results, err := dbSpu.Exec(sql,skuInfo.BrandName, php2go.Time(),1)
if err != nil { if err != nil {
fmt.Println(err) //查询错误 fmt.Println(err) //查询错误
...@@ -304,9 +301,10 @@ func (t *CommonLyService) writeBrandInfo(skuInfo *model.LyClearGoodsList) int64 ...@@ -304,9 +301,10 @@ func (t *CommonLyService) writeBrandInfo(skuInfo *model.LyClearGoodsList) int64
} }
brandId,_ = results.LastInsertId() brandId,_ = results.LastInsertId()
common.PrintDebugHtml("创建品牌：") common.PrintDebugHtml("创建品牌："+common.MyInt64ToStr(brandId))
common.PrintDebugHtml(brandId)
} }
common.PrintDebugHtml("更新品牌到redis："+brandkey)
//更新redis 品牌数据 //更新redis 品牌数据
redisWrite.Do("HSET",config.Get("redis_all.BRAND_NAME_ALL").String(),brandkey,brandId) redisWrite.Do("HSET",config.Get("redis_all.BRAND_NAME_ALL").String(),brandkey,brandId)
redisWrite.Do("HSET",config.Get("redis_all.BRAND").String(),brandkey,brandId) redisWrite.Do("HSET",config.Get("redis_all.BRAND").String(),brandkey,brandId)
......
service/ly/mouser.go
...@@ -30,8 +30,8 @@ func OutLinkMouser(goodsName *string) map[string]*model.LyClearGoodsList { ...@@ -30,8 +30,8 @@ func OutLinkMouser(goodsName *string) map[string]*model.LyClearGoodsList {
print(err) print(err)
} }
result = resp.String(); //请求外链拿到结果 result = resp.String(); //请求外链拿到结果
common.PrintDebugHtml(result)
} }
common.PrintDebugHtml("原始数据："+result)
productList := make(map[string]*model.LyClearGoodsList,0) productList := make(map[string]*model.LyClearGoodsList,0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment