Skip to content

杨树贤 / search_server

Go to a project
Commit 97e60f9f by huangchengyi
Options

Merge branch 'dev' of http://119.23.72.7/mushishixian/search_server into dev 

# Conflicts:
#	conf/config_prod.ini.demo
#	conf/rabmq_key_prod.ini.demo
#	conf/redis_config_prod.ini.demo
#	conf/redis_key_prod.ini.demo
#	conf/search.prod.ini.demo
parents 6dd5e4e8 129fe3d2
Inline Side-by-side
Showing with 539 additions and 5 deletions
.gitignore
/go.sum /go.sum
/.idea/ /.idea/
config.ini config.ini
database.ini
message.ini
rabmq_key.ini
redis_config.ini
redis_key.ini
search.ini
gowatch.yml gowatch.yml
*.exe *.exe
*.exe~ *.exe~
......
conf/config_prod.ini.demo 0 → 100644
[web]
port = 10001
mode = release
cors_domain = https://bom.ichunt.com
[message]
api_domain = http://api.ichunt.com/msg/sendMessageByAuto
api_md5_str = fh6y5t4rr351d2c3bryi
[goods]
api_url = http://47.106.60.211:60004
[es]
url = http://172.18.137.29:9211
urls = http://172.18.137.29:9211,http://172.18.137.30:9211
index_name = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,zhuanmai,peigenesis,powell,rs,buerklin,liexin_ziying
search_supplier = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,peigenesis,powell,rs,buerklin,zhuanmai
hk_delivery_type_supplier = future,rochester,tme,verical,element14,digikey,chip1stop,aipco,arrow,alliedelec,avnet,mouser,peigenesis,powell,rs,buerklin
attr_index = goods_map
[redis]
write_host = 172.18.137.38:6379
read_host = 172.18.137.39:6379
write_password = icDb29mLy2s
read_password = icDb29mLy2s
max_idle = 10
max_active = 10
idle_timeout = 20
conf/database_prod.ini.demo 0 → 100644
[gorm]
mode = debug
#线上地址
#mysql://LxiCSpu:Mysx3Tyzlo00oxlmlly@172.18.137.29:3306
[spu]
user_name = LxiCSpu
password = Mysx3Tyzlo00oxlmlly
host = 172.18.137.29
database = liexin_spu
table_prefix =lie_
type = mysql
#线上地址
#SupDbUser:Supssy2@@!!@$#yxy@172.18.137.21:3306/liexin_supp
[supp]
user_name = SupDbUser
password = `Supssy2@@!!@$#yxy`
host = 172.18.137.21
database = liexin_supp
table_prefix =lie_
type = mysql
#线上地址
#mysql://dtuserread:dAtaL#ym2902m2lLX2y33@172.18.137.22:3306/icdata#utf8
[cms]
user_name = dtuserread
password = `dAtaL#ym2902m2lLX2y33`
host = 172.18.137.22
database = icdata
table_prefix =
type = mysql
conf/message_prod.ini.demo 0 → 100644
[DINGDING]
SEARCH_API_MONITOR = cd1661d0ce733a8ff0b603b532b317aae9de6dbaf39186b0352f272e47a860e8
\ No newline at end of file
conf/rabmq_key_prod.ini.demo 0 → 100644
[rabmq]
url = amqp://huntweb:xMtMi29zTm@119.23.79.136:5672/
;存放本系统所有的队列名称
[rabmq_all]
; bom任务id
MQ_BOM_ITEMS_LIST=bom_items_list
; bom更新数据队列
MQ_BOM_SKU_LIST=bom_sku_list
;联营数据更新-推送入队列-》go后台任务消费（斌哥脚本）
SEARCH_SKU_UPDATE_LIST=search_sku_update_list
;联营数据新增-推送入队列-》go后台任务消费（斌哥脚本）
SEARCH_SKU_LIST=search_sku_list
; bom任务id
ZIYING_MQ_BOM_ITEMS_LIST=ziying_bom_items_list
; bom更新数据队列
ZIYING_MQ_BOM_SKU_LIST=ziying_bom_sku_list
\ No newline at end of file
conf/redis_config_prod.ini.demo 0 → 100644
[default_redis_read]
host = 172.18.137.39:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
[default_redis_write]
host = 172.18.137.38:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
[api_redis_read]
host = 172.18.137.38:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
[api_redis_write]
host = 172.18.137.39:6379
password = icDb29mLy2s
max_idle = 50
max_active = 100
idle_timeout = 20
conf/redis_key_prod.ini.demo 0 → 100644
;存放本系统所有的redis键值对和解析
[redis_all]
;供应商redis的首key
SUPPLIER_REDIS_PRE=SUPPLIER_REDIS_INFO_
;需要展示的供应商列表
SUPPLIER_REDIS_LIST_PRE=SUPPLIER_REDIS_LIST_INFO_
;新增sku时判读sku是否存在
SKU_UNIQUE_JUDGE=sku_unique_judge
;新增sku时判读sku是否存在
SPU_UNIQUE_JUDGE=sku_unique_judge
;品牌信息判断,根据md5后的品牌名称获取id
BRAND_NAME_ALL=brand_name_all
;品牌id，获取品牌名称
BRAND=brand
;供应商品牌信息判断，存在多对一的情况,根据md5后的品牌名称获取标准品牌id
SUPPLIER_BRAND_ALL=supplier_brand_all
;统计调用外联接口的次数的key的前缀 后面接mouser、verical、arrow、master
SEARCH_API_TOTAL_PRE=search_api_total_
;统计调用外联接口失败的次数的key的前缀 后面接mouser、verical、arrow、master
SEARCH_API_ERORR_PRE=search_api_overtime_
;曝光时间记录哈希集合
SEARCH_SHOW_SKU_TIME=search_show_sku_time
;获取关税信息，根据goods_name,brand_name获取
TAX_CUSTOMS_INFO=tax_customs_info
;存放联营外链原始映射
SKU_RAW_MAP=sku_raw_map
[redis_ziyin]
AGG_TABLE=sz_ziying_class_aggs
FIRST_LISTS_TABLE=ziying_first_lists
ZIYING_CLASS_KEY=ziying_class_string
TAX_CUSTOMS_INFO=tax_customs_info
;存放联营所有sku
SKU=sku
;存放联营所有spu
SPU=SPU
;存放digikey 型号映射
SKU_RAW_MAP=sku_raw_map
POOL_ALIKE_GOODS=pool_alike_goods
conf/search.prod.ini.demo 0 → 100644
[SUPPLIER_ALL]
1 = future
2 = powerandsignal
3 = rochester
4 = tme
5 = verical
6 = element14
7 = digikey
8 = chip1stop
9 = aipco
10 = arrow
11 = bisco
12 = alliedelec
13 = avnet
14 = mouser
15 = company
16 = liexin_lianying1
17 = zhuanmai
18 = liexin_ti
19 = peigenesis
20 = powell
21 = rs
22 = liexin_sell
100 = ziying
101 = liexin_jingdiao
1672 = master
1673 = rutronik
1675 = corestaff
1676 = buerklin
1677 = microchip
1678 = heilind
1680 = wpg
1681 = jameco
1679 = ti
1684 = distrelec
1685 = tlc
1686 = adi
#搜索结果页合并供应商
[SEARCH_SUPPLIER_MERGE]
1 = future
3 = rochester
6 = element14
#7 = digikey
8 = chip1stop
9 = aipco
12 = alliedelec
13 = avnet
19 = peigenesis
20 = powell
21 = rs
1675 = corestaff
1677 = microchip
1678 = heilind
1680 = wpg
1681 = jameco
1679 = ti
1684 = distrelec
1685 = tlc
1686 = adi
#相似物料供应商
[ALIKE_SUPPLIER]
1 = ti
[common]
MOUSER_API = 'http://footstone.liexin.net/webapi/handle_mouser'
[BRAND]
IS_NOT_TI = 0
TI_LY_BRAND_IDS = 23,45069,7754,17484,43215,13421,5220,7636,44534,43556,44543,9429,758,9470,10430,10653,10707,11227,11276,11819,12608,12965,12997,13327,13675,13680,13752
TI_ZY_BRAND_IDS = 23
[supplier]
# 供应商redis的首key
SUPPLIER_REDIS_PRE = SUPPLIER_REDIS_INFO_
[ZIYING_BRAND_REPLACE]
FH = FH(风华)
BL = BL(上海贝岭)
TE = TE(泰科)
FM = FM(富满)
CJ = CJ(长电)
ST = ST(意法)
SK = SK(时科)
ON = ON(安森美)
TM = TM(天微)
TP = TP(拓微)
TI = TI(德州仪器)
MD = MD(明达)
EG = EG(屹晶微)
HF = HF(宏发)
DY = DY(迪一)
MK = MK(米客方德)
国民 = Nationz(国民技术)
中芯 = SMIC(中芯国际)
富士 = Fuji Electric(富士电机)
国星 = Nationstar(国星光电)
贝岭 = BL(上海贝岭)
[ZIYING_CONFIG]
Index = liexin_ziying
[SEARCH_KEYWORD_MAPPING]
`^(.* )?([\d\.]+)(欧|欧姆|R|r)( .*)?$` = ${1}${2}Ω${4}
`^(.* )?([\d\.]+)(U|u|μ)(.?)( .*)?$` = ${1}${2}Μ${4}${5}
`COG` = C0G
`NPO` = NP0
`华科|华新科技|华新科` = 华新
[mathExactGoodsName]
0=M7
1=M1
2=M4
3=M2
4=A7
5=F7
6=X2
7=G4
8=X4
9=m7
10=m1
11=m4
12=m2
13=a7
14=f7
15=x2
16=g4
17=x4
;自营以某些单位进行搜索时，进行特殊处理
[ZIYING_MEASURE]
0=kΩ
1=uf
2=v
3=kg
4=MHz
5=pf
6=pin
7=ppm
8=kw
[OFFICIAL_WEBSITE]
1 = https://www.futureelectronics.cn/search/?text=liexin
2 = http://www.powerandsignal.com/Products/Search?searchBox=liexin
3 = https://www.rocelec.com/search?q=liexin
4 = https://www.tme.eu/zh/katalog/#search=liexin
5 = https://www.verical.com/s/liexin/
6 = https://cn.element14.com/search?st=liexin
7 = https://www.digikey.com/products/en?keywords=liexin
8 = https://www.chip1stop.com/CHN/zh/view/searchResult/SearchResultTop?keyword=liexin
9 = http://www.aipco.com/search.php?pn=liexin
10 = https://www.arrow.com/en/products/search?cat=&q=liexin
11 = https://www.biscoind.com?SEARCHTEXT=liexin
12 = https://www.alliedelec.com/view/search?keyword=liexin
13 = https://www.avnet.com/shop/apac/search/liexin
14 = https://www.mouser.com/Search/Refine?Keyword=liexin
15 =
16 =
18 =
19 = https://www.peigenesis.cn/cn/shop/f/liexin.html
20 = https://www.powell.com/e2wItemQuickSearch.aspx?searchText=liexin
21 = https://uk.rs-online.com/web/c/semiconductors/amplifiers-comparators/op-amps/?searchTerm=liexin
22 =
1672 = https://www.masterelectronics.com/parts.aspx?text=liexin
1673 = https://www.rutronik24.com/search-result/qs:liexin
1675 = https://www.zaikostore.com/zaikostore/stockList?productName_forFind=liexin
1676 = https://www.buerklin.com/en/search?text=liexin
1677 = https://www.microchipdirect.com/product/search/all/liexin
1678 = https://estore.heilindasia.com/search.asp?p=liexin
1680 = http://shop.wpgam.com/WPGAStore/index.php?route=product/search&filter_description=true&filter_name=liexin
[AUTH]
SUPER_AUTH_KEY = fh6y5t4rr351d2c3bryi
SEARCH_TOKEN_EXPIRE_TIME = 30
[UNIT_SUPPLIER_LOG_CODE]
333333 = arrow
444444 = master
555555 = mouser
666666 = verical
777777 = tme
888888 = buerklin
999999 = zhuanmai
[SEARCH_API_LOG]
SEARCH_API_ERROR_PRE = search_api_overtime_
\ No newline at end of file
controller/waf_controller.go 0 → 100644
package controller
import (
"github.com/gin-gonic/gin"
"github.com/gomodule/redigo/redis"
"github.com/syyongx/php2go"
"search_server/pkg/common"
"search_server/pkg/gredis"
"search_server/requests"
"strings"
)
//封禁ip开关
func SetButton(c *gin.Context) {
var r requests.SetButtonRequest
c.ShouldBind(&r)
redisCon := gredis.Conn("search_w")
res, err := redis.Bool(redisCon.Do("SET", "spider_config_button", r.Val))
if err != nil {
common.Output(1, "操作redis失败", err.Error())
return
}
if res {
common.Output(0, "成功", res)
return
} else {
common.Output(1, "设置开关失败", res)
return
}
}
//获取白名单与黑名单信息
func GetWbList(c *gin.Context) {
var r requests.GetWbListRequest
c.ShouldBind(&r)
redisCon := gredis.Conn("search_w")
if r.ListType == 0 {
common.Output(1, "参数为空", nil)
return
}
keyName := "spider_black_list"
if r.ListType == 1 {
keyName = "spider_white_list"
}
res, err := redis.Strings(redisCon.Do("SMEMBERS", keyName))
if err != nil {
common.Output(1, "操作redis失败", err.Error())
return
}
list := make(map[string]interface{})
list["list"] = res
list["total"] = len(res)
common.Output(0, "成功", list)
return
}
//设置白名单与黑名单信息
func SetWbList(c *gin.Context) {
var r requests.SetWbListRequest
c.ShouldBind(&r)
if r.Type == "" || r.IpList == "" || r.AddOrRm == "" {
common.Output(1, "参数为空", nil)
}
redisCon := gredis.Conn("search_w")
keyName := "spider_black_list"
if r.Type == "1" {
keyName = "spider_white_list"
}
ipListSlice := strings.Split(r.IpList, ",")
if len(ipListSlice) > 0 {
for _, ip := range ipListSlice {
if r.AddOrRm == "1" {
redis.Bool(redisCon.Do("SADD", keyName, strings.TrimSpace(ip)))
} else {
redis.Bool(redisCon.Do("SREM", keyName, strings.TrimSpace(ip)))
}
}
}
common.Output(0, "成功!", nil)
}
//获取自我解禁ip访问频率自增数据
func GetIpalg() {
redisCon := gredis.Conn("search_r")
spiderAlgs, _ := redis.StringMap(redisCon.Do("HGETALL", "spider_connect_count_alg"))
spiderBans, _ := redis.Strings(redisCon.Do("SMEMBERS", "spider_ban"))
list := make(map[string]interface{})
if len(spiderAlgs) > 0 {
var dataList []map[string]string
for key, alg := range spiderAlgs {
data := make(map[string]string)
data["ip"] = key
data["num"] = alg
if !php2go.InArray(key, spiderBans) {
dataList = append(dataList, data)
}
}
list["list"] = dataList
list["total"] = len(dataList)
}
common.Output(0, "成功", list)
return
}
//设置自我解禁ip访问频率自增数据
func SetIpalg(c *gin.Context) {
var r requests.SetIpalgRequest
c.ShouldBind(&r)
if r.Ip == "" || r.ListType == 0 {
common.Output(1, "参数为空", nil)
return
}
redisCon := gredis.Conn("search_w")
ip := strings.TrimSpace(r.Ip)
if r.ListType == 1 {
_, err := redis.Bool(redisCon.Do("HSET", "spider_connect_count_alg", ip, r.Num))
if err != nil {
common.Output(1, "操作redis失败", err.Error())
}
} else {
_, err := redis.Bool(redisCon.Do("HDEL", "spider_connect_count_alg", ip))
if err != nil {
common.Output(1, "操作redis失败", err.Error())
}
}
common.Output(0, "成功", nil)
}
//设置参数配置信息
func SetConfig(c *gin.Context) {
request := common.GetAllRequestParams(c)
if request["server_id"] == "" {
//common.
}
}
pkg/common/request.go
package common package common
import (
"github.com/gin-gonic/gin"
"strings"
)
type RecommendRequest struct { type RecommendRequest struct {
GoodsName string `form:"goods_name"` GoodsName string `form:"goods_name"`
Attrs string `form:"attrs"` Attrs string `form:"attrs"`
...@@ -9,3 +14,15 @@ type RecommendRequest struct { ...@@ -9,3 +14,15 @@ type RecommendRequest struct {
Flag int `form:"flag"` Flag int `form:"flag"`
BrandName string `form:"brand_name"` BrandName string `form:"brand_name"`
} }
//获取所有请求参数放到字典里面
func GetAllRequestParams(c *gin.Context) (request map[string]string){
request = make(map[string]string)
c.MultipartForm()
for name, value := range c.Request.Form {
if value[0] != "" {
request[name] = strings.TrimSpace(value[0])
}
}
return
}
requests/waf_request.go 0 → 100644
package requests
type SetButtonRequest struct {
Val int `form:"val"`
}
type GetWbListRequest struct {
ListType int `form:"type"`
}
type SetWbListRequest struct {
Type string `form:"type"`
IpList string `form:"ip_list"`
AddOrRm string `form:"add_or_rm"`
}
type SetIpalgRequest struct {
Ip string `form:"ip"`
ListType int `form:"type"`
Num int `json:"num"`
}
\ No newline at end of file
routes/router.go
...@@ -15,8 +15,6 @@ func InitRouter() *gin.Engine { ...@@ -15,8 +15,6 @@ func InitRouter() *gin.Engine {
r.Use(middleware.Cors()) r.Use(middleware.Cors())
r.Use(middleware.ContextVars()) r.Use(middleware.ContextVars())
//路由 //路由
r.POST("/search/bom/autospu", controller.AutoSpu)
r.POST("/search/bom/recommend", controller.Recommend)
r.POST("search/ZiYing/zyh", controller.Zyh) r.POST("search/ZiYing/zyh", controller.Zyh)
r.GET("search/ZiYing/zyh", controller.Zyh) r.GET("search/ZiYing/zyh", controller.Zyh)
r.GET("search/ZiYing/index", controller.Index) r.GET("search/ZiYing/index", controller.Index)
...@@ -24,6 +22,10 @@ func InitRouter() *gin.Engine { ...@@ -24,6 +22,10 @@ func InitRouter() *gin.Engine {
r.POST("search/mouser/a", controller.GetMouserData) r.POST("search/mouser/a", controller.GetMouserData)
r.GET("search/mouser/a", controller.GetMouserData) r.GET("search/mouser/a", controller.GetMouserData)
//bom相关
r.POST("/search/bom/autospu", controller.AutoSpu)
r.POST("/search/bom/recommend", controller.Recommend)
//快手平台相关 //快手平台相关
r.GET("/search/quote", controller.QuoteIndex) r.GET("/search/quote", controller.QuoteIndex)
r.POST("/search/quote", controller.QuoteIndex) r.POST("/search/quote", controller.QuoteIndex)
......
search_server.bash
#!/bin/bash #!/bin/bash
cd /data2/gocode/search_server cd /data2/gocode/search_server
git pull origin master git pull origin dev
cp /data2/gocode/search_server/config_prod.ini.demo /data2/gocode/search_server/config.ini
cp /data2/gocode/search_server/database_prod.ini.demo /data2/gocode/search_server/database.ini
cp /data2/gocode/search_server/message_prod.ini.demo /data2/gocode/search_server/message.ini
cp /data2/gocode/search_server/rabmq_key_prod.ini.demo /data2/gocode/search_server/rabmq_key.ini
cp /data2/gocode/search_server/redis_config_prod.ini.demo /data2/gocode/search_server/redis_config.ini
cp /data2/gocode/search_server/redis_key_prod.ini.demo /data2/gocode/search_server/redis_key.ini
cp /data2/gocode/search_server/search_prod.ini.demo /data2/gocode/search_server/search.ini
go build -o ./cmd/search_http_server ./cmd/search_http_server.go go build -o ./cmd/search_http_server ./cmd/search_http_server.go
supervisorctl restart search_server supervisorctl restart search_server
service/transformer/goods_transformer.go
...@@ -310,8 +310,7 @@ func ShowSku(goods model.DullGoodsData) { ...@@ -310,8 +310,7 @@ func ShowSku(goods model.DullGoodsData) {
return return
} }
referer := middleware.Context.Request.Header.Get("referer") referer := middleware.Context.Request.Header.Get("referer")
//if strings.Contains(referer, "www.ichunt.com/s/?k") { if strings.Contains(referer, "www.ichunt.com/s/?k") {
if strings.Contains(referer, "") {
if goods.ProviderName != "" { if goods.ProviderName != "" {
goods.BrandName = goods.ProviderName goods.BrandName = goods.ProviderName
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment