Merge branch 'ysx-search-20200724' into dev

.gitignore

 /go.sum
 /.idea/
 config.ini
+database.ini
+message.ini
+rabmq_key.ini
+redis_config.ini
+redis_key.ini
+search.ini
 gowatch.yml
 *.exe
 *.exe~

conf/database_prod.ini.demo

+[gorm]
+mode = debug
+
+[spu]
+user_name = spu
+password = spu
+host = 192.168.1.234
+database = liexin_spu
+table_prefix =lie_
+type = mysql
+test = 1
+
+#线上地址
+#SupDbUser:Supssy2@@!!@$#yxy@172.18.137.21:3306/liexin_supp
+[supp]
+user_name = liexin_ass
+password = `liexin_ass#zsyM`
+host = 192.168.2.232
+database = liexin_ass
+table_prefix =lie_
+type = mysql
+
+[cms]
+user_name = ichuntcms
+password = `ichuntcms#zsyM`
+host = 192.168.2.232
+database = ichuntcms
+table_prefix =
+type = mysql

conf/message_prod.ini.demo

+[DINGDING]
+SEARCH_API_MONITOR = 6d0fa85e01a02c39347d011ae973fd21b76c6c7ce582d3ea470c6b65a318848d
\ No newline at end of file

conf/rabmq_key_prod.ini.demo

+[rabmq]
+url = amqp://guest:guest@192.168.2.232:5672/
+;存放本系统所有的队列名称
+[rabmq_all]
+; bom任务id
+MQ_BOM_ITEMS_LIST=bom_items_list
+; bom更新数据队列
+MQ_BOM_SKU_LIST=bom_sku_list
+;联营数据更新-推送入队列-》go后台任务消费（斌哥脚本）
+SEARCH_SKU_UPDATE_LIST=search_sku_update_list
+;联营数据新增-推送入队列-》go后台任务消费（斌哥脚本）
+SEARCH_SKU_LIST=search_sku_list
+; bom任务id
+ZIYING_MQ_BOM_ITEMS_LIST=ziying_bom_items_list
+; bom更新数据队列
+ZIYING_MQ_BOM_SKU_LIST=ziying_bom_sku_list
\ No newline at end of file

conf/redis_config_prod.ini.demo

+[default_redis_read]
+host = 192.168.1.235:6379
+password = icDb29mLy2s
+max_idle = 50
+max_active = 100
+idle_timeout = 20
+
+[default_redis_write]
+host = 192.168.1.235:6379
+password = icDb29mLy2s
+max_idle = 50
+max_active = 100
+idle_timeout = 20
+
+
+[api_redis]
+host = 192.168.1.235:6379
+password = icDb29mLy2s
+max_idle = 50
+max_active = 100
+idle_timeout = 20
\ No newline at end of file

conf/redis_key_prod.ini.demo

+;存放本系统所有的redis键值对和解析
+[redis_all]
+;供应商redis的首key
+SUPPLIER_REDIS_PRE=SUPPLIER_REDIS_INFO_
+;需要展示的供应商列表
+SUPPLIER_REDIS_LIST_PRE=SUPPLIER_REDIS_LIST_INFO_
+;新增sku时判读sku是否存在
+SKU_UNIQUE_JUDGE=sku_unique_judge
+;新增sku时判读sku是否存在
+SPU_UNIQUE_JUDGE=sku_unique_judge
+;品牌信息判断,根据md5后的品牌名称获取id
+BRAND_NAME_ALL=brand_name_all
+;品牌id，获取品牌名称
+BRAND=brand
+;供应商品牌信息判断，存在多对一的情况,根据md5后的品牌名称获取标准品牌id
+SUPPLIER_BRAND_ALL=supplier_brand_all
+;统计调用外联接口的次数的key的前缀 后面接mouser、verical、arrow、master
+SEARCH_API_TOTAL_PRE=search_api_total_
+;统计调用外联接口失败的次数的key的前缀  后面接mouser、verical、arrow、master
+SEARCH_API_ERORR_PRE=search_api_overtime_
+;曝光时间记录哈希集合
+SEARCH_SHOW_SKU_TIME=search_show_sku_time
+;获取关税信息，根据goods_name,brand_name获取
+TAX_CUSTOMS_INFO=tax_customs_info
+;存放联营外链原始映射
+SKU_RAW_MAP=sku_raw_map
+
+[redis_ziyin]
+AGG_TABLE=sz_ziying_class_aggs
+FIRST_LISTS_TABLE=ziying_first_lists
+ZIYING_CLASS_KEY=ziying_class_string
+TAX_CUSTOMS_INFO=tax_customs_info
+;存放联营所有sku
+SKU=sku
+;存放联营所有spu
+SPU=SPU
+;存放digikey 型号映射
+SKU_RAW_MAP=sku_raw_map
+POOL_ALIKE_GOODS=pool_alike_goods

conf/search.prod.ini.demo

+[SUPPLIER_ALL]
+1 = future
+2 = powerandsignal
+3 = rochester
+4 = tme
+5 = verical
+6 = element14
+7 = digikey
+8 = chip1stop
+9 = aipco
+10 = arrow
+11 = bisco
+12 = alliedelec
+13 = avnet
+14 = mouser
+#16 = liexin_lianying1
+17 = zhuanmai
+18 = liexin_ti
+19 = peigenesis
+20 = powell
+21 = rs
+1676 = buerklin
+
+[SEARCH_SUPPLIER_MERGE]
+17 = zhuanmai
+1 = future
+3 = rochester
+6 = element14
+7 = digikey
+8 = chip1stop
+9 = aipco
+12 = alliedelec
+13 = avnet
+19 = peigenesis
+20 = powell
+21 = rs
+
+[SEARCH_SUPPLIER]
+1 = future
+#2 = powerandsignal
+3 = rochester
+4 = tme
+5 = verical
+6 = element14
+7 = digikey
+8 = chip1stop
+9 = aipco
+10 = arrow
+#11 = bisco
+12 = alliedelec
+13 = avnet
+14 = mouser
+#15 = company
+#16 = liexin_lianying1
+17 = zhuanmai
+#18 = liexin_ti
+19 = peigenesis
+20 = powell
+21 = rs
+#22 = liexin_sell
+
+#100 = ziying
+#101 = liexin_jingdiao
+1676 = buerklin
+
+#相似物料供应商
+[ALIKE_SUPPLIER]
+1 = ti
+
+[common]
+MOUSER_API = 'http://footstone.liexin.net/webapi/handle_mouser'
+
+
+[BRAND]
+IS_NOT_TI = 0
+TI_LY_BRAND_IDS = 23,45069,7754,17484,43215,13421,5220,7636,44534,43556,44543,9429,758,9470,10430,10653,10707,11227,11276,11819,12608,12965,12997,13327,13675,13680,13752
+TI_ZY_BRAND_IDS = 23
+
+[supplier]
+# 供应商redis的首key
+SUPPLIER_REDIS_PRE = SUPPLIER_REDIS_INFO_
+
+
+[ZIYING_BRAND_REPLACE]
+FH = FH(风华)
+BL = BL(上海贝岭)
+TE = TE(泰科)
+FM = FM(富满)
+CJ = CJ(长电)
+ST = ST(意法)
+SK = SK(时科)
+ON = ON(安森美)
+TM = TM(天微)
+TP = TP(拓微)
+TI = TI(德州仪器)
+MD = MD(明达)
+EG = EG(屹晶微)
+HF = HF(宏发)
+DY = DY(迪一)
+MK = MK(米客方德)
+国民 = Nationz(国民技术)
+中芯 = SMIC(中芯国际)
+富士 = Fuji Electric(富士电机)
+国星 = Nationstar(国星光电)
+贝岭 = BL(上海贝岭)
+
+[ZIYING_CONFIG]
+Index = liexin_ziying
+
+[SEARCH_KEYWORD_MAPPING]
+`^(.* )?([\d\.]+)(欧|欧姆|R|r)( .*)?$` = ${1}${2}Ω${4}
+`^(.* )?([\d\.]+)(U|u|μ)(.?)( .*)?$` = ${1}${2}Μ${4}${5}
+`COG` = C0G
+`NPO` = NP0
+`华科|华新科技|华新科` = 华新
+
+
+[mathExactGoodsName]
+0=M7
+1=M1
+2=M4
+3=M2
+4=A7
+5=F7
+6=X2
+7=G4
+8=X4
+9=m7
+10=m1
+11=m4
+12=m2
+13=a7
+14=f7
+15=x2
+16=g4
+17=x4
+
+
+;自营以某些单位进行搜索时，进行特殊处理
+[ZIYING_MEASURE]
+0=kΩ
+1=uf
+2=v
+3=kg
+4=MHz
+5=pf
+6=pin
+7=ppm
+8=kw
+
+
+[OFFICIAL_WEBSITE]
+1 = https://www.futureelectronics.cn/search/?text=liexin
+2 = http://www.powerandsignal.com/Products/Search?searchBox=liexin
+3 = https://www.rocelec.com/search?q=liexin
+4 = https://www.tme.eu/zh/katalog/#search=liexin
+5 = https://www.verical.com/s/liexin/
+6 = https://cn.element14.com/search?st=liexin
+7 = https://www.digikey.com/products/en?keywords=liexin
+8 = https://www.chip1stop.com/CHN/zh/view/searchResult/SearchResultTop?keyword=liexin
+9 = http://www.aipco.com/search.php?pn=liexin
+10 = https://www.arrow.com/en/products/search?cat=&q=liexin
+11 = https://www.biscoind.com?SEARCHTEXT=liexin
+12 = https://www.alliedelec.com/view/search?keyword=liexin
+13 = https://www.avnet.com/shop/apac/search/liexin
+14 = https://www.mouser.com/Search/Refine?Keyword=liexin
+15 =
+16 =
+18 =
+19 = https://www.peigenesis.cn/cn/shop/f/liexin.html
+20 = https://www.powell.com/e2wItemQuickSearch.aspx?searchText=liexin
+21 = https://uk.rs-online.com/web/c/semiconductors/amplifiers-comparators/op-amps/?searchTerm=liexin
+22 =
+1672 = https://www.masterelectronics.com/parts.aspx?text=liexin
+1673 = https://www.rutronik24.com/search-result/qs:liexin
+1675 = https://www.zaikostore.com/zaikostore/stockList?productName_forFind=liexin
+1676 = https://www.buerklin.com/en/search?text=liexin
+1677 = https://www.microchipdirect.com/product/search/all/liexin
+1678 = https://estore.heilindasia.com/search.asp?p=liexin
+1680 = http://shop.wpgam.com/WPGAStore/index.php?route=product/search&filter_description=true&filter_name=liexin
+
+
+[AUTH]
+SUPER_AUTH_KEY = fh6y5t4rr351d2c3bryi
+SEARCH_TOKEN_EXPIRE_TIME = 30
+
+[UNIT_SUPPLIER_LOG_CODE]
+333333 = arrow
+444444 = master
+555555 = mouser
+666666 = verical
+777777 = tme
+888888 = buerklin
+999999 = zhuanmai
+
+[SEARCH_API_LOG]
+SEARCH_API_ERROR_PRE = search_api_overtime_
\ No newline at end of file

controller/waf_controller.go

+package controller
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/gomodule/redigo/redis"
+	"github.com/syyongx/php2go"
+	"search_server/pkg/common"
+	"search_server/pkg/gredis"
+	"search_server/requests"
+	"strings"
+)
+
+//封禁ip开关
+func SetButton(c *gin.Context) {
+	var r requests.SetButtonRequest
+	c.ShouldBind(&r)
+	redisCon := gredis.Conn("search_w")
+	res, err := redis.Bool(redisCon.Do("SET", "spider_config_button", r.Val))
+	if err != nil {
+		common.Output(1, "操作redis失败", err.Error())
+		return
+	}
+	if res {
+		common.Output(0, "成功", res)
+		return
+	} else {
+		common.Output(1, "设置开关失败", res)
+		return
+	}
+}
+
+//获取白名单与黑名单信息
+func GetWbList(c *gin.Context) {
+	var r requests.GetWbListRequest
+	c.ShouldBind(&r)
+	redisCon := gredis.Conn("search_w")
+	if r.ListType == 0 {
+		common.Output(1, "参数为空", nil)
+		return
+	}
+	keyName := "spider_black_list"
+	if r.ListType == 1 {
+		keyName = "spider_white_list"
+	}
+
+	res, err := redis.Strings(redisCon.Do("SMEMBERS", keyName))
+	if err != nil {
+		common.Output(1, "操作redis失败", err.Error())
+		return
+	}
+	list := make(map[string]interface{})
+	list["list"] = res
+	list["total"] = len(res)
+	common.Output(0, "成功", list)
+	return
+}
+
+//设置白名单与黑名单信息
+func SetWbList(c *gin.Context) {
+	var r requests.SetWbListRequest
+	c.ShouldBind(&r)
+	if r.Type == "" || r.IpList == "" || r.AddOrRm == "" {
+		common.Output(1, "参数为空", nil)
+	}
+	redisCon := gredis.Conn("search_w")
+	keyName := "spider_black_list"
+	if r.Type == "1" {
+		keyName = "spider_white_list"
+	}
+	ipListSlice := strings.Split(r.IpList, ",")
+	if len(ipListSlice) > 0 {
+		for _, ip := range ipListSlice {
+			if r.AddOrRm == "1" {
+				redis.Bool(redisCon.Do("SADD", keyName, strings.TrimSpace(ip)))
+			} else {
+				redis.Bool(redisCon.Do("SREM", keyName, strings.TrimSpace(ip)))
+			}
+		}
+
+	}
+	common.Output(0, "成功!", nil)
+}
+
+//获取自我解禁ip访问频率自增数据
+func GetIpalg() {
+	redisCon := gredis.Conn("search_r")
+	spiderAlgs, _ := redis.StringMap(redisCon.Do("HGETALL", "spider_connect_count_alg"))
+	spiderBans, _ := redis.Strings(redisCon.Do("SMEMBERS", "spider_ban"))
+	list := make(map[string]interface{})
+	if len(spiderAlgs) > 0 {
+		var dataList []map[string]string
+		for key, alg := range spiderAlgs {
+			data := make(map[string]string)
+			data["ip"] = key
+			data["num"] = alg
+			if !php2go.InArray(key, spiderBans) {
+				dataList = append(dataList, data)
+			}
+		}
+		list["list"] = dataList
+		list["total"] = len(dataList)
+	}
+	common.Output(0, "成功", list)
+	return
+}
+
+//设置自我解禁ip访问频率自增数据
+func SetIpalg(c *gin.Context) {
+	var r requests.SetIpalgRequest
+	c.ShouldBind(&r)
+	if r.Ip == "" || r.ListType == 0 {
+		common.Output(1, "参数为空", nil)
+		return
+	}
+
+	redisCon := gredis.Conn("search_w")
+	ip := strings.TrimSpace(r.Ip)
+	if r.ListType == 1 {
+		_, err := redis.Bool(redisCon.Do("HSET", "spider_connect_count_alg", ip, r.Num))
+		if err != nil {
+			common.Output(1, "操作redis失败", err.Error())
+		}
+	} else {
+		_, err := redis.Bool(redisCon.Do("HDEL", "spider_connect_count_alg", ip))
+		if err != nil {
+			common.Output(1, "操作redis失败", err.Error())
+		}
+	}
+
+	common.Output(0, "成功", nil)
+}
+
+//设置参数配置信息
+func SetConfig(c *gin.Context) {
+	request := common.GetAllRequestParams(c)
+	if request["server_id"] == "" {
+		//common.
+	}
+}

pkg/common/request.go

 package common
 
+import (
+	"github.com/gin-gonic/gin"
+	"strings"
+)
+
 type RecommendRequest struct {
 	GoodsName    string `form:"goods_name"`
 	Attrs        string `form:"attrs"`
@@ -9,3 +14,15 @@ type RecommendRequest struct {
 	Flag         int    `form:"flag"`
 	BrandName    string `form:"brand_name"`
 }
+
+//获取所有请求参数放到字典里面
+func GetAllRequestParams(c *gin.Context) (request map[string]string){
+	request = make(map[string]string)
+	c.MultipartForm()
+	for name, value := range c.Request.Form {
+		if value[0] != "" {
+			request[name] = strings.TrimSpace(value[0])
+		}
+	}
+	return
+}

requests/waf_request.go

+package requests
+
+type SetButtonRequest struct {
+	Val int `form:"val"`
+}
+
+type GetWbListRequest struct {
+	ListType int `form:"type"`
+}
+
+type SetWbListRequest struct {
+	Type    string `form:"type"`
+	IpList  string `form:"ip_list"`
+	AddOrRm string `form:"add_or_rm"`
+}
+
+type SetIpalgRequest struct {
+	Ip string `form:"ip"`
+	ListType int `form:"type"`
+	Num int `json:"num"`
+}
\ No newline at end of file

routes/router.go

@@ -15,8 +15,6 @@ func InitRouter() *gin.Engine {
 	r.Use(middleware.Cors())
 	r.Use(middleware.ContextVars())
 	//路由
-	r.POST("/search/bom/autospu", controller.AutoSpu)
-	r.POST("/search/bom/recommend", controller.Recommend)
 	r.POST("search/ZiYing/zyh", controller.Zyh)
 	r.GET("search/ZiYing/zyh", controller.Zyh)
 	r.GET("search/ZiYing/index", controller.Index)
@@ -24,6 +22,10 @@ func InitRouter() *gin.Engine {
 	r.POST("search/mouser/a", controller.GetMouserData)
 	r.GET("search/mouser/a", controller.GetMouserData)
 
+	//bom相关
+	r.POST("/search/bom/autospu", controller.AutoSpu)
+	r.POST("/search/bom/recommend", controller.Recommend)
+
 	//快手平台相关
 	r.GET("/search/quote", controller.QuoteIndex)
 	r.POST("/search/quote", controller.QuoteIndex)

search_server.bash

 #!/bin/bash
 cd /data2/gocode/search_server
-git pull origin master
+git pull origin dev
+cp /data2/gocode/search_server/config_prod.ini.demo /data2/gocode/search_server/config.ini
+cp /data2/gocode/search_server/database_prod.ini.demo /data2/gocode/search_server/database.ini
+cp /data2/gocode/search_server/message_prod.ini.demo /data2/gocode/search_server/message.ini
+cp /data2/gocode/search_server/rabmq_key_prod.ini.demo /data2/gocode/search_server/rabmq_key.ini
+cp /data2/gocode/search_server/redis_config_prod.ini.demo /data2/gocode/search_server/redis_config.ini
+cp /data2/gocode/search_server/redis_key_prod.ini.demo /data2/gocode/search_server/redis_key.ini
+cp /data2/gocode/search_server/search_prod.ini.demo /data2/gocode/search_server/search.ini
 go build -o ./cmd/search_http_server ./cmd/search_http_server.go
 supervisorctl restart search_server