Skip to content

杨树贤 / search_server

Go to a project
Commit 657403bc by huangchengyi
Options

Merge branch 'dev' of http://119.23.72.7/mushishixian/search_server into dev

parents a71b2f30 b17148bf
Inline Side-by-side
Showing with 330 additions and 74 deletions
conf/local/config.ini.local
......@@ -11,6 +11,7 @@ api_md5_str = fh6y5t4rr351d2c3bryi
[goods]
api_url = http://192.168.2.232:60004
sz_api_url = http://192.168.2.232:60004
[es]
url = http://192.168.2.232:9200
......@@ -46,9 +47,9 @@ is_sz = true
#自营mongodb缓存表 正式线ziying_class_aggs
[AGG_TABLE]
agg_table = sz_ziying_class_aggs
agg_table = ziying_class_aggs
#自营mongodb缓存表 正式线 ziying_first_lists
[FIRSTLISTS_TABLE]
firstLists_table = sz_ziying_first_lists
\ No newline at end of file
firstLists_table = ziying_first_lists
\ No newline at end of file
conf/prod/config.ini.prod
......@@ -10,6 +10,7 @@ api_md5_str = fh6y5t4rr351d2c3bryi
[goods]
api_url = http://47.106.60.211:60004
sz_api_url = http://47.106.60.211:60004
[es]
url = http://172.18.137.29:9211
......
conf/prod/database.ini.prod
......@@ -24,9 +24,9 @@ type = mysql
#线上地址
#mysql://dtuserread:dAtaL#ym2902m2lLX2y33@172.18.137.22:3306/icdata#utf8
[cms]
user_name = dtuserread
user_name = dtuser
password = `dAtaL#ym2902m2lLX2y33`
host = 172.18.137.22
host = appdb-master.ichunt.db
database = icdata
table_prefix =
type = mysql
controller/quote_controller.go
......@@ -150,3 +150,7 @@ func GetData(keyword, supplierName string) (res interface{}) {
res, _ = quoteService.GetData(keyword, supplierName)
return
}
func QuoteTest(c *gin.Context) {
c.String(200, "测试")
}
controller/ziyin_controller.go
......@@ -29,11 +29,56 @@ func Zyh(ctx *gin.Context){
if err != nil{
zyhResponse.ErrorCode = 1
zyhResponse.ErrorMsg = fmt.Sprintf("%s",err)
ctx.JSON(200,zyhResponse)
common.Output(int(zyhResponse.ErrorCode), zyhResponse.ErrorMsg, zyhResponse.Data)
}
//fmt.Printf("%+v",zyhResponse)
ctx.JSON(200,zyhResponse)
common.Output(int(zyhResponse.ErrorCode), zyhResponse.ErrorMsg, zyhResponse.Data)
}
func Aggs(ctx *gin.Context){
zyhRequest := make(map[string]string,0);
ctx.MultipartForm()
for requestName, requstValue := range ctx.Request.Form {
if requstValue[0] != "" {
zyhRequest[requestName] = strings.TrimSpace(requstValue[0])
}
}
//fmt.Println(zyhRequest)
zyServiceImpl := service.NewZyServiceImpl()
zyhResponse := &model.ZyhResponse{}
zyhResponse,err := zyServiceImpl.Aggs(ctx,zyhRequest)
if err != nil{
zyhResponse.ErrorCode = 1
zyhResponse.ErrorMsg = fmt.Sprintf("%s",err)
common.Output(int(zyhResponse.ErrorCode), zyhResponse.ErrorMsg, zyhResponse.Data)
}
common.Output(int(zyhResponse.ErrorCode), zyhResponse.ErrorMsg, zyhResponse.Data)
}
//统计样片一级分类总条数
func AggSample(ctx *gin.Context){
zyhRequest := make(map[string]string,0);
ctx.MultipartForm()
for requestName, requstValue := range ctx.Request.Form {
if requstValue[0] != "" {
zyhRequest[requestName] = strings.TrimSpace(requstValue[0])
}
}
//fmt.Println(zyhRequest)
zyServiceImpl := service.NewZyServiceImpl()
zyhResponse := &model.ZyhResponse{}
zyhResponse = zyServiceImpl.AggSample(ctx,zyhRequest)
common.Output(int(zyhResponse.ErrorCode), zyhResponse.ErrorMsg, zyhResponse.Data)
}
func Index(ctx *gin.Context){
......
model/ziyin.go
package model
//type ZyhRequest struct {
// Yo4tewUid string `form:"Yo4teW_uid"`
// ClassIdcondition int64 `form:"class_id/condition"`
// ClassId2condition int64 `form:"class_id2/condition"`
// Brand_idCondition string `form:"brand_id/condition"`
// StockRank string `form:"stock_rank"` //库存排序
// AvailRank string `form:"avail_rank"` //显示有货的
// ComRank string `form:"com_rank"` //综合排序
// SingleRank string `form:"single_rank"` //价格排序
// PackingCondition string `form:"packing/condition"`
// EncapCondition string `form:"encap/condition"`
// P int `form:"p"`
// NeedAggs string `form:"need_aggs"`
// GoodsNameCondition string `form:"goods_name/condition"`
//
//}
import (
"search_server/pkg/mysql"
)
type ZhyClassListModel struct {
ClassID1Name string `json:"class_id1_name"`
......@@ -47,3 +29,16 @@ type ZyhResponse struct {
}
type LieXinData struct {
Id int `json:"id" xorm:"id"`
Sort int `json:"sort" xorm:"sort"`
}
func GetSampleClass () ( sample_class_data []LieXinData) {
sample_class_data = make([]LieXinData,0)
spuMsql := mysql.Conn("liexin_data")
sql := "select id,sort from lie_sample_class where status = 1 order by sort asc "
_ = spuMsql.SQL(sql).Find(&sample_class_data)
return
}
\ No newline at end of file
pkg/config/mysql.go
......@@ -34,5 +34,12 @@ func BuildDatabaseList() (DatabaseList map[string]BaseDatabase) {
Database: Get("cms.database").String(),
Prefix: Get("cms.table_prefix").String(),
},
"liexin_data": {
UserName: Get("liexin_data.user_name").String(),
Password: Get("liexin_data.password").String(),
Host: Get("liexin_data.host").String(),
Database: Get("liexin_data.database").String(),
Prefix: Get("liexin_data.table_prefix").String(),
},
}
}
routes/router.go
......@@ -19,6 +19,10 @@ func InitRouter() *gin.Engine {
r.GET("search/ZiYing/zyh", controller.Zyh)
r.GET("search/ZiYing/index", controller.Index)
r.POST("search/ZiYing/index", controller.Index)
r.GET("search/ZiYing/aggs", controller.Aggs)
r.POST("search/ZiYing/aggs", controller.Aggs)
r.GET("search/ZiYing/aggSample", controller.AggSample)
r.POST("search/ZiYing/aggSample", controller.AggSample)
r.POST("search/mouser/a", controller.GetMouserData)
r.GET("search/mouser/a", controller.GetMouserData)
r.POST("search/digikey/a", controller.GetDigikeyData)
......@@ -29,6 +33,7 @@ func InitRouter() *gin.Engine {
r.POST("/search/bom/recommend", controller.Recommend)
//快手平台相关
r.GET("/search/quote/test", controller.QuoteTest)
r.GET("/search/quote", controller.QuoteIndex)
r.POST("/search/quote", controller.QuoteIndex)
r.GET("/search/quote/getDataPur", controller.GetDataPur)
......
search_server.bash
#!/bin/bash
cd /data2/gocode/search_server
cd /data/gocode/search_server
git pull origin dev
cp /data2/gocode/search_server/config_prod.ini.demo /data2/gocode/search_server/config.ini
cp /data2/gocode/search_server/database_prod.ini.demo /data2/gocode/search_server/database.ini
cp /data2/gocode/search_server/message_prod.ini.demo /data2/gocode/search_server/message.ini
cp /data2/gocode/search_server/rabmq_key_prod.ini.demo /data2/gocode/search_server/rabmq_key.ini
cp /data2/gocode/search_server/redis_config_prod.ini.demo /data2/gocode/search_server/redis_config.ini
cp /data2/gocode/search_server/redis_key_prod.ini.demo /data2/gocode/search_server/redis_key.ini
cp /data2/gocode/search_server/search_prod.ini.demo /data2/gocode/search_server/search.ini
cp /data/gocode/search_server/conf/prod/config.ini.prod /data/gocode/search_server/conf/config.ini
cp /data/gocode/search_server/conf/prod/database.ini.prod /data/gocode/search_server/conf/database.ini
cp /data/gocode/search_server/conf/prod/message.ini.prod /data/gocode/search_server/conf/message.ini
cp /data/gocode/search_server/conf/prod/rabmq_key.ini.prod /data/gocode/search_server/conf/rabmq_key.ini
cp /data/gocode/search_server/conf/prod/redis_config.ini.prod /data/gocode/search_server/conf/redis_config.ini
cp /data/gocode/search_server/conf/prod/redis_key.ini.prod /data/gocode/search_server/conf/redis_key.ini
cp /data/gocode/search_server/conf/prod/search.ini.prod /data/gocode/search_server/conf/search.ini
go build -o ./cmd/search_http_server ./cmd/search_http_server.go
supervisorctl restart search_server
service/quote_service.go
......@@ -2,6 +2,7 @@ package service
import (
"encoding/json"
"fmt"
"github.com/imroc/req"
"github.com/syyongx/php2go"
"github.com/tidwall/gjson"
......@@ -50,7 +51,9 @@ func (qs *QuoteService) GetGoodsBySupplierName(r requests.GetGoodsBySupplierRequ
if err != nil {
return
}
fmt.Println(esResult)
goodsIds := GetGoodsIdsByEs(esResult)
fmt.Println(goodsIds)
goodsList, err := GetGoodsInfo(goodsIds)
dullData, err := transformer.DullDataInfo(r.Keyword, goodsList)
quoteTransformer := transformer.QuoteTransformer{}
......
service/ziyin_service.go
......@@ -40,7 +40,7 @@ func (this *ZyServiceImpl) Zyh(ctx *gin.Context, req map[string]string) (rsp *mo
need_aggs bool
_type byte
)
isAdmin = false
isAdmin = true
if req["need_aggs"] == "1"{
need_aggs = true
}else{
......@@ -53,12 +53,29 @@ func (this *ZyServiceImpl) Zyh(ctx *gin.Context, req map[string]string) (rsp *mo
}else{
_type = 3
}
rsp,ok := ziyingCache(req,_type)
if ok {
return rsp,nil
}
//设置cookie
setCookie(ctx,req)
rsp = zy(req ,_type,0)
return rsp,nil
}
func (this *ZyServiceImpl)Aggs(ctx *gin.Context, req map[string]string) (rsp *model.ZyhResponse,err error){
rsp,ok := ziyingCache(req,2)
if ok {
return rsp,nil
}
rsp = zy(req ,2,1)
return rsp,nil
}
func NewZyServiceImpl() *ZyServiceImpl {
return &ZyServiceImpl{}
}
......@@ -136,15 +153,24 @@ func zy(req map[string]string,_type byte,isCheck byte) (rsp *model.ZyhResponse){
zy_orgin_search = common.CopyMapString(req)//原始搜索數據
delete(req2,"class_id/condition")
rsp =searchZiYingGoods(req2,true,zhyClassList,classLight,_type)
//fmt.Printf("返回數據:%+v",rsp)
if _type == 2{
data,ok := rsp.Data.(map[string]interface{})
if ok {
delete(data,"class")
delete(data,"class_light")
}
}
return rsp
}
func(this *ZyServiceImpl) Index(ctx *gin.Context, req map[string]string) (rsp *model.ZyhResponse){
req2 := req
zy_orgin_search = common.CopyMapString(req)//原始搜索數據
xx := ziyingCache(req,1)
fmt.Println(xx)
rsp,ok := ziyingCache(req,3)
//fmt.Println(ok)
if ok {
return rsp
}
rsp = searchZiYingGoods(req2,false,nil,nil,1)
return rsp
}
......@@ -308,13 +334,13 @@ func searchZiYingGoods( req map[string]string,zy bool,classList *model.ZhyClassL
source.Query(query)
queryJson,_ := elastic.NewSearchRequest().Source(source).Body()
fmt.Println(queryJson)
logger.Info("es搜索查詢條件 %s",queryJson)
result, err := es.CurlES(config.Get("ZIYING_CONFIG.Index").String(), queryJson)
if err != nil {
log.Error(err)
}
//fmt.Println(result)
logger.Info(result)
logger.Info("查詢結果：%s",result)
//单独返回goods_id
if just_id,ok :=req["just_id"];ok && just_id == "2"{
hits2 := gjson.Get(result, "hits.hits.#._source").Array()
......@@ -354,6 +380,7 @@ func searchZiYingGoods( req map[string]string,zy bool,classList *model.ZhyClassL
aggregationsKeyBuckets := v.Map()
id:=aggregationsKeyBuckets["key"].String()
aggregationsKeyBucketsStr := ""
aggregationsKeyBucketsStr = aggregationsKeyBuckets["key"].String()
switch field {
case "brand_id":
brand_j,err :=redis.String(redisConn.Do("Hget","Self_Brand",aggregationsKeyBuckets["key"]))
......@@ -366,6 +393,8 @@ func searchZiYingGoods( req map[string]string,zy bool,classList *model.ZhyClassL
arr_oj,err := redis.String(redisConn.Do("Hget","Self_SelfClassInfo",aggregationsKeyBuckets["key"]))
if err == nil{
aggregationsKeyBucketsStr = gjson.Get(arr_oj,"class_name").String()
}else{
aggregationsKeyBucketsStr = ""
}
case "packing":
Self_UnitAliasInfo,err :=redis.String(redisConn.Do("Hget","Self_UnitAlias",aggregationsKeyBuckets["key"]))
......@@ -395,21 +424,25 @@ func searchZiYingGoods( req map[string]string,zy bool,classList *model.ZhyClassL
}
//attrs
attrsAttrNameBuckets := gjson.Get(aggregations["attrs"].String(),"attr_name.buckets").Array()
if zy && len(attrsAttrNameBuckets) > 0{
//fmt.Println("attrsAttrNameBuckets",attrsAttrNameBuckets)
aggsFieldList := make([]map[string]string,0)
for _,vbuckets:=range attrsAttrNameBuckets{
vbucketsMap := vbuckets.Map()
vbucketsMapKey :=vbucketsMap["key"].String()
attr_value_buckets := gjson.Get(vbuckets.String(),"attr_value.buckets")
for _,v := range attr_value_buckets.Map(){
res["aggs"][vbucketsMapKey] = make(map[string]interface{},0)
aggsFieldList := make([]map[string]string,0)
for _,v := range attr_value_buckets.Array(){
fieldMd5 := common.Md5(vbucketsMap["key"].String())
aggsFieldList = append(aggsFieldList,map[string]string{
"name":v.Map()["key"].String(),
"id":v.Map()["key"].String(),
"num":v.Map()["doc_count"].String(),
})
res["aggs"][vbucketsMap["key"].String()]["list"] = aggsFieldList
res["aggs"][vbucketsMap["key"].String()]["name"] = fieldMd5[8:24]
res["aggs"][vbucketsMapKey]["list"] = aggsFieldList
res["aggs"][vbucketsMapKey]["name"] = fieldMd5[8:24]
}
}
}
......@@ -605,6 +638,7 @@ func makePageInfo(paginator map[string]interface{},currentPage int) string{
return pageHtml
}
func makeQuery(query *elastic.BoolQuery,source *elastic.SearchSource,conditions []string,goods_temp_status *int,searchValue *string,
keyword *string,is_sample *bool,goods_name_arr *[]string) bool{
......@@ -945,18 +979,21 @@ func dullZiYingGoodsInfo(goods_id_str string,keyword string,goods_name_arr []str
符合条件 直接读缓存，不需要再查es+redis
@param string type 获取缓存数据：1 单独获取列表数据（class_id第一页数据） 2 单独获取aggs统计 3 获取列表（class_id第一页数据）+aggs统计
*/
func ziyingCache(req map[string]string,_type byte) ( bool){
func ziyingCache(req map[string]string,_type byte) (*model.ZyhResponse, bool){
var(
err error
cache bool = true
pInt int //page int
class_id int
mongodb *mgo.Session
rsp *model.ZyhResponse
)
rsp = &model.ZyhResponse{ErrorCode:0,ErrorMsg:"成功,数据更新时间: "}
//是否开启缓存
if isOpen,err :=config.Get("ZIYING_CACHE.is_no_ziying_cache").Bool() ;err == nil && !isOpen{
return false
return rsp,false
}
p := req["p"]
if p != "" {
......@@ -977,7 +1014,7 @@ func ziyingCache(req map[string]string,_type byte) ( bool){
}
}
if !cache{
//return false
return rsp,false
}
//需要查缓存
......@@ -985,7 +1022,7 @@ func ziyingCache(req map[string]string,_type byte) ( bool){
if class_idCondition != ""{
class_id,_ = strconv.Atoi(class_idCondition)
}
//agg_table := config.Get("AGG_TABLE.agg_table").String()
agg_table := config.Get("AGG_TABLE.agg_table").String()
firstLists_table := config.Get("FIRSTLISTS_TABLE.firstLists_table").String()
class_id=class_id
err=err
......@@ -995,36 +1032,187 @@ func ziyingCache(req map[string]string,_type byte) ( bool){
defer mongodb.Close()
c :=mongodb.DB("ichunt").C(firstLists_table)
//c1 :=mongodb.DB("ichunt").C(agg_table)
c1 :=mongodb.DB("ichunt").C(agg_table)
//var res interface{}
type res struct {
ID bson.ObjectId `json:"_id" bson:"_id"`
ClassId int `json:"class_id" bson:"class_id"`
//Data string `json:"data",bson:"data"`
Data interface{} `json:"data",bson:"data"`
UpdateTime string `json:"update_time" bson:"update_time"`
}
result := res{}
result2 := res{}
switch _type {
case 1://单独获取列表数据（class_id第一页数据）
err := c.Find(bson.M{"class_id":class_id}).One(&result)
fmt.Println("err",err)
err = c.Find(bson.M{"class_id":class_id}).One(&result)
if err == nil{
rsp.Data = result.Data
rsp.ErrorMsg += result.UpdateTime
}else{
return rsp, false
}
break
case 2://#单独获取aggs
err := c.Find(bson.M{"class_id":class_id}).One(&result)
fmt.Println("err",err)
err = c1.Find(bson.M{"class_id":class_id}).One(&result)
if err == nil{
rsp.Data = result.Data
rsp.ErrorMsg += result.UpdateTime
}else{
return rsp, false
}
break
case 3://获取列表（class_id第一页数据）+aggs统计
err1 := c1.Find(bson.M{"class_id":class_id}).One(&result2)
err2 := c.Find(bson.M{"class_id":class_id}).One(&result)
if err1 == nil && err2 == nil{
result2Data := result2.Data
result2DataBson,ok := result2Data.(bson.M)
if !ok{
return rsp, false
}
aggs,ok := result2DataBson["data"].(bson.M)
if !ok{
return rsp, false
}
aggsBson := aggs["aggs"]
ziying_first_lists_data,ok := result.Data.(bson.M)
if !ok{
return rsp, false
}
ziying_first_lists_data2 := ziying_first_lists_data["data"]
ziying_first_lists_data2_bson,ok := ziying_first_lists_data2.(bson.M)
if !ok{
return rsp, false
}
ziying_first_lists_data2_bson["aggs"] = aggsBson
rsp.Data = ziying_first_lists_data2_bson
rsp.ErrorMsg += result2.UpdateTime
}else{
return rsp, false
}
}
return rsp, true
}
return true
/*
统计样片一级分类总条数
*/
func(this *ZyServiceImpl)AggSample(ctx *gin.Context, req map[string]string) (rsp *model.ZyhResponse){
rsp = &model.ZyhResponse{ErrorCode:-1}
dataJson := `{
"size": 0,
"query": {
"bool": {
"must": [
{
"term": {
"status": "1"
}
},
{
"term": {
"sample_status": "1"
}
},
{
"range": {
"sample_class_id": {
"gte": 1
}
}
}
]
}
}
, "aggs": {
"sample_class_id": {
"terms": {
"size": 500,
"field":"sample_class_id"
}
}
}
}`
reg := regexp.MustCompile("# #")
dataJson = reg.ReplaceAllString(dataJson,"")
for _,v := range []string{"\r\n","\r","\n"}{
dataJson = strings.ReplaceAll(dataJson,v,"")
}
esIndex := config.Get("ZIYING_CONFIG.Index").String()
result,err :=es.CurlES(esIndex,dataJson)
if err != nil{
return rsp
}
redisConn := gredis.Conn("search_r")
defer func(){
redisConn.Close()
}()
aggsSampleBuckets := gjson.Get(result,"aggregations.sample_class_id.buckets").Array()
if len(aggsSampleBuckets) <= 0{
return rsp
}
type classInfoStruct struct {
SmapleClassId int64
SmapleClassName string
Count int64
Sort int
}
temp := make(map[string]classInfoStruct,0)
for _,item:= range aggsSampleBuckets{
itemMap := item.Map()
keyGjsonResult,ok:= itemMap["key"]
if !ok{
continue
}
key := keyGjsonResult.String()
if key == ""{
continue
}
classInfo,err := redis.String(redisConn.Do("Hget","Self_sample_class_list",key))
if err != nil{
continue
}
tempClassInfoStruct := classInfoStruct{
SmapleClassId:keyGjsonResult.Int(),
SmapleClassName:gjson.Get(classInfo,"class_name").String(),
Count:itemMap["doc_count"].Int(),
}
temp[key] = tempClassInfoStruct
}
//排序
sample_class_data := model.GetSampleClass()
res :=make([]classInfoStruct,0)
for _,val:=range sample_class_data{
sample_class_id := val.Id
sample_class_id_str := strconv.FormatInt(int64(sample_class_id),10)
//array_key_exists
bb,ok:= temp[sample_class_id_str]
if ok{
bb.Sort = val.Sort
//temp[sample_class_id_str] = bb
res = append(res,bb)
}
}
if len(res) > 0{
rsp.ErrorCode=0
rsp.Data = res
return
}
return
}
\ No newline at end of file
test/test.go
......@@ -7,7 +7,7 @@ import (
"github.com/syyongx/php2go"
"reflect"
"search_server/boot"
"search_server/pkg/common"
"search_server/service"
)
func StrRandom(lenNum int) string {
......@@ -39,18 +39,8 @@ func main() {
if err := boot.Boot(path); err != nil {
fmt.Println(err)
}
//log.SetFormatter(new(LogFormatter))
////设置output,默认为stderr,可以为任何io.Writer，比如文件*os.File
//str, _ := os.Getwd()
//file, _ := os.OpenFile(str+"/logs/search/1.log", os.O_CREATE|os.O_WRONLY, 0666)
//log.SetOutput(file)
//fmt.Println(str)
//log.Error("sadsadsadsa")
//log.Error("sadsadsadsa")
//log_report.WriteSearchErrorLog("sdasdasdas")
common.Output(1000, "ceshiceshi", nil)
//log.Error("sadsadsadsa")
//controller.GetData("LM358","arrow")
res, _ := service.GetGoodsInfo([]string{"74163","116492","74123","106095","116451","162774","74152","116078","74122","74132"})
fmt.Println(len(res))
}
......
test/test7.go
......@@ -2,13 +2,16 @@ package main
import (
"bytes"
"fmt"
"io"
)
const debug = false
func main() {
var buf *bytes.Buffer
//var buf *bytes.Buffer
var buf io.Writer
fmt.Printf("%T \n",buf)
if debug {
buf = new(bytes.Buffer) // enable collection of output
}
......@@ -21,7 +24,18 @@ func main() {
// If out is non-nil, output will be written to it.
func f(out io.Writer) {
// ...do something...
fmt.Println(out)
fmt.Printf("%T \n",out)
if out != nil {
fmt.Println("!!!!!!!!!!")
out.Write([]byte("done!\n"))
}
var a interface{}
a = map[string]string{
"a":"b",
}
a = nil
b,_ := a.(map[string]string)
fmt.Println(b)
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment