Commit bed4ba30 by mushishixian

SearchLog控制器

parent e0cb0572
......@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/gin-gonic/gin"
"search_server/service/ly"
"time"
)
//搜索型号
......@@ -12,7 +13,7 @@ func GetMouserData(c *gin.Context) {
goodsName, _ := c.GetPostForm("goods_name")
//lycon := new(ly.CommonLyService)
//调外链拿数据
apiData := ly.OutLinkMouser(&goodsName,"-1")
apiData := ly.OutLinkMouser(&goodsName, "-1")
//var shu string
//for _, info := range apiData {
......@@ -42,12 +43,14 @@ func GetMouserData(c *gin.Context) {
if errs != nil {
fmt.Println("错误")
}
c.JSON(200,string(josn))
time.Sleep(1 * time.Second)
temp := string(josn)
c.JSON(200, temp)
return
//供应商详情
//supplierInfo := model.SUPPLIER_REDIS_INFO{
// SupplierId:14,
// SupplierNickname: "mouser",
// SupplierId:14,
// SupplierNickname: "mouser",
//}
//err := lycon.GetSkuByGoodsSn(apiData,&supplierInfo)
//var errCode int
......
package controller
import (
"encoding/json"
"github.com/gin-gonic/gin"
"search_server/pkg/common"
"search_server/pkg/es"
"search_server/requests"
"search_server/service"
"strconv"
"strings"
)
//提供日志接口给日志系统
func SearchLogIndex(c *gin.Context) {
request := make(map[string]string, 0)
c.MultipartForm()
for name, value := range c.Request.Form {
if value[0] != "" {
request[name] = strings.TrimSpace(value[0])
}
}
searchLogService := service.SearchLogService{}
result, err := searchLogService.GetIndexData(request)
if err != nil {
common.ReturnData(1, "获取ES数据失败 : "+err.Error(), nil)
}
common.ReturnData(0, "", result)
return
}
func Scroll(c *gin.Context) {
var r requests.ScrollRequest
c.ShouldBind(&r)
if r.ScrollId == 0 {
common.ReturnData(1, "", nil)
}
params := make(map[string]interface{})
if r.Scroll > 0 {
params["scroll"] = strconv.Itoa(r.Scroll) + "m"
} else {
params["scroll"] = "5m"
}
params["scroll_id"] = r.ScrollId
paramsStr, _ := json.Marshal(params)
_, err := es.ScrollES(string(paramsStr))
if err != nil {
common.ReturnData(1, "Scroll失败 : "+err.Error(), nil)
}
}
//根据搜索日志统计出时间短访问最多的ip'
func SumIp(c *gin.Context) {
//统计品牌下的商品数量
flag := common.CheckSignApi()
if flag > 0 {
common.ReturnData(flag, "", nil)
return
}
var r requests.SumIpRequest
c.ShouldBind(&r)
searchLogService := service.SearchLogService{}
res, err := searchLogService.GetSumIp(r)
if err != nil {
common.ReturnData(1, err.Error(), nil)
return
}
common.ReturnData(0, "", res)
return
}
......@@ -51,7 +51,7 @@ type ApiGoods struct {
ScmBrandName string `json:"scm_brand_name,omitempty"`
AllowCoupon int `json:"allow_coupon"`
ClassId1Name string `json:"class_id1_name"`
CLassId2Name string `json:"c_lass_id2_name"`
CLassId2Name string `json:"class_id2_name"`
SpuId string `json:"spu_id,omitempty"`
BatchSn string `json:"batch_sn"`
Canal string `json:"canal"`
......
package model
type SearchLogHits struct {
Index string `json:"_index"`
Type string `json:"_type"`
ID string `json:"_id"`
Score int64 `json:"_score"`
Source SearchLogSource `json:"_source"`
}
type SearchLogSource struct {
IP int64 `json:"ip"`
UserID int64 `json:"user_id"`
UserSign string `json:"user_sign"`
Ptag string `json:"ptag"`
BrowserInfo string `json:"browser_info"`
CreateTime int64 `json:"create_time"`
Keyword string `json:"keyword"`
Flag int64 `json:"flag"`
Platform int64 `json:"platform"`
Adtag string `json:"adtag"`
}
......@@ -61,8 +61,25 @@ func Output(errCode int, errMsg string, data interface{}) {
ctx.Header("Access-Control-Allow-Credentials", "true")
//允许跨站访问的站点域名
//跨域请求头设置
ctx.JSONP(200, response)
ctx.JSON(200, response)
}
}
//简单的返回数据方法
func ReturnData(errCode int, errMsg string, data interface{}) {
if data == nil {
data = []string{}
if errCode == 0 {
errCode = 1
}
}
response := Response{
ErrCode: errCode,
ErrMsg: errMsg,
Data: data,
}
ctx := middleware.Context
ctx.JSON(200, response)
}
//错误的搜索日志记录
......
......@@ -70,6 +70,8 @@ func BulkES(param string) (result string, err error) {
result = resp.String()
return
}
//多条件多索引查询,需要查询的索引和条件自己拼接成queryJson里面
func CurlESMSearch(queryJson string) (result string, err error) {
endpoints := config.Get("es.urls").Strings(",")
//随机获取一个节点进行请求
......@@ -83,3 +85,18 @@ func CurlESMSearch(queryJson string) (result string, err error) {
result = resp.String()
return
}
//滚动搜索和索引之间的文档重索引
func ScrollES(param string) (result string, err error) {
endpoints := config.Get("es.urls").Strings(",")
//随机获取一个节点进行请求
req.Debug = false
esUrl := endpoints[rand.Intn(len(endpoints))]
params := req.BodyJSON(param)
resp, err := req.Post(esUrl+"/_search/scroll", params)
if err != nil {
return
}
result = resp.String()
return
}
package requests
type SumIpRequest struct {
StartTime int64 `form:"start_time"`
EndTime int64 `form:"end_time"`
Size int `form:"size"`
}
type ScrollRequest struct {
Scroll int `form:"scroll"`
ScrollId int `form:"scroll_id"`
}
......@@ -4,7 +4,6 @@ import (
"github.com/gin-gonic/gin"
"search_server/controller"
"search_server/middleware"
"search_server/pkg/common"
)
//初始化路由
......@@ -15,7 +14,6 @@ func InitRouter() *gin.Engine {
r.Use(gin.Recovery())
r.Use(middleware.Cors())
r.Use(middleware.ContextVars())
//路由
r.POST("/search/bom/autospu", controller.AutoSpu)
r.POST("/search/bom/recommend", controller.Recommend)
......@@ -35,11 +33,17 @@ func InitRouter() *gin.Engine {
//Other控制器相关
r.POST("/search/other/hasStock", controller.CheckHasStock)
r.GET("/search/other/exactGoods", controller.ExactGoods)
r.POST("/search/other/exactGoods", controller.ExactGoods)
r.GET("/test", func(c *gin.Context) {
common.Output(1000, "testest", nil)
})
//SearchLog控制器相关
r.GET("/search/search_log/sumIp", controller.SumIp)
r.POST("/search/search_log/sumIp", controller.SumIp)
r.POST("/search/search_log/scroll", controller.Scroll)
r.GET("/search/search_log/index", controller.SearchLogIndex)
r.POST("/search/search_log/index", controller.SearchLogIndex)
r.GET("/search/search_log", controller.SearchLogIndex)
r.POST("/search/search_log", controller.SearchLogIndex)
return r
}
......@@ -141,7 +141,7 @@ func CurlGoodsInfo(goodsIdsStr string, params req.Param) (goodsList []model.ApiG
goods.LadderPrice = ladderPrice
//other_attrs
goods.OtherAttrs.GrossWeight = data.Get("other_attrs.gross_weight").String()
goods.OtherAttrs.GrossWeight = data.Get("other_attrs.gross_wegiht").String()
//Attrs
attrMap := make(map[string]model.Attr, 0)
......
package query
import (
"github.com/syyongx/php2go"
"gopkg.in/olivere/elastic.v5"
"search_server/requests"
"strconv"
"strings"
)
func GetSumIpQuery(r requests.SumIpRequest) (queryString string) {
query := elastic.NewBoolQuery()
source := elastic.NewSearchSource()
query.Must(elastic.NewRangeQuery("create_time").Gte(r.StartTime).Lte(r.EndTime))
source.Aggregation("tatol", elastic.NewTermsAggregation().Field("ip").Size(100))
source.Size(0)
source.Query(query)
queryRequest := elastic.NewSearchRequest().Source(source)
queryString, _ = queryRequest.Body()
return
}
func GetSearchLogIndexQuery(request map[string]string) (queryString string) {
page, _ := strconv.Atoi(request["p"])
if page == 0 {
page = 1
}
//需要转换成大写的字段
needUpperField := []string{"keyword"}
//开始构建ES的查询条件
query := elastic.NewBoolQuery()
source := elastic.NewSearchSource()
for name, value := range request {
//为什么要切割这个参数名呢,因为传上来的参数名,是类似
//goods_name/condition , stock/range 这样子的形式
fieldNameAndCondition := strings.Split(name, "/")
if len(fieldNameAndCondition) == 2 {
filedName := fieldNameAndCondition[0]
condition := fieldNameAndCondition[1]
if condition == "condition" {
//es的term条件数组
//用interface{}类型主要是为了下面构建查询条件方便
termSlice := []interface{}{value}
if filedName != "keyword" && filedName != "browser_info" && filedName != "adtag" {
if filedName == "ip" {
value = strconv.Itoa(int(php2go.IP2long(value)))
}
termSlice[0] = value
} else {
//判断参数名是否属于需要转换成大写的字段
if php2go.InArray(filedName, needUpperField) {
if request["is_exact"] != "" {
termSlice[0] = value
termSlice = append(termSlice, strings.ToLower(value))
termSlice = append(termSlice, strings.ToUpper(value))
} else {
termSlice[0] = strings.ToUpper(value)
}
}
}
//判断is_exact字段
if request["is_exact"] != "" {
query.Must(elastic.NewTermsQuery("keyword.raw", termSlice...))
} else {
query.Must(elastic.NewTermsQuery(filedName, termSlice...))
}
}
hasOrder := false
//排序相关的参数
if condition == "order" {
hasOrder = true
//默认正序
orderFlag := true
if value == "false" || value == "desc" || value == "-1" {
orderFlag = false
}
source.Sort(filedName, orderFlag)
}
//范围相关的查询
if condition == "range" {
rangeCondition := strings.Split(value, ",")
if len(rangeCondition) >= 2 {
gteValue := rangeCondition[0]
lteValue := rangeCondition[1]
query.Must(elastic.NewRangeQuery(filedName).Gte(gteValue).Lte(lteValue))
}
}
//单边范围查询
if condition == "sr" {
rangeCondition := strings.Split(value, ",")
if len(rangeCondition) >= 2 {
}
}
//如果没有排序,就开启默认排序
if !hasOrder {
source.Sort("create_time", false)
}
if page > 1000 {
page = 1000
}
//获取size
size := 10
if request["offset"] != "" {
size, _ = strconv.Atoi(request["offset"])
}
start := (page - 1) * size
source.From(start)
source.Size(size)
}
}
source.Query(query)
searchRequest := elastic.NewSearchRequest().Source(source)
queryString, _ = searchRequest.Body()
return
}
package service
import (
"encoding/json"
"fmt"
"github.com/tidwall/gjson"
"github.com/uniplaces/carbon"
"search_server/model"
"search_server/pkg/es"
"search_server/requests"
"search_server/service/query"
"strconv"
)
type SearchLogService struct {
}
//获取index方法的数据
func (sls *SearchLogService) GetIndexData(request map[string]string) (result map[string]interface{}, err error) {
//索引/类型
index := "keyword_search_log_1/log"
var esResult string
_, exist := request["scroll_id"]
if exist || request["scroll_id"] == "" {
queryString := query.GetSearchLogIndexQuery(request)
esResult, err = es.CurlES(index, queryString)
if err != nil {
return
}
} else {
params := make(map[string]string)
scroll, _ := strconv.Atoi(request["scroll"])
if scroll > 0 {
params["scroll"] = params["scroll"] + "m"
} else {
params["scroll"] = "5m"
}
params["scroll_id"] = request["scroll_id"]
paramsStr, _ := json.Marshal(params)
esResult, err = es.ScrollES(string(paramsStr))
if err != nil {
return
}
}
fmt.Println(esResult)
result = make(map[string]interface{})
//循环遍历出数据
result["total"] = gjson.Get(esResult, "hits.total").Int()
result["scroll_id"] = gjson.Get(esResult, "_scroll_id").String()
sources := gjson.Get(esResult, "hits.hits.#_source").Array()
var list []map[string]interface{}
for _, source := range sources {
listItem := make(map[string]interface{})
var hits model.SearchLogHits
var platform string
_ = json.Unmarshal([]byte(source.String()), &hits)
if hits.Source.Platform == 1 {
platform = "PC"
} else if hits.Source.Platform == 2 {
platform = "H5"
} else {
platform = ""
}
listItem["create_time"] = hits.Source.CreateTime
listItem["user_id"] = hits.Source.UserID
listItem["flag"] = hits.Source.Flag
listItem["ip"] = hits.Source.IP
listItem["keyword"] = hits.Source.Keyword
listItem["browser_info"] = hits.Source.BrowserInfo
listItem["platform"] = platform
listItem["adtag"] = hits.Source.Adtag
listItem["ptag"] = hits.Source.Ptag
listItem["user_sign"] = hits.Source.UserSign
list = append(list, listItem)
}
result["list"] = list
return
}
func (sls *SearchLogService) GetSumIp(r requests.SumIpRequest) (res []map[string]interface{}, err error) {
//默认24小时
if r.StartTime == 0 {
r.StartTime = carbon.Now().SubDays(1).Timestamp()
}
if r.EndTime == 0 {
r.EndTime = carbon.Now().Timestamp()
}
if r.Size == 0 {
r.Size = 100
}
if r.Size > 20000 {
r.Size = 20000
}
queryString := query.GetSumIpQuery(r)
fmt.Println(queryString)
index := "keyword_search_log_1"
esResult, err := es.CurlES(index, queryString)
if err != nil {
return
}
buckets := gjson.Get(esResult, "aggregations.tatol.buckets").Array()
for _, bucket := range buckets {
item := map[string]interface{}{
"ip": bucket.Map()["key"].String(),
"num": bucket.Map()["doc_count"].Int(),
}
res = append(res, item)
}
return
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment