Skip to content

lzzzzl / golang_seo_server

Go to a project
Commit 34b2a24d by lzzzzl
Options

项目重构

parents
Inline Side-by-side
Showing with 1390 additions and 0 deletions
.gitignore 0 → 100644
/.idea
/.DS_Store
/logs/*
/log/*
/config/*.toml
cmd/main.go 0 → 100644
package main
import (
"flag"
"fmt"
"github.com/ichunt2019/golang-rbmq-sl/utils/rabbitmq"
"github.com/ichunt2019/logger"
"github.com/robfig/cron/v3"
"github.com/tidwall/gjson"
"golang-seo-server/collect/lxmain"
"golang-seo-server/collect/ly"
"golang-seo-server/collect/mip"
//"golang-seo-server/collect/lxmain"
//"golang-seo-server/collect/mip"
//"golang-seo-server/collect/lxmain"
//"golang-seo-server/collect/ly"
//"golang-seo-server/collect/mip"
"golang-seo-server/common/ding"
//"github.com/robfig/cron/v3"
//"golang-seo-server/collect/mip"
"golang-seo-server/dal/db"
"golang-seo-server/dal/function"
"golang-seo-server/util"
)
var ConfigDir string
var LogDir string
type RecvPro struct{}
// 解析命令行参数
func initArgs() {
flag.StringVar(&ConfigDir, "configDir", "", "配置文件")
flag.StringVar(&LogDir, "logDir", "", "日志目录")
flag.Parse()
}
// 初始化DB
func initDb(action string, dns string) (err error) {
err = db.CreateDB(action, dns)
if err != nil {
return
}
return
}
//消息已经消费3次 失败了 请进行处理
func (t *RecvPro) FailAction(err error, dataByte []byte) error {
logger.Error("任务处理失败了，发送钉钉消息")
logger.Error(string(dataByte))
logger.Error("错误原因：%s", err)
ding.Send(
util.Configs.Ding_msg.Webhook,
"SEO推送",
fmt.Sprintf("队列seo_log 失败原因:%s \n %s", err, string(dataByte)),
make([]string, 0),
false,
)
return nil
}
// 实现消费者 消费消息失败 自动进入延时尝试 尝试3次之后入库db
func (t *RecvPro) Consumer(dataByte []byte) error {
url := gjson.Get(string(dataByte), "url").String()
status := gjson.Get(string(dataByte), "status").String()
urlType := function.RegxUrl(url)
function.InsertLog(url, status, urlType)
return nil
}
func main() {
initArgs()
util.Init(ConfigDir)
// log配置
logConfig := make(map[string]string)
logConfig["log_path"] = LogDir + "seo"
logConfig["log_chan_size"] = "1000"
_ = logger.InitLogger("file", logConfig)
logger.Init()
_ = initDb("InitSeoDB", util.Configs.Seo_databases.Dns)
_ = initDb("InitSpuDB", util.Configs.Spu_databases.Dns)
_ = initDb("InitGoodsDB", util.Configs.Goods_databases.Dns)
_ = initDb("InitSkuDB", util.Configs.Sku_databases.Dns)
_ = initDb("InitLxDB", util.Configs.Liexin_databases.Dns)
t := &RecvPro{}
go rabbitmq.Recv(rabbitmq.QueueExchange{
util.Configs.Rabbitmq_seo_log.QueueName,
util.Configs.Rabbitmq_seo_log.RoutingKey,
util.Configs.Rabbitmq_seo_log.Exchange,
util.Configs.Rabbitmq_seo_log.Type,
util.Configs.Rabbitmq_seo_log.Dns,
}, t, 3)
// 剩余条数
var lyRemain = function.RemainCount(util.Configs.Request_url.LyUrl, util.Configs.Request_url.LyData)
var mainRemain = function.RemainCount(util.Configs.Request_url.MainUrl, util.Configs.Request_url.MainData)
var mipRemain = function.RemainCount(util.Configs.Request_url.MipUrl, util.Configs.Request_url.MipData)
fmt.Println(lyRemain)
fmt.Println(mainRemain)
fmt.Println(mipRemain)
ly.RunLyTask(lyRemain, ConfigDir)
lxmain.RunMainTask(mainRemain, ConfigDir)
mip.RunMipTask(mipRemain, ConfigDir)
// 根据cron表达式进行时间调度，cron可以精确到秒
//crontab := cron.New() 默认从分开始进行时间调度
crontab := cron.New(cron.WithSeconds()) //精确到秒
//定义定时器调用的任务函数
task := func() {
logger.Info("定时任务启动，每天凌晨1点运行")
ly.RunLyTask(lyRemain, ConfigDir)
lxmain.RunMainTask(mainRemain, ConfigDir)
mip.RunMipTask(mipRemain, ConfigDir)
//
defer db.SeoDB.Close()
defer db.SpuDB.Close()
defer db.GoodsDB.Close()
defer db.SkuDB.Close()
defer db.LxDB.Close()
////fmt.Println("******** ******* *******")
//fmt.Println("Time now:", time.Now().Format("2006-01-02 15:04:05"))
}
//定时任务
spec := "0 0 1 * * ?" //cron表达式，每天凌晨1点运行
//spec := "2 * * * * *" //cron表达式，表示每分钟的第2s执行一次
// 添加定时任务,
_, _ = crontab.AddFunc(spec, task)
// 启动定时器
crontab.Start()
//阻塞主线程停止
select {}
}
collect/lxmain/lxmain.go 0 → 100644
package lxmain
import (
"fmt"
"github.com/ichunt2019/logger"
"github.com/jmoiron/sqlx"
"golang-seo-server/common/ding"
"golang-seo-server/dal/colentry"
"golang-seo-server/dal/colsql"
"golang-seo-server/dal/colurl"
"golang-seo-server/dal/db"
"golang-seo-server/dal/function"
"golang-seo-server/model/liexin_seo/Urls"
"golang-seo-server/util"
"strconv"
"strings"
)
var mobile []string
var isAtAll bool
var minCount = 1000
var MAX_TABLE = 9
var MAX_CONN = 9
var LY_SPU = "ly_spu"
var LY_DETAIL = "ly_detail"
/**
* @author zzl
* @description // 执行任务
* @date 11:47 2020/11/13
* @param
* @return
**/
func RunMainTask(remainCount int, configDir string) {
util.Init(configDir)
// 获取联营任务状态
var sql = "SELECT t.collect_code,t.collect_id,t.db_conn,t.db_table,t.db_count " +
"FROM lie_collect t LEFT JOIN lie_class c ON t.class_id = c.class_id " +
"WHERE t.status = 0 AND c.parent_id AND c.parent_id = 1"
row, err := db.SeoDB.Queryx(sql)
if err != nil {
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "查询主站任务状态失败，原因："+err.Error(),
mobile, isAtAll)
return
}
for row.Next() {
// 剩余条数少于100则退出
if remainCount <= minCount {
break
}
var collect Urls.Collect
_ = row.StructScan(&collect)
var collectCode = collect.CollectCode
var collectId = collect.CollectId
var collectDbConn = collect.DbConn
var collectDbCount = collect.DbCount
var collectDbTable = collect.DbTable
var tableIndex int
var colDb *sqlx.DB
//
var DbConn = ""
var DbConnIndex = 0
if collectCode == LY_SPU {
tableIndex, _ = strconv.Atoi(strings.Split(collectDbTable, "_")[2])
} else if collectCode == LY_DETAIL {
DbConn = strings.Split(collectDbConn, "_")[0]
DbConnIndex, _ = strconv.Atoi(strings.Split(collectDbConn, "_")[1])
collectDbConn = DbConn
_ = db.InitSkuDB(fmt.Sprintf(util.Configs.Sku_databases.Dns, DbConnIndex))
tableIndex, _ = strconv.Atoi(strings.Split(collectDbTable, "_")[2])
}
var cdCount, _ = strconv.Atoi(collectDbCount)
var sql = colsql.MainSQL(collectCode, tableIndex, cdCount, cdCount+remainCount)
colDb = db.GetDB(collectDbConn)
res, err := colDb.Queryx(sql)
fmt.Println(sql)
if err != nil {
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "查询主站ID任务失败，原因："+err.Error(),
mobile, isAtAll)
continue
}
var conUrl = []string{}
var dbCount, _ = strconv.Atoi(collectDbCount)
// 遍历ID数据
for res.Next() {
dbCount++
remainCount--
if remainCount > minCount {
var entryId = colentry.MainEntryId(collectCode, res)
var url = fmt.Sprintf(colurl.MainUrl(collectCode, configDir), entryId)
fmt.Println(url)
logger.Info("URL：%s，count：%d", url, dbCount)
conUrl = append(conUrl, url)
if dbCount%10 == 0 {
function.RabbitSeoSend(strings.Join(conUrl, "\n"))
conUrl = []string{}
}
} else {
break
}
}
// 发送剩余消息
if len(conUrl) > 0 {
function.RabbitSeoSend(strings.Join(conUrl, "\n"))
}
var status = 1
var dbCountStr = strconv.Itoa(dbCount)
// remainCount > minCount，说明数据已跑完
if remainCount > minCount {
// 判断采集类型
if collectCode == LY_SPU {
dbCount = 0
if tableIndex < MAX_TABLE {
status = 0
tableIndex++
} else {
status = 1
}
collectDbTable = "lie_spu" + "_" + strconv.Itoa(tableIndex)
}
if collectCode == LY_DETAIL {
dbCount = 0
if tableIndex < MAX_TABLE {
status = 0
tableIndex++
} else if tableIndex == MAX_TABLE {
if DbConnIndex == MAX_CONN {
status = 1
} else {
status = 0
// 库索引++
DbConnIndex++
// 表重置0
tableIndex = 0
}
}
collectDbTable = "lie_sku" + "_" + strconv.Itoa(tableIndex)
collectDbConn = "sku" + "_" + strconv.Itoa(DbConnIndex)
}
function.UpdateCollect(collectId, status, dbCount, collectDbTable, collectDbConn)
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "主站任务 {"+
collectCode+"} 推送完成, 数量："+dbCountStr, mobile, isAtAll)
} else {
// 记录写入位置
status = 0
// 恢复DBconn
if collectCode == LY_DETAIL {
collectDbConn = "sku" + "_" + strconv.Itoa(DbConnIndex)
}
function.UpdateCollect(collectId, status, dbCount, collectDbTable, collectDbConn)
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "主站任务 {"+
collectCode+"} 推送未完成, 数量："+dbCountStr, mobile, isAtAll)
}
}
}
collect/ly/ly.go 0 → 100644
package ly
import (
"fmt"
"github.com/ichunt2019/logger"
"golang-seo-server/common/ding"
"golang-seo-server/dal/colentry"
"golang-seo-server/dal/colsql"
"golang-seo-server/dal/colurl"
"golang-seo-server/dal/db"
"golang-seo-server/dal/function"
"golang-seo-server/model/liexin_seo/Urls"
"golang-seo-server/util"
"strconv"
"strings"
)
var mobile []string
var isAtAll bool
/**
* @author zzl
* @description // 执行任务
* @date 11:47 2020/11/13
* @param
* @return
**/
func RunLyTask(remainCount int, configDir string) {
util.Init(configDir)
// 获取联营任务状态
var sql = "SELECT t.collect_code,t.collect_id,t.db_conn,t.db_table,t.db_count " +
"FROM lie_collect t LEFT JOIN lie_class c ON t.class_id = c.class_id " +
"WHERE t.status = 0 AND c.parent_id AND c.parent_id = 19"
row, err := db.SeoDB.Queryx(sql)
if err != nil {
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "查询联营任务状态失败，原因："+err.Error(), mobile, isAtAll)
return
}
for row.Next() {
// 剩余条数少于100则退出
if remainCount <= 100 {
break
}
var collect Urls.Collect
_ = row.StructScan(&collect)
var collectCode = collect.CollectCode
var collectId = collect.CollectId
var collectDbConn = collect.DbConn
var collectDbCount = collect.DbCount
var collectDbTable = collect.DbTable
var sql = colsql.LySQL(collectCode)
var colDb = db.GetDB(collectDbConn)
res, err := colDb.Queryx(sql)
if err != nil {
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "查询联营ID任务失败，原因："+err.Error(),
mobile, isAtAll)
continue
}
var conUrl = []string{}
var dbCount, _ = strconv.Atoi(collectDbCount)
// 遍历ID数据
for res.Next() {
dbCount++
remainCount--
if remainCount > 100 {
var entryId = colentry.LyEntryId(collectCode, res)
var url = fmt.Sprintf(colurl.LyUrl(collectCode, configDir), entryId)
logger.Info("URL：%s，count：%d", url, dbCount)
conUrl = append(conUrl, url)
if dbCount%10 == 0 {
function.RabbitSeoSend(strings.Join(conUrl, "\n"))
conUrl = []string{}
}
} else {
break
}
}
// 发送剩余消息
if len(conUrl) > 0 {
function.RabbitSeoSend(strings.Join(conUrl, "\n"))
}
var dbCountStr = strconv.Itoa(dbCount)
if remainCount > 100 {
function.UpdateCollect(collectId, 1, dbCount, collectDbTable, collectDbConn)
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "联营任务 {"+
collectCode+"} 推送完成, 数量："+dbCountStr, mobile, isAtAll)
} else {
// 记录写入位置
function.UpdateCollect(collectId, 0, dbCount, collectDbTable, collectDbConn)
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "联营任务 {"+
collectCode+"} 推送未完成, 数量："+dbCountStr, mobile, isAtAll)
}
}
}
collect/mip/mip.go 0 → 100644
package mip
import (
"fmt"
"github.com/ichunt2019/logger"
"github.com/jmoiron/sqlx"
"golang-seo-server/common/ding"
"golang-seo-server/dal/colentry"
"golang-seo-server/dal/colsql"
"golang-seo-server/dal/colurl"
"golang-seo-server/dal/db"
"golang-seo-server/dal/function"
"golang-seo-server/model/liexin_seo/Urls"
"golang-seo-server/util"
"strconv"
"strings"
)
var mobile []string
var isAtAll bool
var minCount = 1000
var MAX_TABLE = 9
var MAX_CONN = 9
var LY_SPU = "mip_ly_spu"
var LY_DETAIL = "mip_ly_detail"
/**
* @author zzl
* @description // 执行任务
* @date 11:47 2020/11/13
* @param
* @return
**/
func RunMipTask(remainCount int, configDir string) {
util.Init(configDir)
// 获取联营任务状态
var sql = "SELECT t.collect_code,t.collect_id,t.db_conn,t.db_table,t.db_count " +
"FROM lie_collect t LEFT JOIN lie_class c ON t.class_id = c.class_id " +
"WHERE t.status = 0 AND c.parent_id AND c.parent_id = 10"
row, err := db.SeoDB.Queryx(sql)
if err != nil {
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "查询MIP任务状态失败，原因："+err.Error(),
mobile, isAtAll)
return
}
for row.Next() {
// 剩余条数少于100则退出
if remainCount <= 100 {
break
}
var collect Urls.Collect
_ = row.StructScan(&collect)
var collectCode = collect.CollectCode
var collectId = collect.CollectId
var collectDbConn = collect.DbConn
var collectDbCount = collect.DbCount
var collectDbTable = collect.DbTable
var tableIndex int
var colDb *sqlx.DB
var DbConn = ""
var DbConnIndex = 0
if collectCode == LY_SPU {
tableIndex, _ = strconv.Atoi(strings.Split(collectDbTable, "_")[2])
} else if collectCode == LY_DETAIL {
DbConn = strings.Split(collectDbConn, "_")[0]
DbConnIndex, _ = strconv.Atoi(strings.Split(collectDbConn, "_")[1])
collectDbConn = DbConn
_ = db.InitSkuDB(fmt.Sprintf(util.Configs.Sku_databases.Dns, DbConnIndex))
tableIndex, _ = strconv.Atoi(strings.Split(collectDbTable, "_")[2])
}
var cdCount, _ = strconv.Atoi(collectDbCount)
var sql = colsql.MipSQL(collectCode, tableIndex, cdCount, cdCount+remainCount)
colDb = db.GetDB(collectDbConn)
res, err := colDb.Queryx(sql)
fmt.Println(sql)
if err != nil {
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "查询MIP_ID任务失败，原因："+err.Error(),
mobile, isAtAll)
continue
}
var conUrl = []string{}
var dbCount, _ = strconv.Atoi(collectDbCount)
// 遍历ID数据
for res.Next() {
dbCount++
remainCount--
if remainCount > minCount {
var entryId = colentry.MipEntryId(collectCode, res)
var url = fmt.Sprintf(colurl.MipUrl(collectCode, configDir), entryId)
conUrl = append(conUrl, url)
fmt.Println(url)
logger.Info("URL：%s，count：%d", url, dbCount)
if dbCount%10 == 0 {
function.RabbitSeoSend(strings.Join(conUrl, "\n"))
conUrl = []string{}
}
} else {
break
}
}
// 发送剩余消息
if len(conUrl) > 0 {
function.RabbitSeoSend(strings.Join(conUrl, "\n"))
}
var status = 1
var dbCountStr = strconv.Itoa(dbCount)
// remainCount > minCount，说明数据已跑完
if remainCount > minCount {
// 判断采集类型
if collectCode == LY_SPU {
dbCount = 0
if tableIndex < MAX_TABLE {
status = 0
tableIndex++
} else {
status = 1
}
collectDbTable = "lie_spu" + "_" + strconv.Itoa(tableIndex)
}
if collectCode == LY_DETAIL {
dbCount = 0
if tableIndex < MAX_TABLE {
status = 0
tableIndex++
} else if tableIndex == MAX_TABLE {
if DbConnIndex == MAX_CONN {
status = 1
} else {
status = 0
// 库索引++
DbConnIndex++
// 表重置0
tableIndex = 0
}
}
collectDbTable = "lie_sku" + "_" + strconv.Itoa(tableIndex)
collectDbConn = "sku" + "_" + strconv.Itoa(DbConnIndex)
}
function.UpdateCollect(collectId, status, dbCount, collectDbTable, collectDbConn)
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "MIP任务 {"+
collectCode+"} 推送完成, 数量："+dbCountStr, mobile, isAtAll)
} else {
// 记录写入位置
status = 0
// 恢复DBconn
if collectCode == LY_DETAIL {
collectDbConn = "sku" + "_" + strconv.Itoa(DbConnIndex)
}
function.UpdateCollect(collectId, status, dbCount, collectDbTable, collectDbConn)
ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送", "MIP任务 {"+
collectCode+"} 推送未完成, 数量："+dbCountStr, mobile, isAtAll)
}
}
}
common/ding/ding.go 0 → 100644
package ding
import (
"encoding/json"
_ "fmt"
"github.com/ichunt2019/logger"
"io/ioutil"
"net/http"
"strings"
)
type Msg struct {
Msgtype string `json:"msgtype"`
Text Contents `json:"text"`
At Ats `json:"at"`
}
type Contents struct {
Content string `json:"content"`
}
type Ats struct {
AtMobiles []string `json:"atMobiles"`
IsAtAll bool `json:"isAtAll"`
}
// json 返回值
type JosnResp struct {
Errcode int `json:"errcode"`
Errmsg string `json:"errmsg"`
}
func Send(ding_url string, ding_tag string, textMsg string, mobiles []string, isAtAll bool) (jsonStr string) {
var msg Msg
msg = Msg{Msgtype: "text"}
msg.Text.Content = ding_tag + "：" + textMsg // 固定标签 + 文本
msg.At.AtMobiles = mobiles
msg.At.IsAtAll = isAtAll
content, _ := json.Marshal(msg)
// content := `{
// "msgtype": "text",
// "text": {
// "content": "`+ msg + `"
// }
// }`
client := &http.Client{}
req, _ := http.NewRequest("POST", ding_url, strings.NewReader(string(content)))
req.Header.Set("Content-Type", "application/json; charset=utf-8")
resp, err := client.Do(req)
defer resp.Body.Close()
if err != nil {
logger.Info(err.Error())
}
body, _ := ioutil.ReadAll(resp.Body) // 获取接口返回数据
res := JosnResp{}
if err1 := json.Unmarshal([]byte(body), &res); err1 != nil { // 将接口数据写入res
logger.Info(err1.Error())
}
result, _ := json.Marshal(res)
return string(result)
}
config/db.toml.online 0 → 100644
[Liexin_databases]
dns="huntdbslave:mLssy2@@!!@$#yy@tcp(172.18.137.22:3306)/hunt2016?parseTime=true"
[seo_databases]
dns="LsEoUx:s2ysTsmyzlmxzt@tcp(bigdata.ichunt.db:3306)/liexin_seo?parseTime=true"
[goods_databases]
dns="LxDDUsedRead:0o9u0U2oixoYmosflmxXtZmyt@tcp(172.18.137.33:3306)/liexin_data?parseTime=true"
[spu_databases]
dns="LxiCSpu:Mysx3Tyzlo00oxlmlly@tcp(spu-master.ichunt.db:3306)/liexin_spu?parseTime=true"
[sku_databases]
dns="LxiCSpuR:Mysx3Tyzlo00oxlmllyR@tcp(spu-slave.ichunt.db:3306)/liexin_sku_%d?parseTime=true"
[redis_config]
host="172.18.137.23"
password="icDb29mLy2s"
port="6379"
[rabbitmq_seo]
queue_name="liexin_seo"
routing_key="liexin_seo"
exchange=""
type="direct"
dns="amqp://SeoLxt:sl2PlqszuQp2@172.18.137.33:5672/"
dal/colentry/colentry.go 0 → 100644
package colentry
import (
"github.com/jmoiron/sqlx"
"golang-seo-server/model/liexin_seo/Urls"
)
/**
* @author zzl
* @description 联营实体ID
* @date 16:45 2020/11/16
* @param
* @return
**/
func LyEntryId(colCode string, row *sqlx.Rows) (entryId string) {
switch colCode {
case "ly_supplier":
var lySupplier Urls.LySupplier
_ = row.StructScan(&lySupplier)
return lySupplier.SupplierId
case "ly_brand":
var lyBrand Urls.LyBrand
_ = row.StructScan(&lyBrand)
return lyBrand.BrandId
case "ly_class":
var lyClass Urls.LyClass
_ = row.StructScan(&lyClass)
return lyClass.ClassId
}
return "NULL"
}
/**
* @author zzl
* @description //TODO
* @date 16:55 2020/11/17
* @param
* @return
**/
func MainEntryId(colCode string, row *sqlx.Rows) (entryId string) {
switch colCode {
case "article":
var article Urls.Article
_ = row.StructScan(&article)
return article.ArtId
case "self_class":
var zyClass Urls.ZyClass
_ = row.StructScan(&zyClass)
return zyClass.ClassId
case "self_detail":
var zyGoods Urls.ZyGoods
_ = row.StructScan(&zyGoods)
return zyGoods.GoodsId
case "ly_spu":
var lySpu Urls.LySpu
_ = row.StructScan(&lySpu)
return lySpu.SpuName
case "ly_detail":
var lySku Urls.LySku
_ = row.StructScan(&lySku)
return lySku.GoodsId
}
return "NULL"
}
/**
* @author zzl
* @description //TODO
* @date 17:40 2020/11/17
* @param
* @return
**/
func MipEntryId(colCode string, row *sqlx.Rows) (entryId string) {
switch colCode {
case "mip_ly_supplier":
var lySupplier Urls.LySupplier
_ = row.StructScan(&lySupplier)
return lySupplier.SupplierId
case "mip_ly_brand":
var lyBrand Urls.LyBrand
_ = row.StructScan(&lyBrand)
return lyBrand.BrandId
case "mip_ly_class":
var lyClass Urls.LyClass
_ = row.StructScan(&lyClass)
return lyClass.ClassId
case "mip_article":
var article Urls.Article
_ = row.StructScan(&article)
return article.ArtId
case "mip_self_class":
var zyClass Urls.ZyClass
_ = row.StructScan(&zyClass)
return zyClass.ClassId
case "mip_self_detail":
var zyGoods Urls.ZyGoods
_ = row.StructScan(&zyGoods)
return zyGoods.GoodsId
case "mip_ly_spu":
var lySpu Urls.LySpu
_ = row.StructScan(&lySpu)
return lySpu.SpuName
case "mip_ly_detail":
var lySku Urls.LySku
_ = row.StructScan(&lySku)
return lySku.GoodsId
}
return "NULL"
}
dal/colsql/colsql.go 0 → 100644
package colsql
import "fmt"
func LySQL(colCode string) (sql string) {
switch colCode {
case "ly_supplier":
sql = "SELECT supplier_id FROM lie_supplier WHERE type_id = 1 AND is_type = 0 AND status = 1"
return sql
case "ly_brand":
sql = "SELECT brand_id FROM lie_brand WHERE status = 1"
return sql
case "ly_class":
sql = "SELECT class_id FROM lie_classify WHERE status = 1"
return sql
}
return ""
}
func MainSQL(colCode string, index int, currPage int, pageSize int) (sql string) {
switch colCode {
case "article":
sql = "SELECT art_id FROM lie_article WHERE status = 1"
return sql
case "self_class":
sql = "SELECT class_id FROM lie_self_classify WHERE status = 1"
return sql
case "self_detail":
sql = fmt.Sprintf("SELECT goods_id FROM lie_goods WHERE status = 1 AND goods_type = 0 "+
" ORDER BY goods_id ASC LIMIT %d,%d",
currPage, pageSize)
return sql
case "ly_spu":
sql = fmt.Sprintf("SELECT spu_name FROM lie_spu_%d WHERE status=1 ORDER BY update_time ASC LIMIT %d,%d",
index, currPage, pageSize)
return sql
case "ly_detail":
sql = fmt.Sprintf("SELECT goods_id FROM lie_sku_%d ORDER BY update_time ASC LIMIT %d,%d",
index, currPage, pageSize)
return sql
}
return ""
}
func MipSQL(colCode string, index int, currPage int, pageSize int) (sql string) {
switch colCode {
case "mip_ly_supplier":
sql = "SELECT supplier_id FROM lie_supplier WHERE type_id = 1 AND is_type = 0 AND status = 1"
return sql
case "mip_ly_brand":
sql = "SELECT brand_id FROM lie_brand WHERE status = 1"
return sql
case "mip_ly_class":
sql = "SELECT class_id FROM lie_classify WHERE status = 1"
return sql
case "mip_article":
sql = "SELECT art_id FROM lie_article WHERE status = 1"
return sql
case "mip_self_class":
sql = "SELECT class_id FROM lie_self_classify WHERE status = 1"
return sql
case "mip_self_detail":
sql = fmt.Sprintf("SELECT goods_id FROM lie_goods WHERE status = 1 AND goods_type = 0 "+
" ORDER BY goods_id ASC LIMIT %d,%d",
currPage, pageSize)
return sql
case "mip_ly_spu":
sql = fmt.Sprintf("SELECT spu_name FROM lie_spu_%d WHERE status=1 ORDER BY update_time ASC LIMIT %d,%d",
index, currPage, pageSize)
return sql
case "mip_ly_detail":
sql = fmt.Sprintf("SELECT goods_id FROM lie_sku_%d ORDER BY update_time ASC LIMIT %d,%d",
index, currPage, pageSize)
return sql
}
return ""
}
dal/colurl/colurl.go 0 → 100644
package colurl
import (
"golang-seo-server/util"
)
func LyUrl(colCode string, configDir string) (sql string) {
util.Init(configDir)
switch colCode {
case "ly_supplier":
return util.Configs.Seo_domain.LySupplierUrl
case "ly_brand":
return util.Configs.Seo_domain.LyBrandUrl
case "ly_class":
return util.Configs.Seo_domain.LyClassUrl
}
return ""
}
func MainUrl(colCode string, configDir string) (sql string) {
util.Init(configDir)
switch colCode {
case "article":
return util.Configs.Seo_domain.ArticleUrl
case "self_class":
return util.Configs.Seo_domain.ZyClassUrl
case "self_detail":
return util.Configs.Seo_domain.ZyDetailUrl
case "ly_spu":
return util.Configs.Seo_domain.LySpuUrl
case "ly_detail":
return util.Configs.Seo_domain.LySkuUrl
}
return ""
}
func MipUrl(colCode string, configDir string) (sql string) {
util.Init(configDir)
switch colCode {
case "mip_ly_supplier":
return util.Configs.Seo_domain.MipLySupplierUrl
case "mip_ly_brand":
return util.Configs.Seo_domain.MipLyBrandUrl
case "mip_ly_class":
return util.Configs.Seo_domain.MipLyClassUrl
case "mip_article":
return util.Configs.Seo_domain.MipArticleUrl
case "mip_self_class":
return util.Configs.Seo_domain.MipZyClassUrl
case "mip_self_detail":
return util.Configs.Seo_domain.MipZyDetailUrl
case "mip_ly_spu":
return util.Configs.Seo_domain.MipLySpuUrl
case "mip_ly_detail":
return util.Configs.Seo_domain.MipLySkuUrl
}
return ""
}
dal/db/db.go 0 → 100644
package db
import (
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
)
var (
DB *sqlx.DB
SeoDB *sqlx.DB
GoodsDB *sqlx.DB
SpuDB *sqlx.DB
SkuDB *sqlx.DB
LxDB *sqlx.DB
)
/**
* @author zzl
* @description //TODO
* @date 16:38 2020/11/12
* @param action: 选择DB dns: dns
* @return
**/
func CreateDB(action string, dns string) error {
conn, err := sqlx.Open("mysql", dns)
if err != nil {
return err
}
err = conn.Ping()
if err != nil {
return err
}
conn.SetMaxOpenConns(50)
conn.SetMaxIdleConns(10)
switch action {
case "Init":
DB = conn
case "InitSeoDB":
SeoDB = conn
case "InitGoodsDB":
GoodsDB = conn
case "InitSpuDB":
SpuDB = conn
case "InitSkuDB":
SkuDB = conn
case "InitLxDB":
LxDB = conn
}
return nil
}
func GetDB(dbName string) *sqlx.DB {
switch dbName {
case "spu":
return SpuDB
case "goods":
return GoodsDB
case "lx":
return LxDB
case "sku":
return SkuDB
}
return nil
}
func InitSkuDB(dns string) error {
var err error
SkuDB, err = sqlx.Open("mysql", dns)
if err != nil {
return err
}
err = SkuDB.Ping()
if err != nil {
return err
}
SkuDB.SetMaxOpenConns(50)
SkuDB.SetMaxIdleConns(10)
return nil
}
dal/function/fuction.go 0 → 100644
package function
import (
"encoding/json"
"fmt"
"github.com/ichunt2019/golang-rbmq-sl/utils/rabbitmq"
"github.com/ichunt2019/logger"
"github.com/tidwall/gjson"
"golang-seo-server/dal/db"
"golang-seo-server/util"
"io/ioutil"
"net/http"
"regexp"
"strings"
"time"
)
// json 返回值
type JosnResp struct {
Errcode int `json:"err_code"`
Errmsg string `json:"err_msg"`
}
/**
* @author zzl
* @description HTTP POST
* @date 15:52 2020/11/13
* @param
* @return
**/
func HttpPost(url string, data string, header map[string]string) (str string) {
client := &http.Client{}
req, _ := http.NewRequest("POST", url, strings.NewReader(string(data)))
if len(header) > 0 {
for k, v := range header {
req.Header.Set(k, v)
}
}
resp, err := client.Do(req)
defer resp.Body.Close()
if err != nil {
logger.Info(err.Error())
}
body, _ := ioutil.ReadAll(resp.Body)
res := JosnResp{}
if err := json.Unmarshal([]byte(body), &res); err != nil {
json_res, _ := json.Marshal(res)
logger.Info(string(json_res)) // 记录body错误信息
}
if res.Errcode != 0 {
logger.Info(res.Errmsg) // 记录错误信息
}
return string(body)
}
/**
* @author zzl
* @description // 剩余条数
* @date 15:44 2020/11/13
* @param
* @return
**/
func RemainCount(url string, data string) (count int) {
var header = make(map[string]string)
header["Content-Type"] = "text/plain"
var body = HttpPost(url, data, header)
return int(gjson.Get(body, "remain").Int())
}
/**
* @author zzl
* @description // 更新采集
* @date 10:31 2020/11/17
* @param
* @return
**/
func UpdateCollect(collectId string, status int, db_count int, db_table string, db_conn string) {
_, err := db.SeoDB.Exec("UPDATE lie_collect SET status = ?,db_count = ?,db_table = ?,"+
"db_conn = ?,update_time = ? WHERE collect_id = ?",
status, db_count, db_table, db_conn, time.Now().Unix(), collectId)
if err != nil {
fmt.Println(err)
// TODO 钉钉
}
}
/**
* @author zzl
* @description
* @date 15:07 2020/11/17
* @param
* @return
**/
func RabbitSeoSend(body string) {
queueExchange := rabbitmq.QueueExchange{
util.Configs.Rabbitmq_seo.QueueName,
util.Configs.Rabbitmq_seo.RoutingKey,
util.Configs.Rabbitmq_seo.Exchange,
util.Configs.Rabbitmq_seo.Type,
util.Configs.Rabbitmq_seo.Dns,
}
rabbitmq.Send(queueExchange, body)
}
/**
* url解析
* @author zzl
* @description //TODO
* @date 17:35 2020/11/25
* @param
* @return
**/
func RegxUrl(url string) (res string) {
matched1, _ := regexp.MatchString("^(https://www.ichunt)", url)
matched2, _ := regexp.MatchString("^(https://ly.ichunt)", url)
matched3, _ := regexp.MatchString("^(https://mip.ichunt)", url)
if matched1 {
return "main"
} else if matched2 {
return "ly"
} else if matched3 {
return "mip"
}
return "main"
}
/**
* 写入日志
* @author zzl
* @description //TODO
* @date 11:40 2020/11/26
* @param
* @return
**/
func InsertLog(urls string, status string, url_type string) {
url := strings.Split(urls, "\n")
for i := 0; i < len(url); i++ {
_, err1 := db.SeoDB.Exec("INSERT INTO lie_url_log (url,status,url_type,create_time) VALUES (?,?,?,?)", url[i], status, url_type, time.Now().Unix())
if err1 != nil {
logger.Fatal("数据库写入失败 url %s, status %d", url, status)
}
}
}
go.mod 0 → 100644
module golang-seo-server
go 1.12
require (
github.com/BurntSushi/toml v0.3.1
github.com/go-sql-driver/mysql v1.4.1
github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d
github.com/ichunt2019/logger v1.0.5
github.com/jmoiron/sqlx v1.2.0
github.com/robfig/cron/v3 v3.0.1
github.com/tidwall/gjson v1.6.1
google.golang.org/appengine v1.6.7 // indirect
)
go.sum 0 → 100644
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/ichunt2019/go-rabbitmq v1.0.1/go.mod h1:TQsZ1XWULyvm4UwpYHwNPtOXYbuVvLLI0GM7g/BRy68=
github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d h1:wN4ay65hYnXyqn1BWc/WgCpPf+IYwYDRbLASYZyQrPw=
github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d/go.mod h1:sUQJFISXhgUd5EjkbKphSoxHiGL5BMkTQ/9rfT0lVCw=
github.com/ichunt2019/logger v1.0.5 h1:85C6kJCH9xlbLt1VmwHp/8iScm+bIlenK6nanWwwq/o=
github.com/ichunt2019/logger v1.0.5/go.mod h1:5IWMrrqJIWwOIGav9ACWOI+KOuYeteUvOei4zubclwg=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 h1:2MR0pKUzlP3SGgj5NYJe/zRYDwOu9ku6YHy+Iw7l5DM=
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
model/liexin_seo/Urls/Urls.go 0 → 100644
package Urls
type Collect struct {
CollectCode string `db:"collect_code"`
CollectId string `db:"collect_id"`
DbConn string `db:"db_conn"`
DbTable string `db:"db_table"`
DbCount string `db:"db_count"`
}
type Article struct {
ArtId string `db:"art_id"`
}
type ZyClass struct {
ClassId string `db:"class_id"`
}
type ZyGoods struct {
GoodsId string `db:"goods_id"`
}
type LySpu struct {
SpuName string `db:"spu_name"`
}
type LySku struct {
GoodsId string `db:"goods_id"`
}
type LySupplier struct {
SupplierId string `db:"supplier_id"`
}
type LyBrand struct {
BrandId string `db:"brand_id"`
}
type LyClass struct {
ClassId string `db:"class_id"`
}
readme.md 0 → 100644
采集任务
运行命令
```
go run ./cmd/main.go -configDir=./config/ -logDir=./log/
```
\ No newline at end of file
util/config.go 0 → 100644
package util
import (
"github.com/BurntSushi/toml"
)
//订制配置文件解析载体
type Config struct {
Liexin_databases *LiexinMysqlConfig
Sku_databases *SkuDatabasesMysqlConfig
Seo_databases *SeoDatabasesMysqlConfig
Goods_databases *GoodsDatabasesMysqlConfig
Spu_databases *SpuDatabasesMysqlConfig
Rabbitmq_seo *RabbitmqSeo
Rabbitmq_seo_log *RabbitmqSeoLog
Redis_config *RedisConn
Request_url *RequestUrl
Seo_domain *SeoDomain
Ding_msg *Ding
}
type LiexinMysqlConfig struct {
Dns string `toml:"dns"`
}
type SeoDatabasesMysqlConfig struct {
Dns string `toml:"dns"`
}
type GoodsDatabasesMysqlConfig struct {
Dns string `toml:"dns"`
}
type SpuDatabasesMysqlConfig struct {
Dns string `toml:"dns"`
}
type SkuDatabasesMysqlConfig struct {
Dns string `toml:"dns"`
}
type RabbitmqIchunt struct {
QueueName string `toml:"queue_name"`
RoutingKey string `toml:"routing_key"`
Exchange string `toml:"exchange"`
Type string `toml:"type"`
Dns string `toml:"dns"`
}
type RabbitmqSeo struct {
QueueName string `toml:"queue_name"`
RoutingKey string `toml:"routing_key"`
Exchange string `toml:"exchange"`
Type string `toml:"type"`
Dns string `toml:"dns"`
}
type RabbitmqSeoLog struct {
QueueName string `toml:"queue_name"`
RoutingKey string `toml:"routing_key"`
Exchange string `toml:"exchange"`
Type string `toml:"type"`
Dns string `toml:"dns"`
}
type RequestUrl struct {
MainUrl string `toml:"main_url"`
MainData string `toml:"main_data"`
LyUrl string `toml:"ly_url"`
LyData string `toml:"ly_data"`
MipUrl string `toml:"mip_url"`
MipData string `toml:"mip_data"`
}
type SeoDomain struct {
DetailUrl string `toml:"detail_url"`
ArticleUrl string `toml:"article_url"`
ZyClassUrl string `toml:"zy_class_url"`
ZyDetailUrl string `toml:"zy_detail_url"`
LySpuUrl string `toml:"ly_spu_url"`
LySkuUrl string `toml:"ly_sku_url"`
LySupplierUrl string `toml:"ly_supplier_url"`
LyBrandUrl string `toml:"ly_brand_url"`
LyClassUrl string `toml:"ly_class_url"`
MipArticleUrl string `toml:"mip_article_url"`
MipZyClassUrl string `toml:"mip_zy_class_url"`
MipZyDetailUrl string `toml:"mip_zy_detail_url"`
MipLySpuUrl string `toml:"mip_ly_spu_url"`
MipLySkuUrl string `toml:"mip_ly_sku_url"`
MipLySupplierUrl string `toml:"mip_ly_supplier_url"`
MipLyBrandUrl string `toml:"mip_ly_brand_url"`
MipLyClassUrl string `toml:"mip_ly_class_url"`
}
type RedisConn struct {
Host string `toml:"host"`
Password string `toml:"password"`
Port string `toml:"port"`
}
type Ding struct {
Webhook string `toml:"webhook"`
}
var Configs = new(Config)
func Init(ConfigDir string) {
//fmt.Println(ConfigDir+"config/config.toml")
var err error
_, err = toml.DecodeFile(ConfigDir+"config.toml", Configs)
_, err = toml.DecodeFile(ConfigDir+"db.toml", Configs)
if err != nil {
panic(err)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment