项目重构

.gitignore

+/.idea
+/.DS_Store
+/logs/*
+/log/*
+/config/*.toml

cmd/main.go

+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/ichunt2019/golang-rbmq-sl/utils/rabbitmq"
+	"github.com/ichunt2019/logger"
+	"golang-seo-server/common/ding"
+	"golang-seo-server/dal/db"
+	"golang-seo-server/push"
+	"golang-seo-server/util"
+)
+
+var ConfigDir string
+var LogDir string
+
+type RecvPro struct {
+}
+
+// 解析命令行参数
+func initArgs() {
+	flag.StringVar(&ConfigDir, "configDir", "", "配置文件")
+	flag.StringVar(&LogDir, "logDir", "", "日志目录")
+	flag.Parse()
+}
+
+// 初始化DB
+func initDb(action string, dns string) (err error) {
+	err = db.CreateDB(action, dns)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// 实现消费者 消费消息失败 自动进入延时尝试  尝试3次之后入库db
+func (t *RecvPro) Consumer(dataByte []byte) error {
+	fmt.Println(string(dataByte))
+	push.Push(string(dataByte))
+	return nil
+}
+
+//消息已经消费3次 失败了 请进行处理
+func (t *RecvPro) FailAction(err error, dataByte []byte) error {
+	logger.Error("任务处理失败了，发送钉钉消息")
+	logger.Error(string(dataByte))
+	logger.Error("错误原因：%s", err)
+	ding.Send(
+		util.Configs.Ding_msg.Webhook,
+		"SEO推送",
+		fmt.Sprintf("队列seo 失败原因:%s \n %s", err, string(dataByte)),
+		make([]string, 0),
+		false,
+	)
+	return nil
+}
+
+func main() {
+	initArgs()
+	util.Init(ConfigDir)
+
+	// log配置
+	logConfig := make(map[string]string)
+	logConfig["log_path"] = LogDir + "seo"
+	logConfig["log_chan_size"] = "1000"
+	_ = logger.InitLogger("file", logConfig)
+	logger.Init()
+
+	t := &RecvPro{}
+	// 队列发送消息
+	rabbitmq.Recv(rabbitmq.QueueExchange{
+		util.Configs.Rabbitmq_seo.QueueName,
+		util.Configs.Rabbitmq_seo.RoutingKey,
+		util.Configs.Rabbitmq_seo.Exchange,
+		util.Configs.Rabbitmq_seo.Type,
+		util.Configs.Rabbitmq_seo.Dns,
+	}, t, 3)
+}

common/ding/ding.go

+package ding
+
+import (
+	"encoding/json"
+	_ "fmt"
+	"github.com/ichunt2019/logger"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+type Msg struct {
+	Msgtype string   `json:"msgtype"`
+	Text    Contents `json:"text"`
+	At      Ats      `json:"at"`
+}
+
+type Contents struct {
+	Content string `json:"content"`
+}
+
+type Ats struct {
+	AtMobiles []string `json:"atMobiles"`
+	IsAtAll   bool     `json:"isAtAll"`
+}
+
+// json 返回值
+type JosnResp struct {
+	Errcode int    `json:"errcode"`
+	Errmsg  string `json:"errmsg"`
+}
+
+func Send(ding_url string, ding_tag string, textMsg string, mobiles []string, isAtAll bool) (jsonStr string) {
+	var msg Msg
+	msg = Msg{Msgtype: "text"}
+	msg.Text.Content = ding_tag + "：" + textMsg // 固定标签 + 文本
+
+	msg.At.AtMobiles = mobiles
+	msg.At.IsAtAll = isAtAll
+
+	content, _ := json.Marshal(msg)
+
+	// content := `{
+	// 	"msgtype": "text",
+	// 	"text": {
+	// 		"content": "`+ msg + `"
+	// 	}
+	// }`
+
+	client := &http.Client{}
+
+	req, _ := http.NewRequest("POST", ding_url, strings.NewReader(string(content)))
+
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	resp, err := client.Do(req)
+
+	defer resp.Body.Close()
+
+	if err != nil {
+		logger.Info(err.Error())
+	}
+
+	body, _ := ioutil.ReadAll(resp.Body) // 获取接口返回数据
+
+	res := JosnResp{}
+
+	if err1 := json.Unmarshal([]byte(body), &res); err1 != nil { // 将接口数据写入res
+		logger.Info(err1.Error())
+	}
+
+	result, _ := json.Marshal(res)
+
+	return string(result)
+}

dal/db/db.go

+package db
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/jmoiron/sqlx"
+)
+
+var (
+	DB      *sqlx.DB
+	SeoDB   *sqlx.DB
+	GoodsDB *sqlx.DB
+	SpuDB   *sqlx.DB
+	SkuDB   *sqlx.DB
+	LxDB    *sqlx.DB
+)
+
+/**
+ * @author zzl
+ * @description //TODO
+ * @date 16:38 2020/11/12
+ * @param action: 选择DB dns: dns
+ * @return
+ **/
+func CreateDB(action string, dns string) error {
+	conn, err := sqlx.Open("mysql", dns)
+
+	if err != nil {
+		return err
+	}
+	err = conn.Ping()
+	if err != nil {
+		return err
+	}
+	conn.SetMaxOpenConns(50)
+	conn.SetMaxIdleConns(10)
+
+	switch action {
+	case "Init":
+		DB = conn
+	case "InitSeoDB":
+		SeoDB = conn
+	case "InitGoodsDB":
+		GoodsDB = conn
+	case "InitSpuDB":
+		SpuDB = conn
+	case "InitSkuDB":
+		SkuDB = conn
+	case "InitLxDB":
+		LxDB = conn
+	}
+
+	return nil
+}
+
+func GetDB(dbName string) *sqlx.DB {
+	switch dbName {
+	case "spu":
+		return SpuDB
+	case "goods":
+		return GoodsDB
+	case "lx":
+		return LxDB
+	case "sku":
+		return SkuDB
+	}
+
+	return nil
+}
+
+func InitSkuDB(dns string) error {
+	var err error
+	SkuDB, err = sqlx.Open("mysql", dns)
+	if err != nil {
+		return err
+	}
+
+	err = SkuDB.Ping()
+	if err != nil {
+		return err
+	}
+	SkuDB.SetMaxOpenConns(50)
+	SkuDB.SetMaxIdleConns(10)
+	return nil
+}

dal/function/fuction.go

+package function
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/ichunt2019/golang-rbmq-sl/utils/rabbitmq"
+	"github.com/ichunt2019/logger"
+	"github.com/tidwall/gjson"
+	"golang-seo-server/dal/db"
+	"golang-seo-server/util"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// json 返回值
+type JosnResp struct {
+	Errcode int    `json:"err_code"`
+	Errmsg  string `json:"err_msg"`
+}
+
+/**
+ * @author zzl
+ * @description HTTP POST
+ * @date 15:52 2020/11/13
+ * @param
+ * @return
+ **/
+func HttpPost(url string, data string, header map[string]string) (str string) {
+	client := &http.Client{}
+
+	req, _ := http.NewRequest("POST", url, strings.NewReader(string(data)))
+
+	if len(header) > 0 {
+		for k, v := range header {
+			req.Header.Set(k, v)
+		}
+	}
+
+	resp, err := client.Do(req)
+
+	defer resp.Body.Close()
+
+	if err != nil {
+		logger.Info(err.Error())
+	}
+
+	body, _ := ioutil.ReadAll(resp.Body)
+
+	res := JosnResp{}
+
+	if err := json.Unmarshal([]byte(body), &res); err != nil {
+		json_res, _ := json.Marshal(res)
+		logger.Info(string(json_res)) // 记录body错误信息
+	}
+
+	if res.Errcode != 0 {
+		logger.Info(res.Errmsg) // 记录错误信息
+	}
+
+	return string(body)
+}
+
+/**
+ * @author zzl
+ * @description // 剩余条数
+ * @date 15:44 2020/11/13
+ * @param
+ * @return
+ **/
+func RemainCount(url string, data string) (count int) {
+	var header = make(map[string]string)
+	header["Content-Type"] = "text/plain"
+	var body = HttpPost(url, data, header)
+	return int(gjson.Get(body, "remain").Int())
+}
+
+/**
+ * @author zzl
+ * @description // 更新采集
+ * @date 10:31 2020/11/17
+ * @param
+ * @return
+ **/
+func UpdateCollect(collectId string, status int, db_count int, db_table string, db_conn string) {
+	_, err := db.SeoDB.Exec("UPDATE lie_collect SET status = ?,db_count = ?,db_table = ?,"+
+		"db_conn = ?,update_time = ? WHERE collect_id = ?",
+		status, db_count, db_table, db_conn, time.Now().Unix(), collectId)
+
+	if err != nil {
+		fmt.Println(err)
+		// TODO 钉钉
+	}
+}
+
+/**
+ * 推送SEO数据
+ * @author zzl
+ * @description
+ * @date 15:07 2020/11/17
+ * @param
+ * @return
+ **/
+func RabbitSeoSend(body string) {
+	queueExchange := rabbitmq.QueueExchange{
+		util.Configs.Rabbitmq_seo.QueueName,
+		util.Configs.Rabbitmq_seo.RoutingKey,
+		util.Configs.Rabbitmq_seo.Exchange,
+		util.Configs.Rabbitmq_seo.Type,
+		util.Configs.Rabbitmq_seo.Dns,
+	}
+
+	rabbitmq.Send(queueExchange, body)
+}
+
+/**
+ * 推送SEO LOG
+ * @author zzl
+ * @description
+ * @date 15:07 2020/11/17
+ * @param
+ * @return
+ **/
+func RabbitSeoLogSend(body string) {
+	queueExchange := rabbitmq.QueueExchange{
+		util.Configs.Rabbitmq_seo_log.QueueName,
+		util.Configs.Rabbitmq_seo_log.RoutingKey,
+		util.Configs.Rabbitmq_seo_log.Exchange,
+		util.Configs.Rabbitmq_seo_log.Type,
+		util.Configs.Rabbitmq_seo_log.Dns,
+	}
+
+	rabbitmq.Send(queueExchange, body)
+}
+
+/**
+ * url解析
+ * @author zzl
+ * @description //TODO
+ * @date 17:35 2020/11/25
+ * @param
+ * @return
+ **/
+func RegxUrl(url string) (res string) {
+	matched1, _ := regexp.MatchString("^(https://www.ichunt)", url)
+	matched2, _ := regexp.MatchString("^(https://ly.ichunt)", url)
+	matched3, _ := regexp.MatchString("^(https://mip.ichunt)", url)
+
+	if matched1 {
+		return "main"
+	} else if matched2 {
+		return "ly"
+	} else if matched3 {
+		return "mip"
+	}
+
+	return "main"
+}

go.mod

+module golang-seo-server
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d
+	github.com/ichunt2019/logger v1.0.5
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/robfig/cron/v3 v3.0.1
+	github.com/tidwall/gjson v1.6.1
+	google.golang.org/appengine v1.6.7 // indirect
+)

go.sum

+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/ichunt2019/go-rabbitmq v1.0.1/go.mod h1:TQsZ1XWULyvm4UwpYHwNPtOXYbuVvLLI0GM7g/BRy68=
+github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d h1:wN4ay65hYnXyqn1BWc/WgCpPf+IYwYDRbLASYZyQrPw=
+github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d/go.mod h1:sUQJFISXhgUd5EjkbKphSoxHiGL5BMkTQ/9rfT0lVCw=
+github.com/ichunt2019/logger v1.0.5 h1:85C6kJCH9xlbLt1VmwHp/8iScm+bIlenK6nanWwwq/o=
+github.com/ichunt2019/logger v1.0.5/go.mod h1:5IWMrrqJIWwOIGav9ACWOI+KOuYeteUvOei4zubclwg=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 h1:2MR0pKUzlP3SGgj5NYJe/zRYDwOu9ku6YHy+Iw7l5DM=
+github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
+github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
+github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

model/liexin_seo/Urls/Urls.go

+package Urls
+
+type Collect struct {
+	CollectCode string `db:"collect_code"`
+	CollectId   string `db:"collect_id"`
+	DbConn      string `db:"db_conn"`
+	DbTable     string `db:"db_table"`
+	DbCount     string `db:"db_count"`
+}
+
+type Article struct {
+	ArtId string `db:"art_id"`
+}
+
+type ZyClass struct {
+	ClassId string `db:"class_id"`
+}
+
+type ZyGoods struct {
+	GoodsId string `db:"goods_id"`
+}
+
+type LySpu struct {
+	SpuName string `db:"spu_name"`
+}
+
+type LySku struct {
+	GoodsId string `db:"goods_id"`
+}
+
+type LySupplier struct {
+	SupplierId string `db:"supplier_id"`
+}
+
+type LyBrand struct {
+	BrandId string `db:"brand_id"`
+}
+
+type LyClass struct {
+	ClassId string `db:"class_id"`
+}

push/push.go

+package push
+
+import (
+	"fmt"
+	"github.com/ichunt2019/logger"
+	"github.com/tidwall/gjson"
+	"golang-seo-server/common/ding"
+	"golang-seo-server/dal/function"
+	"golang-seo-server/util"
+	"strconv"
+)
+
+var header = make(map[string]string)
+var mobile []string
+var isAtAll bool
+
+func Push(url string) {
+	header["Content-Type"] = "text/plain"
+	var postUrl string
+	var postData = url
+	var regxType = function.RegxUrl(postData)
+	if regxType == "main" {
+		postUrl = util.Configs.Request_url.MainUrl
+	} else if regxType == "ly" {
+		postUrl = util.Configs.Request_url.LyUrl
+	} else if regxType == "mip" {
+		postUrl = util.Configs.Request_url.MipUrl
+	}
+
+	res := function.HttpPost(postUrl, postData, header)
+	remain := gjson.Get(res, "remain").Int()
+	success := gjson.Get(res, "success").Int()
+	logger.Info("写入数据 remain %d - success %d - regx_type %s", remain, int(success), regxType)
+
+	var pushJson = fmt.Sprintf("{\"url\":\"%s\",\"status\":\"%d\"}", postData, int(success))
+
+	if success != 0 {
+		function.RabbitSeoLogSend(pushJson)
+		return
+	} else {
+		var retryRes string
+		var retrySuccess int
+		for i := 0; i < 3; i++ {
+			retryRes = function.HttpPost(postUrl, postData, header)
+			retrySuccess = int(gjson.Get(retryRes, "success").Int())
+			if retrySuccess != 0 {
+				function.RabbitSeoLogSend(pushJson)
+				break
+			}
+		}
+		logger.Info("写入数据 remain %d - success %d - regx_type %s", remain, retrySuccess, regxType)
+		ding.Send(util.Configs.Ding_msg.Webhook, "SEO推送",
+			"接口消费失败，数据："+retryRes+" 状态："+strconv.Itoa(retrySuccess)+" URL: "+postUrl,
+			mobile, isAtAll)
+	}
+}

readme.md

+采集任务
+
+运行命令
+
+```
+go run ./cmd/main.go -configDir=./config/ -logDir=./log/
+```
\ No newline at end of file

util/config.go

+package util
+
+import (
+	"github.com/BurntSushi/toml"
+)
+
+//订制配置文件解析载体
+type Config struct {
+	Liexin_databases *LiexinMysqlConfig
+	Sku_databases    *SkuDatabasesMysqlConfig
+	Seo_databases    *SeoDatabasesMysqlConfig
+	Goods_databases  *GoodsDatabasesMysqlConfig
+	Spu_databases    *SpuDatabasesMysqlConfig
+	Rabbitmq_seo     *RabbitmqSeo
+	Rabbitmq_seo_log *RabbitmqSeoLog
+	Redis_config     *RedisConn
+	Request_url      *RequestUrl
+	Seo_domain       *SeoDomain
+	Ding_msg         *Ding
+}
+
+type LiexinMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type SeoDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type GoodsDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type SpuDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type SkuDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type RabbitmqIchunt struct {
+	QueueName  string `toml:"queue_name"`
+	RoutingKey string `toml:"routing_key"`
+	Exchange   string `toml:"exchange"`
+	Type       string `toml:"type"`
+	Dns        string `toml:"dns"`
+}
+
+type RabbitmqSeo struct {
+	QueueName  string `toml:"queue_name"`
+	RoutingKey string `toml:"routing_key"`
+	Exchange   string `toml:"exchange"`
+	Type       string `toml:"type"`
+	Dns        string `toml:"dns"`
+}
+
+type RabbitmqSeoLog struct {
+	QueueName  string `toml:"queue_name"`
+	RoutingKey string `toml:"routing_key"`
+	Exchange   string `toml:"exchange"`
+	Type       string `toml:"type"`
+	Dns        string `toml:"dns"`
+}
+
+type RequestUrl struct {
+	MainUrl  string `toml:"main_url"`
+	MainData string `toml:"main_data"`
+	LyUrl    string `toml:"ly_url"`
+	LyData   string `toml:"ly_data"`
+	MipUrl   string `toml:"mip_url"`
+	MipData  string `toml:"mip_data"`
+}
+
+type SeoDomain struct {
+	DetailUrl        string `toml:"detail_url"`
+	ArticleUrl       string `toml:"article_url"`
+	ZyClassUrl       string `toml:"zy_class_url"`
+	ZyDetailUrl      string `toml:"zy_detail_url"`
+	LySpuUrl         string `toml:"ly_spu_url"`
+	LySkuUrl         string `toml:"ly_sku_url"`
+	LySupplierUrl    string `toml:"ly_supplier_url"`
+	LyBrandUrl       string `toml:"ly_brand_url"`
+	LyClassUrl       string `toml:"ly_class_url"`
+	MiPArticleUrl    string `toml:"mip_article_url"`
+	MiPZyClassUrl    string `toml:"mip_zy_class_url"`
+	MipZyDetailUrl   string `toml:"mip_zy_detail_url"`
+	MipLySpuUrl      string `toml:"mip_ly_spu_url"`
+	MipLySkuUrl      string `toml:"mip_ly_sku_url"`
+	MipLySupplierUrl string `toml:"mip_ly_supplier_url"`
+	MipLyBrandUrl    string `toml:"mip_ly_brand_url"`
+	MipLyClassUrl    string `toml:"mip_ly_class_url"`
+}
+
+type RedisConn struct {
+	Host     string `toml:"host"`
+	Password string `toml:"password"`
+	Port     string `toml:"port"`
+}
+
+type Ding struct {
+	Webhook string `toml:"webhook"`
+}
+
+var Configs = new(Config)
+
+func Init(ConfigDir string) {
+	//fmt.Println(ConfigDir+"config/config.toml")
+	var err error
+	_, err = toml.DecodeFile(ConfigDir+"config.toml", Configs)
+	_, err = toml.DecodeFile(ConfigDir+"db.toml", Configs)
+	if err != nil {
+		panic(err)
+	}
+}