Initial commit

.gitignore

+/.idea
+/.DS_Store
+/logs/*
+/log/*
+/config/*.toml

cmd/main.go

+package cmd

common/ding/ding.go

+package ding
+
+import (
+	"encoding/json"
+	_ "fmt"
+	"github.com/ichunt2019/logger"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+type Msg struct {
+	Msgtype string   `json:"msgtype"`
+	Text    Contents `json:"text"`
+	At      Ats      `json:"at"`
+}
+
+type Contents struct {
+	Content string `json:"content"`
+}
+
+type Ats struct {
+	AtMobiles []string `json:"atMobiles"`
+	IsAtAll   bool     `json:"isAtAll"`
+}
+
+// json 返回值
+type JosnResp struct {
+	Errcode int    `json:"errcode"`
+	Errmsg  string `json:"errmsg"`
+}
+
+func Send(ding_url string, ding_tag string, textMsg string, mobiles []string, isAtAll bool) (jsonStr string) {
+	var msg Msg
+	msg = Msg{Msgtype: "text"}
+	msg.Text.Content = ding_tag + "：" + textMsg // 固定标签 + 文本
+
+	msg.At.AtMobiles = mobiles
+	msg.At.IsAtAll = isAtAll
+
+	content, _ := json.Marshal(msg)
+
+	// content := `{
+	// 	"msgtype": "text",
+	// 	"text": {
+	// 		"content": "`+ msg + `"
+	// 	}
+	// }`
+
+	client := &http.Client{}
+
+	req, _ := http.NewRequest("POST", ding_url, strings.NewReader(string(content)))
+
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	resp, err := client.Do(req)
+
+	defer resp.Body.Close()
+
+	if err != nil {
+		logger.Info(err.Error())
+	}
+
+	body, _ := ioutil.ReadAll(resp.Body) // 获取接口返回数据
+
+	res := JosnResp{}
+
+	if err1 := json.Unmarshal([]byte(body), &res); err1 != nil { // 将接口数据写入res
+		logger.Info(err1.Error())
+	}
+
+	result, _ := json.Marshal(res)
+
+	return string(result)
+}

dal/db/db.go

+package db
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/jmoiron/sqlx"
+)
+
+var (
+	DB      *sqlx.DB
+	SeoDB   *sqlx.DB
+	GoodsDB *sqlx.DB
+	SpuDB   *sqlx.DB
+	SkuDB   *sqlx.DB
+	LxDB    *sqlx.DB
+)
+
+/**
+ * @author zzl
+ * @description //TODO
+ * @date 16:38 2020/11/12
+ * @param action: 选择DB dns: dns
+ * @return 
+ **/ 
+func CreateDB(action string, dns string) error {
+	conn, err :=sqlx.Open("mysql", dns)
+
+	if err != nil {
+		return err
+	}
+	err = conn.Ping()
+	if err != nil {
+		return err
+	}
+	conn.SetMaxOpenConns(50)
+	conn.SetMaxIdleConns(10)
+
+	switch action {
+		case "Init":
+			DB = conn
+		case "InitSeoDB":
+			SeoDB = conn
+		case "InitGoodsDB":
+			GoodsDB = conn
+		case "InitSpuDB":
+			SpuDB = conn
+		case "InitSkuDB":
+			SkuDB = conn
+		case "InitLxDB":
+			LxDB = conn
+	}
+
+	return nil
+}
+
+func GetDB(dbName string) (*sqlx.DB) {
+	switch dbName {
+		case "spu":
+			return SpuDB
+		case "goods":
+			return GoodsDB
+		case "lx":
+			return LxDB
+		case "sku":
+			return SkuDB
+	}
+
+	return nil
+}
+
+func InitSkuDB(dns string) error {
+	var err error
+	SkuDB, err = sqlx.Open("mysql", dns)
+	if err != nil {
+		return err
+	}
+
+	err = SkuDB.Ping()
+	if err != nil {
+		return err
+	}
+	SkuDB.SetMaxOpenConns(50)
+	SkuDB.SetMaxIdleConns(10)
+	return nil
+}
+

dal/function/fuction.go

+package function
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/ichunt2019/golang-rbmq-sl/utils/rabbitmq"
+	"github.com/ichunt2019/logger"
+	"github.com/tidwall/gjson"
+	"golang-seo-server/dal/db"
+	"golang-seo-server/util"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// json 返回值
+type JosnResp struct {
+	Errcode int    `json:"err_code"`
+	Errmsg  string `json:"err_msg"`
+}
+
+/**
+ * @author zzl
+ * @description HTTP POST
+ * @date 15:52 2020/11/13
+ * @param 
+ * @return 
+ **/ 
+func HttpPost(url string, data string, header map[string]string) (str string) {
+	client := &http.Client{}
+
+	req, _ := http.NewRequest("POST", url, strings.NewReader(string(data)))
+
+	if len(header) > 0 {
+		for k, v := range header {
+			req.Header.Set(k, v)
+		}
+	}
+
+	resp, err := client.Do(req)
+
+	defer resp.Body.Close()
+
+	if err != nil {
+		logger.Info(err.Error())
+	}
+
+	body, _ := ioutil.ReadAll(resp.Body)
+
+	res := JosnResp{}
+
+	if err := json.Unmarshal([]byte(body), &res); err != nil {
+		json_res, _ := json.Marshal(res)
+		logger.Info(string(json_res)) // 记录body错误信息
+	}
+
+	if res.Errcode != 0 {
+		logger.Info(res.Errmsg) // 记录错误信息
+	}
+
+	return string(body)
+}
+
+/**
+ * @author zzl
+ * @description // 剩余条数
+ * @date 15:44 2020/11/13
+ * @param
+ * @return
+ **/
+func RemainCount(url string, data string) (count int) {
+	var header = make(map[string]string)
+	header["Content-Type"] = "text/plain"
+	var body = HttpPost(url, data, header)
+	return int(gjson.Get(body, "remain").Int())
+}
+
+/**
+ * @author zzl
+ * @description // 更新采集
+ * @date 10:31 2020/11/17
+ * @param 
+ * @return 
+ **/
+func UpdateCollect(collectId string, status int, db_count int, db_table string, db_conn string)  {
+	_, err := db.SeoDB.Exec("UPDATE lie_collect SET status = ?,db_count = ?,db_table = ?," +
+		"db_conn = ?,update_time = ? WHERE collect_id = ?",
+		status, db_count, db_table, db_conn, time.Now().Unix(), collectId)
+
+	if err != nil {
+		fmt.Println(err)
+		// TODO 钉钉
+	}
+}
+
+/**
+ * @author zzl
+ * @description
+ * @date 15:07 2020/11/17
+ * @param 
+ * @return 
+ **/ 
+func RabbitSeoSend(body string) {
+	queueExchange := rabbitmq.QueueExchange{
+		util.Configs.Rabbitmq_seo.QueueName,
+		util.Configs.Rabbitmq_seo.RoutingKey,
+		util.Configs.Rabbitmq_seo.Exchange,
+		util.Configs.Rabbitmq_seo.Type,
+		util.Configs.Rabbitmq_seo.Dns,
+	}
+
+	rabbitmq.Send(queueExchange, body)
+}
+
+/**
+ * url解析
+ * @author zzl
+ * @description //TODO
+ * @date 17:35 2020/11/25
+ * @param
+ * @return
+ **/
+func RegxUrl(url string) (res string) {
+	matched1, _ := regexp.MatchString("^(https://www.ichunt)", url)
+	matched2, _ := regexp.MatchString("^(https://ly.ichunt)", url)
+	matched3, _ := regexp.MatchString("^(https://mip.ichunt)", url)
+
+	if matched1 {
+		return "main"
+	} else if matched2 {
+		return "ly"
+	} else if matched3 {
+		return "mip"
+	}
+
+	return "main"
+}
+
+/**
+ * 写入日志
+ * @author zzl
+ * @description //TODO
+ * @date 11:40 2020/11/26
+ * @param
+ * @return
+ **/
+func InsertLog(urls string, status string, url_type string) {
+	url := strings.Split(urls, "\n")
+	for i := 0; i < len(url); i++ {
+		_, err1 := db.SeoDB.Exec("INSERT INTO lie_url_log (url,status,url_type,create_time) VALUES (?,?,?,?)", url[i], status, url_type, time.Now().Unix())
+
+		if err1 != nil {
+			logger.Fatal("数据库写入失败 url %s, status %d", url, status)
+		}
+	}
+}
+

go.mod

+module golang-seo-server
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d
+	github.com/ichunt2019/logger v1.0.5
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/robfig/cron/v3 v3.0.1
+	github.com/tidwall/gjson v1.6.1
+	google.golang.org/appengine v1.6.7 // indirect
+)

go.sum

+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/ichunt2019/go-rabbitmq v1.0.1/go.mod h1:TQsZ1XWULyvm4UwpYHwNPtOXYbuVvLLI0GM7g/BRy68=
+github.com/ichunt2019/golang-rbmq-sl v0.0.0-20200515075131-59a37ab77d7d/go.mod h1:sUQJFISXhgUd5EjkbKphSoxHiGL5BMkTQ/9rfT0lVCw=
+github.com/ichunt2019/logger v1.0.5/go.mod h1:5IWMrrqJIWwOIGav9ACWOI+KOuYeteUvOei4zubclwg=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

model/liexin_seo/Urls/Urls.go

+package Urls
+
+type Collect struct {
+	CollectCode string `db:"collect_code"`
+	CollectId   string `db:"collect_id"`
+	DbConn      string `db:"db_conn"`
+	DbTable     string `db:"db_table"`
+	DbCount     string `db:"db_count"`
+}
+
+type Article struct {
+	ArtId string `db:"art_id"`
+}
+
+type ZyClass struct {
+	ClassId string `db:"class_id"`
+}
+
+type ZyGoods struct {
+	GoodsId string `db:"goods_id"`
+}
+
+type LySpu struct {
+	SpuName string `db:"spu_name"`
+}
+
+type LySku struct {
+	GoodsId string `db:"goods_id"`
+}
+
+type LySupplier struct {
+	SupplierId string `db:"supplier_id"`
+}
+
+type LyBrand struct {
+	BrandId string `db:"brand_id"`
+}
+
+type LyClass struct {
+	ClassId string `db:"class_id"`
+}

readme.md

+运行命令
+
+```
+go run ./cmd/main.go -configDir=./config/ -logDir=./log/
+```
\ No newline at end of file

util/config.go

+package util
+
+import (
+	"github.com/BurntSushi/toml"
+)
+
+//订制配置文件解析载体
+type Config struct {
+	Liexin_databases          *LiexinMysqlConfig
+	Sku_databases             *SkuDatabasesMysqlConfig
+	Seo_databases             *SeoDatabasesMysqlConfig
+	Goods_databases           *GoodsDatabasesMysqlConfig
+	Spu_databases             *SpuDatabasesMysqlConfig
+	Rabbitmq_seo              *RabbitmqSeo
+	Rabbitmq_seo_log          *RabbitmqSeoLog
+	Redis_config              *RedisConn
+	Request_url               *RequestUrl
+	Seo_domain                *SeoDomain
+	Ding_msg                  *Ding
+}
+
+type LiexinMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type SeoDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type GoodsDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type SpuDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type SkuDatabasesMysqlConfig struct {
+	Dns string `toml:"dns"`
+}
+
+type RabbitmqIchunt struct {
+	QueueName  string `toml:"queue_name"`
+	RoutingKey string `toml:"routing_key"`
+	Exchange   string `toml:"exchange"`
+	Type       string `toml:"type"`
+	Dns        string `toml:"dns"`
+}
+
+type RabbitmqSeo struct {
+	QueueName  string `toml:"queue_name"`
+	RoutingKey string `toml:"routing_key"`
+	Exchange   string `toml:"exchange"`
+	Type       string `toml:"type"`
+	Dns        string `toml:"dns"`
+}
+
+type RabbitmqSeoLog struct {
+	QueueName  string `toml:"queue_name"`
+	RoutingKey string `toml:"routing_key"`
+	Exchange   string `toml:"exchange"`
+	Type       string `toml:"type"`
+	Dns        string `toml:"dns"`
+}
+
+type RequestUrl struct {
+	MainUrl  string `toml:"main_url"`
+	MainData string `toml:"main_data"`
+	LyUrl    string `toml:"ly_url"`
+	LyData   string `toml:"ly_data"`
+	MipUrl   string `toml:"mip_url"`
+	MipData  string `toml:"mip_data"`
+}
+
+type SeoDomain struct {
+	DetailUrl        string `toml:"detail_url"`
+	ArticleUrl       string `toml:"article_url"`
+	ZyClassUrl       string `toml:"zy_class_url"`
+	ZyDetailUrl      string `toml:"zy_detail_url"`
+	LySpuUrl         string `toml:"ly_spu_url"`
+	LySkuUrl         string `toml:"ly_sku_url"`
+	LySupplierUrl    string `toml:"ly_supplier_url"`
+	LyBrandUrl       string `toml:"ly_brand_url"`
+	LyClassUrl       string `toml:"ly_class_url"`
+	MiPArticleUrl    string `toml:"mip_article_url"`
+	MiPZyClassUrl    string `toml:"mip_zy_class_url"`
+	MipZyDetailUrl   string `toml:"mip_zy_detail_url"`
+	MipLySpuUrl      string `toml:"mip_ly_spu_url"`
+	MipLySkuUrl      string `toml:"mip_ly_sku_url"`
+	MipLySupplierUrl string `toml:"mip_ly_supplier_url"`
+	MipLyBrandUrl    string `toml:"mip_ly_brand_url"`
+	MipLyClassUrl    string `toml:"mip_ly_class_url"`
+}
+
+type RedisConn struct {
+	Host     string `toml:"host"`
+	Password string `toml:"password"`
+	Port     string `toml:"port"`
+}
+
+type Ding struct {
+	Webhook  string `toml:"webhook"`
+}
+
+var Configs = new(Config)
+
+func Init(ConfigDir string) {
+	//fmt.Println(ConfigDir+"config/config.toml")
+	var err error
+	_, err = toml.DecodeFile(ConfigDir + "config.toml", Configs)
+	_, err = toml.DecodeFile(ConfigDir + "db.toml", Configs)
+	if err != nil {
+		panic(err)
+	}
+}
\ No newline at end of file