fix(mysql): mysql-monitor-start-failed #10156

Author: xfwduke

dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/v2/crond/check_start.go

@@ -46,6 +46,13 @@ func (c *MySQLCrondComp) CheckStart() (err error) {
 			}
 		}
 
+		/*
+			这里算是偷懒了, 现在 mysql-crond 启动
+			在调用 third-party 的时候有个最大 10s 的超时
+			所以即使进程存在, 也暴力等 15s
+		*/
+		time.Sleep(15 * time.Second)
+
 		return nil // 如果能走到这里, 说明正常启动了
 	}
 

dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/v2/monitor/init.go

@@ -3,6 +3,7 @@ package monitor
 import (
 	"dbm-services/mysql/db-tools/dbactuator/pkg/components"
 	"dbm-services/mysql/db-tools/dbactuator/pkg/tools"
+	ma "dbm-services/mysql/db-tools/mysql-crond/api"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
 )
 
@@ -15,6 +16,9 @@ type MySQLMonitorComp struct {
 func (c *MySQLMonitorComp) Init() (err error) {
 	c.tools = tools.NewToolSetWithPickNoValidate(tools.ToolMySQLMonitor)
 
+	manager := ma.NewManager("http://127.0.0.1:9999")
+	_ = manager.UpdateInstanceInfo()
+
 	return nil
 }
 

dbm-services/mysql/db-tools/mysql-crond/api/update_instance_info.go

@@ -0,0 +1,35 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+func (m *Manager) UpdateInstanceInfo() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer func() {
+		cancel()
+	}()
+
+	errChan := make(chan error, 1)
+	go func() {
+		_, err := m.do("/third-party/update-instance-info", http.MethodGet, nil)
+		errChan <- err
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case err := <-errChan:
+			if err != nil {
+				return errors.Wrapf(err, "manager call %s", m.apiUrl)
+			}
+			return nil
+		default:
+		}
+	}
+}

dbm-services/mysql/db-tools/mysql-crond/cmd/root.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"dbm-services/mysql/db-tools/mysql-crond/pkg/third_party"
 	"fmt"
 	"io"
 	"log/slog"
@@ -57,6 +58,13 @@ var rootCmd = &cobra.Command{
 			return err
 		}
 
+		slog.Info("start crond start third party update")
+		err = third_party.Updater()
+		if err != nil {
+			slog.Error("start crond", slog.String("error", err.Error()))
+		}
+		slog.Info("start crond init third party update finish")
+
 		quit := make(chan struct{})
 
 		go func() {

dbm-services/mysql/db-tools/mysql-crond/pkg/service/service.go

@@ -2,6 +2,7 @@
 package service
 
 import (
+	"dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/instance_info_updater"
 	"fmt"
 	"log/slog"
 	"net/http"
@@ -362,5 +363,15 @@ func Start(version string, buildStamp string, gitHash string, quit chan struct{}
 			context.JSON(http.StatusOK, gin.H{})
 		},
 	)
+	r.GET(
+		"/third-party/update-instance-info", func(ctx *gin.Context) {
+			err := instance_info_updater.Updater()
+			if err != nil {
+				ctx.AbortWithStatusJSON(http.StatusInternalServerError, api.NewErrorResp(500, err))
+				return
+			}
+
+			ctx.JSON(http.StatusOK, gin.H{})
+		})
 	return r.Run(fmt.Sprintf("127.0.0.1:%d", config.RuntimeConfig.Port))
 }

dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/init.go

@@ -3,6 +3,7 @@ package third_party
 import (
 	"dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/instance_info_updater"
 	"dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/nginx_updater"
+	"errors"
 
 	"github.com/robfig/cron/v3"
 )
@@ -15,3 +16,17 @@ func init() {
 		instance_info_updater.Register,
 	}
 }
+
+func Updater() (err error) {
+	e := instance_info_updater.Updater()
+	if e != nil {
+		err = errors.Join(err, e)
+	}
+
+	e = nginx_updater.Updater()
+	if e != nil {
+		err = errors.Join(err, e)
+	}
+
+	return err
+}

dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/instance_info_updater/init.go

@@ -34,16 +34,21 @@ func Register(cj *cron.Cron) {
 }
 
 func updater() error {
-	err := os.MkdirAll(rconfig.CommonConfigDir, 0777)
-	if err != nil {
-		return errors.Wrap(err, "can't create config directory")
-	}
-
 	sleepN := time.Second * time.Duration(rand.Intn(120))
 	slog.Info("rand sleep", slog.Float64("seconds", sleepN.Seconds()))
 	time.Sleep(sleepN)
 	slog.Info("rand sleep awake")
 
+	return Updater()
+}
+
+func Updater() error {
+	err := os.MkdirAll(rconfig.CommonConfigDir, 0777)
+	if err != nil {
+		return errors.Wrap(err, "can't create config directory")
+	}
+
+	slog.Info("call reverse api", slog.Any("runtime config", config.RuntimeConfig))
 	info, layer, err := mysql.ListInstanceInfo(*config.RuntimeConfig.BkCloudID)
 	if err != nil {
 		slog.Error("list instance info failed", slog.String("err", err.Error()))

dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/nginx_updater/init.go

@@ -35,16 +35,20 @@ func Register(cj *cron.Cron) {
 }
 
 func updater() error {
-	err := os.MkdirAll(rconfig.CommonConfigDir, 0777)
-	if err != nil {
-		return errors.Wrap(err, "can't create config directory")
-	}
-
 	sleepN := time.Second * time.Duration(rand.Intn(120))
 	slog.Info("rand sleep", slog.Float64("seconds", sleepN.Seconds()))
 	time.Sleep(sleepN)
 	slog.Info("rand sleep awake")
 
+	return Updater()
+}
+
+func Updater() error {
+	err := os.MkdirAll(rconfig.CommonConfigDir, 0777)
+	if err != nil {
+		return errors.Wrap(err, "can't create config directory")
+	}
+
 	addrs, err := common.ListNginxAddrs(*config.RuntimeConfig.BkCloudID)
 	if err != nil {
 		return errors.Wrap(err, "list nginx addrs failed")

dbm-services/mysql/db-tools/mysql-monitor/cmd/init.go

@@ -9,6 +9,7 @@ import (
 
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
 
+	"github.com/google/uuid"
 	"gopkg.in/natefinch/lumberjack.v2"
 )
 
@@ -73,12 +74,13 @@ func initLogger(cfg *config.LogConfig) {
 		handleOpt.Level = slog.LevelInfo
 	}
 
-	var logger *slog.Logger
 	if cfg.Json {
-		logger = slog.New(slog.NewJSONHandler(io.MultiWriter(ioWriters...), &handleOpt))
+		config.Logger = slog.New(slog.NewJSONHandler(io.MultiWriter(ioWriters...), &handleOpt))
 	} else {
-		logger = slog.New(slog.NewTextHandler(io.MultiWriter(ioWriters...), &handleOpt))
+		config.Logger = slog.New(slog.NewTextHandler(io.MultiWriter(ioWriters...), &handleOpt))
 	}
 
-	slog.SetDefault(logger)
+	config.Logger = config.Logger.With("uuid", uuid.New().String())
+
+	slog.SetDefault(config.Logger)
 }

dbm-services/mysql/db-tools/mysql-monitor/go.mod

@@ -23,6 +23,7 @@ require (
 
 require (
 	github.com/gofrs/flock v0.12.1
+	github.com/google/uuid v1.1.2
 	github.com/mitchellh/mapstructure v1.5.0
 	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
 	golang.org/x/time v0.1.0

dbm-services/mysql/db-tools/mysql-monitor/go.sum

@@ -133,6 +133,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=

dbm-services/mysql/db-tools/mysql-monitor/pkg/config/init.go

@@ -12,6 +12,8 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+var Logger *slog.Logger
+
 var HeartBeatName = "mysql_monitor_heart_beat"
 var HeartBeatSchedule = "@every 5m"
 var MonitorConfig *Config

dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go

@@ -41,6 +41,9 @@ func Run(hardcode bool) error {
 		iNames = viper.GetStringSlice("run-items")
 	}
 	slog.Info("main loop", slog.String("items", strings.Join(iNames, ",")))
+	config.Logger = config.Logger.With("items", strings.Join(iNames, ","))
+	slog.SetDefault(config.Logger)
+
 	slog.Info("main loop", slog.Bool("hardcode", hardcode))
 
 	lockFileName := fmt.Sprintf("%d-%s.lock", config.MonitorConfig.Port, strings.Join(iNames, "."))
@@ -115,19 +118,21 @@ func Run(hardcode bool) error {
 	slog.Info("make connection collect", slog.Any("connection collect", cc))
 
 	if hardcode {
+		slog.Info("hardcode finish")
 		return nil
 	}
 
 	randSleepN := rand.Intn(5)
 	slog.Info(
 		"run monitor items",
-		slog.Any("items", iNames),
 		slog.Int("randSleepN", randSleepN),
 	)
 	// 每次整体随机休眠 [0:5), 多实例场景时稍微错开
 	time.Sleep(time.Duration(randSleepN) * time.Second)
 
 	for _, iName := range iNames {
+		config.Logger = config.Logger.With("current item", iName)
+		slog.SetDefault(config.Logger)
 
 		if constructor, ok := itemscollect.RegisteredItemConstructor()[iName]; ok {
 
@@ -159,5 +164,7 @@ func Run(hardcode bool) error {
 			continue
 		}
 	}
+
+	slog.Info("main loop round finish")
 	return nil
 }

dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/internal/connect_db.go

@@ -21,7 +21,7 @@ func connectDB(ip string, port int, ca *config.ConnectAuth, withPing bool, isPro
 			),
 		)
 		if err != nil {
-			slog.Error("connect db with ping", err)
+			slog.Error("connect db with ping", slog.String("err", err.Error()))
 			return nil, err
 		}
 	} else {
@@ -35,7 +35,7 @@ func connectDB(ip string, port int, ca *config.ConnectAuth, withPing bool, isPro
 			),
 		)
 		if err != nil {
-			slog.Error("connect db without ping", err)
+			slog.Error("connect db without ping", slog.String("err", err.Error()))
 			return nil, err
 		}
 		// 没有 ping 可能返回的是一个无效连接
@@ -48,7 +48,7 @@ func connectDB(ip string, port int, ca *config.ConnectAuth, withPing bool, isPro
 			sr, err = db.Queryx(`SELECT 1`)
 		}
 		if err != nil {
-			slog.Error("ping proxy failed", err)
+			slog.Error("ping proxy failed", slog.String("err", err.Error()))
 			return nil, err
 		}
 		defer func() {