feat(dbm-services): 库表分析识别子查询 TencentBlueKing#10151

Author: ymakedaq

dbm-services/mysql/db-simulation/app/config/config.go

@@ -185,8 +185,8 @@ func IsEmptyTdbctlPodResourceConfig() bool {
 func loadConfig() (err error) {
 	viper.SetConfigName("config")
 	viper.SetConfigType("yaml")
-	viper.AddConfigPath("$HOME/conf")
 	viper.AddConfigPath("./conf")
+	viper.AddConfigPath("$HOME/conf")
 	if err = viper.ReadInConfig(); err != nil {
 		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
 			logger.Error("config file not found,maybe read by env")

dbm-services/mysql/db-simulation/app/syntax/parse_relation_db.go

@@ -50,7 +50,6 @@ func (tf *TmysqlParseFile) DoParseRelationDbs(version string) (createDbs, relati
 	}
 	logger.Info("all sqlfiles download ok ~")
 	alreadExecutedSqlfileChan := make(chan string, len(tf.Param.FileNames))
-
 	go func() {
 		if err = tf.Execute(alreadExecutedSqlfileChan, version); err != nil {
 			logger.Error("failed to execute tmysqlparse: %s", err.Error())
@@ -168,40 +167,41 @@ func (t *TmysqlParse) analyzeRelationDbs(inputfileName, mysqlVersion string) (
 			logger.Error("json unmasrshal line:%s failed %s", string(line), err.Error())
 			return nil, nil, nil, false, err
 		}
-		// 判断是否有语法错误
+		//  ErrorCode !=0 就是语法错误
 		if res.ErrorCode != 0 {
 			return nil, nil, nil, false, fmt.Errorf("%s", res.ErrorMsg)
 		}
-		if lo.IsNotEmpty(res.Command) {
-			if res.Command == SQLTypeCreateTable {
-				var c CreateTableResult
-				if err = json.Unmarshal(line, &c); err != nil {
-					logger.Error("json unmasrshal line:%s failed %s", string(line), err.Error())
-					return nil, nil, nil, false, err
-				}
-				// 需要排除create table like
-				if c.IsCreateTableLike || c.IsCreateTableSelect {
-					return nil, nil, allCommandType, true, nil
-				}
-			} else {
-				allCommandType = append(allCommandType, res.Command)
-			}
-		}
-		if slices.Contains([]string{SQLTypeCreateProcedure, SQLTypeCreateFunction, SQLTypeCreateView, SQLTypeCreateTrigger,
-			SQLTypeInsertSelect, SQLTypeRelaceSelect},
-			res.Command) {
-			return nil, nil, allCommandType, true, nil
-		}
-		if lo.IsEmpty(res.DbName) {
+		if lo.IsEmpty(res.Command) {
 			continue
 		}
+		if lo.IsNotEmpty(res.DbName) {
+			relationDbs = append(relationDbs, res.DbName)
+		}
 		// create db not need dump db
 		if slices.Contains([]string{SQLTypeCreateDb}, res.Command) {
 			createDbs = append(createDbs, res.DbName)
 			continue
 		}
-		relationDbs = append(relationDbs, res.DbName)
-
+		if res.HasSubQuery {
+			return nil, relationDbs, allCommandType, true, nil
+		}
+		if slices.Contains([]string{SQLTypeCreateProcedure, SQLTypeCreateFunction, SQLTypeCreateView, SQLTypeCreateTrigger,
+			SQLTypeInsertSelect, SQLTypeRelaceSelect},
+			res.Command) {
+			return nil, relationDbs, allCommandType, true, nil
+		}
+		if res.Command == SQLTypeCreateTable {
+			var c CreateTableResult
+			if err = json.Unmarshal(line, &c); err != nil {
+				logger.Error("json unmasrshal line:%s failed %s", string(line), err.Error())
+				return nil, nil, nil, false, err
+			}
+			// 需要排除create table like
+			if c.IsCreateTableLike || c.IsCreateTableSelect {
+				return nil, relationDbs, allCommandType, true, nil
+			}
+		}
+		allCommandType = append(allCommandType, res.Command)
 	}
 	return createDbs, relationDbs, allCommandType, false, nil
 }

dbm-services/mysql/db-simulation/app/syntax/syntax.go

@@ -248,8 +248,9 @@ func (t *TmysqlParse) Init() (err error) {
 	return nil
 }
 
-// DelTempDir TODO
+// DelTempDir del tempDir
 func (t *TmysqlParse) DelTempDir() {
+	logger.Info("start to remove tempDir:%s", t.tmpWorkdir)
 	if err := os.RemoveAll(t.tmpWorkdir); err != nil {
 		logger.Warn("remove tempDir:" + t.tmpWorkdir + ".error info:" + err.Error())
 	}

dbm-services/mysql/db-simulation/app/syntax/tmysqlpase_schema.go

@@ -232,6 +232,7 @@ type ParseLineQueryBase struct {
 	ErrorMsg        string `json:"error_msg,omitempty"`
 	MinMySQLVersion int    `json:"min_mysql_version"`
 	MaxMySQLVersion int    `json:"max_my_sql_version"`
+	HasSubQuery     bool   `json:"has_subquery,omitempty"`
 }
 
 // IsSysDb sql modify target db is sys db

dbm-services/mysql/db-simulation/handler/syntax_check.go

@@ -252,6 +252,7 @@ func (s SyntaxHandler) ParseSQLFileRelationDb(r *gin.Context) {
 			FileNames:      param.Files,
 		},
 	}
+	defer p.DelTempDir()
 	createDbs, dbs, allCommands, dumpall, err := p.DoParseRelationDbs("")
 	if err != nil {
 		s.SendResponse(r, err, nil)
@@ -270,7 +271,7 @@ func (s SyntaxHandler) ParseSQLFileRelationDb(r *gin.Context) {
 	defer p.DelTempDir()
 	// 如果所有的命令都是alter table, dump指定库表
 	logger.Debug("debug: %v,%d", allCommands, len(allCommands))
-	if isAllOperateTable(allCommands) {
+	if isAllOperateTable(allCommands) && !dumpall {
 		relationTbls, err := p.ParseSpecialTbls("")
 		if err != nil {
 			s.SendResponse(r, err, nil)
@@ -320,7 +321,7 @@ func isAllOperateTable(allCommands []string) bool {
 	return lo.Every([]string{
 		syntax.SQLTypeAlterTable, syntax.SQLTypeUseDb,
 		syntax.SQLTypeCreateIndex, syntax.SQLTypeDropTable,
-		syntax.SQLTypeInsert, syntax.SQLTypeDelete,
+		syntax.SQLTypeInsert, syntax.SQLTypeDelete, syntax.SQLTypeUpdate,
 		syntax.SQLTypeCreateTable, syntax.SQLTypeReplace,
 	}, allCommands)
 }
@@ -358,15 +359,15 @@ func (s *SyntaxHandler) ParseSQLRelationDb(r *gin.Context) {
 			FileNames:      []string{fileName},
 		},
 	}
-	// defer p.DelTempDir()
+	defer p.DelTempDir()
 	createDbs, dbs, allCommands, dumpall, err := p.DoParseRelationDbs("")
 	if err != nil {
 		s.SendResponse(r, err, nil)
 		return
 	}
 	// 如果所有的命令都是alter table, dump指定库表
 	logger.Info("all command types: %v,%d", allCommands, len(allCommands))
-	if isAllOperateTable(allCommands) {
+	if isAllOperateTable(allCommands) && !dumpall {
 		relationTbls, err := p.ParseSpecialTbls("")
 		if err != nil {
 			s.SendResponse(r, err, nil)
@@ -382,7 +383,6 @@ func (s *SyntaxHandler) ParseSQLRelationDb(r *gin.Context) {
 		})
 		return
 	}
-
 	s.SendResponse(r, nil, gin.H{
 		"create_dbs": createDbs,
 		"dbs":        dbs,