Code quality

.github/workflows/build_and_test.yml

@@ -3,7 +3,7 @@ name: Run Build and Test
 on:
   push:
     branches:
-      - 'main'
+      - "main"
   pull_request:
     branches:
       - "*"
@@ -18,7 +18,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22.0"
+          go-version: "1.22.11"
       - shell: bash
         run: |
           make build
@@ -31,26 +31,30 @@ jobs:
       fail-fast: false
       matrix:
         suite: [
-          # CLI test suites
-          { group: "cmd", name: "cmd", path: "" },
-          # providers suites, some of the providers are too heavy to run as single test
-          { group: "pkg/providers", name: "yt", path: "yt", yt: true },
-          { group: "pkg/providers", name: "providers-postgres", path: "postgres" },
-          # e2e test suites
-          { group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
-          { group: "tests/e2e", name: "pg2pg", path: "pg2pg" },
-          { group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
-          { group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
-          { group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
-          { group: "tests/e2e", name: "ch2s3", path: "ch2s3" },
-        ]
+            # CLI test suites
+            { group: "cmd", name: "cmd", path: "" },
+            # providers suites, some of the providers are too heavy to run as single test
+            { group: "pkg/providers", name: "yt", path: "yt", yt: true },
+            {
+              group: "pkg/providers",
+              name: "providers-postgres",
+              path: "postgres",
+            },
+            # e2e test suites
+            { group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
+            { group: "tests/e2e", name: "pg2pg", path: "pg2pg" },
+            { group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
+            { group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
+            { group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
+            { group: "tests/e2e", name: "ch2s3", path: "ch2s3" },
+          ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22.0"
+          go-version: "1.22.11"
       - shell: bash
         run: |
           go install gotest.tools/gotestsum@latest
@@ -94,44 +98,64 @@ jobs:
       fail-fast: false
       matrix:
         suite: [
-          # canon test suites
-          { group: "tests/canon", name: "canon-parser", path: "parser" },
-          { group: "tests/storage", name: "storage-pg", path: "pg" },
-          # internal test suites
-          { group: "internal", name: "internal", path: "..." },
-          # provider test suites
-          { group: "pkg/providers", name: "providers-mongo", path: "mongo" },
-          { group: "pkg/providers", name: "providers-mysql", path: "mysql" },
-          { group: "pkg/providers", name: "providers-sample", path: "sample" },
-          { group: "pkg/providers", name: "providers-kafka", path: "kafka" },
-          { group: "pkg/providers", name: "providers-kinesis", path: "kinesis" },
-          { group: "pkg/providers", name: "providers-greenplum", path: "greenplum" },
-          { group: "pkg/providers", name: "providers-clickhouse", path: "clickhouse" },
-          { group: "pkg/providers", name: "providers-elastic", path: "elastic" },
-          # pkg test suites
-          { group: "pkg", name: "abstract", path: "abstract" },
-          { group: "pkg", name: "transformer", path: "transformer" },
-          { group: "pkg", name: "predicate", path: "predicate" },
-          { group: "pkg", name: "dblog", path: "dblog" },
-          { group: "pkg", name: "functions", path: "functions" },
-          { group: "pkg", name: "maplock", path: "maplock" },
-          { group: "pkg", name: "middlewares", path: "middlewares" },
-          { group: "pkg", name: "parsequeue", path: "parsequeue" },
-          { group: "pkg", name: "util", path: "util" },
-          { group: "pkg", name: "stringutil", path: "stringutil" },
-          { group: "pkg", name: "serializer", path: "serializer" },
-          { group: "pkg", name: "worker", path: "worker" },
-          { group: "pkg", name: "schemaregistry", path: "schemaregistry" },
-          { group: "pkg", name: "parsers-generic", path: "parsers/generic" },
-          { group: "pkg", name: "parsers-scanner", path: "parsers/scanner" }
-        ]
+            # canon test suites
+            { group: "tests/canon", name: "canon-parser", path: "parser" },
+            { group: "tests/storage", name: "storage-pg", path: "pg" },
+            # internal test suites
+            { group: "internal", name: "internal", path: "..." },
+            # provider test suites
+            { group: "pkg/providers", name: "providers-mongo", path: "mongo" },
+            { group: "pkg/providers", name: "providers-mysql", path: "mysql" },
+            {
+              group: "pkg/providers",
+              name: "providers-sample",
+              path: "sample",
+            },
+            { group: "pkg/providers", name: "providers-kafka", path: "kafka" },
+            {
+              group: "pkg/providers",
+              name: "providers-kinesis",
+              path: "kinesis",
+            },
+            {
+              group: "pkg/providers",
+              name: "providers-greenplum",
+              path: "greenplum",
+            },
+            {
+              group: "pkg/providers",
+              name: "providers-clickhouse",
+              path: "clickhouse",
+            },
+            {
+              group: "pkg/providers",
+              name: "providers-elastic",
+              path: "elastic",
+            },
+            # pkg test suites
+            { group: "pkg", name: "abstract", path: "abstract" },
+            { group: "pkg", name: "transformer", path: "transformer" },
+            { group: "pkg", name: "predicate", path: "predicate" },
+            { group: "pkg", name: "dblog", path: "dblog" },
+            { group: "pkg", name: "functions", path: "functions" },
+            { group: "pkg", name: "maplock", path: "maplock" },
+            { group: "pkg", name: "middlewares", path: "middlewares" },
+            { group: "pkg", name: "parsequeue", path: "parsequeue" },
+            { group: "pkg", name: "util", path: "util" },
+            { group: "pkg", name: "stringutil", path: "stringutil" },
+            { group: "pkg", name: "serializer", path: "serializer" },
+            { group: "pkg", name: "worker", path: "worker" },
+            { group: "pkg", name: "schemaregistry", path: "schemaregistry" },
+            { group: "pkg", name: "parsers-generic", path: "parsers/generic" },
+            { group: "pkg", name: "parsers-scanner", path: "parsers/scanner" },
+          ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22.0"
+          go-version: "1.22.11"
       - shell: bash
         run: |
           go install gotest.tools/gotestsum@latest
@@ -167,10 +191,7 @@ jobs:
         run: exit 1
 
   test-report:
-    needs: [
-      generic-tests,
-      e2e-tests,
-    ]
+    needs: [generic-tests, e2e-tests]
     name: test-report
     if: always() && !contains(needs.*.result, 'skipped')
     runs-on: ubuntu-latest

.github/workflows/security.yml

@@ -0,0 +1,53 @@
+name: Security Testing
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+env:
+  GO_VERSION: "1.22.5"
+
+jobs:
+  SAST:
+    name: Perform SAST analysis (golangci-lint)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Run SAST (golangci-lint)
+        uses: golangci/golangci-lint-action@v4
+        with:
+          version: "latest"
+          only-new-issues: true
+          args: --timeout=10m
+          skip-pkg-cache: true
+          skip-build-cache: true
+
+  SCA:
+    name: Perform SCA analysis (govulncheck)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Run SCA (govulncheck)
+        uses: golang/govulncheck-action@v1
+        with:
+          go-version-input: ${{ env.GO_VERSION }}
+          go-package: ./...
+          cache: false

.golangci.yml

@@ -0,0 +1,70 @@
+linters:
+  disable-all: true
+  enable:
+    #- goimports
+    #- gosec
+    - unused
+    #- asasalint
+    - asciicheck
+    - bidichk
+    - bodyclose
+    - decorder
+    #- dogsled
+    #- durationcheck
+    #- errcheck
+    #- errname
+    #- errorlint
+    #- exhaustive
+    #- exportloopref
+    #- funlen
+    #- goconst
+    #- gocritic
+    - godot
+    #- gofmt
+    - gosimple
+    - govet
+    - ineffassign
+    - mirror
+    #- musttag
+    #- nestif
+    - nosprintfhostport
+    #- prealloc
+    #- predeclared
+    - staticcheck
+    - tenv
+    - tparallel
+    - typecheck
+    - unconvert
+    - usestdlibvars
+    - wastedassign
+
+issues:
+  exclude-rules:
+    - linters:
+        - govet
+      text: "printf: non-constant format string in call"
+    - linters:
+        - staticcheck
+      text: "SA1019"
+    - path: '(.+)_test\.go'
+      linters:
+        - gosec
+        - funlen
+        - goconst
+        - gocritic
+        - staticcheck
+        - ineffassign
+    - linters:
+        - gosimple
+      text: "S1000"
+    - path: "tests/.*"
+      linters:
+        - gosec
+        - funlen
+        - goconst
+        - gocritic
+        - staticcheck
+        - ineffassign
+    - linters:
+        - gosimple
+      text: "S1000"

cmd/trcli/config/config_test.go

@@ -9,10 +9,8 @@ import (
 )
 
 func TestParseTransferYaml_WithEnvSubstitution(t *testing.T) {
-	require.NoError(t, os.Setenv("FOO", "secret1"))
-	require.NoError(t, os.Setenv("BAR", "secret2"))
-	defer os.Unsetenv("FOO")
-	defer os.Unsetenv("BAR")
+	t.Setenv("FOO", "secret1")
+	t.Setenv("BAR", "secret2")
 
 	transfer, err := ParseTransferYaml([]byte(`
 src:
@@ -31,8 +29,8 @@ dst:
 }
 
 func TestParserTransferYaml_WithRawYaml(t *testing.T) {
-	require.NoError(t, os.Setenv("FOO", "secret1"))
-	require.NoError(t, os.Setenv("BAR", "secret2"))
+	t.Setenv("FOO", "secret1")
+	t.Setenv("BAR", "secret2")
 	defer os.Unsetenv("FOO")
 	defer os.Unsetenv("BAR")
 
@@ -53,8 +51,8 @@ dst:
 }
 
 func TestParserTransferYaml_WithYaml(t *testing.T) {
-	require.NoError(t, os.Setenv("FOO", "secret1"))
-	require.NoError(t, os.Setenv("BAR", "secret2"))
+	t.Setenv("FOO", "secret1")
+	t.Setenv("BAR", "secret2")
 	defer os.Unsetenv("FOO")
 	defer os.Unsetenv("BAR")
 

cmd/trcli/config/model.go

@@ -25,27 +25,27 @@ func (e Endpoint) RawParams() string {
 }
 
 type Runtime struct {
-	Type   string
-	Params interface{}
+	Type   string      `json:"type" yaml:"type"`
+	Params interface{} `json:"params" yaml:"params"`
 }
 
 type UploadTables struct {
-	Tables []abstract.TableDescription
+	Tables []abstract.TableDescription `json:"tables" yaml:"tables"`
 }
 
 type TransferYamlView struct {
-	ID                string
-	TransferName      string
-	Description       string
-	Labels            string
-	Status            model.TransferStatus
-	Type              abstract.TransferType
-	FolderID          string
-	CloudID           string
-	Src               Endpoint
-	Dst               Endpoint
-	RegularSnapshot   *abstract.RegularSnapshot `yaml:"regular_snapshot"`
-	Transformation    *transformer.Transformers `yaml:"transformation"`
-	DataObjects       *model.DataObjects        `yaml:"data_objects"`
-	TypeSystemVersion int                       `yaml:"type_system_version"`
+	ID                string                    `json:"id" yaml:"id"`
+	TransferName      string                    `json:"transfer_name" yaml:"transfer_name"`
+	Description       string                    `json:"description" yaml:"description"`
+	Labels            string                    `json:"labels" yaml:"labels"`
+	Status            model.TransferStatus      `json:"status" yaml:"status"`
+	Type              abstract.TransferType     `json:"type" yaml:"type"`
+	FolderID          string                    `json:"folder_id" yaml:"folder_id"`
+	CloudID           string                    `json:"cloud_id" yaml:"cloud_id"`
+	Src               Endpoint                  `json:"src" yaml:"src"`
+	Dst               Endpoint                  `json:"dst" yaml:"dst"`
+	RegularSnapshot   *abstract.RegularSnapshot `json:"regular_snapshot" yaml:"regular_snapshot"`
+	Transformation    *transformer.Transformers `json:"transformation" yaml:"transformation"`
+	DataObjects       *model.DataObjects        `json:"data_objects" yaml:"data_objects"`
+	TypeSystemVersion int                       `json:"type_system_version" yaml:"type_system_version"`
 }