Merge pull request #25 from ahdark-services/refactor/settings

Refactor(settings): use settings as system service instead of database repository, and use etcd instead of database to store configurations.

Author: AH-dark

.env

@@ -15,8 +15,10 @@ RABBITMQ_PORT=5672
 RABBITMQ_USERNAME=payment
 RABBITMQ_PASSWORD=payment
 
+ETCD_ENDPOINTS=localhost:2379
+
 TRACE_EXPORTER=otlp-grpc
-TRACE_OTLP_ENDPOINT=http://localhost:4317
+TRACE_OTLP_ENDPOINT=localhost:4317
 TRACE_OTLP_INSECURE=true
 
 METRICS_EXPORTER=prometheus

.env.docker

@@ -12,6 +12,8 @@ DB_NAME=payment
 REDIS_MODE=standalone
 REDIS_HOST=redis
 
+ETCD_ENDPOINTS=etcd:2379
+
 RABBITMQ_HOST=mq
 RABBITMQ_PORT=5672
 RABBITMQ_USERNAME=payment
@@ -22,7 +24,7 @@ CONSUL_ADDRESS=consul:8500
 JWT_SIGNING_KEY=123456
 
 TRACE_EXPORTER=otlp-grpc
-TRACE_OTLP_ENDPOINT=http://jaeger:4317
+TRACE_OTLP_ENDPOINT=jaeger:4317
 TRACE_OTLP_INSECURE=true
 
 METRICS_EXPORTER=prometheus

.github/workflows/build.yaml

@@ -22,6 +22,24 @@ jobs:
         with:
           go-version: '1.20'
 
+      - name: Cache Go build paths
+        id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+
+      - name: Go Mod Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+
       - name: Download dependencies
         run: |
           sudo apt install -y make

.github/workflows/test.yaml

@@ -27,6 +27,24 @@ jobs:
         with:
           go-version: '1.20'
 
+      - name: Cache Go build paths
+        id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+
+      - name: Go Mod Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+
       - name: Download dependencies
         run: |
           go mod download

.idea/dataSources.xml

@@ -9,6 +9,7 @@ import (
 	"github.com/ahdark-services/nextpayt/internal/consul"
 	"github.com/ahdark-services/nextpayt/internal/currency_rate_api"
 	"github.com/ahdark-services/nextpayt/internal/dcron"
+	"github.com/ahdark-services/nextpayt/internal/etcd"
 	"github.com/ahdark-services/nextpayt/internal/jwt"
 	"github.com/ahdark-services/nextpayt/internal/limiter"
 	"github.com/ahdark-services/nextpayt/internal/log"
@@ -31,6 +32,7 @@ func AppEntries() []fx.Option {
 		jwt.Module(),
 
 		redis.Module(),
+		etcd.Module(),
 		cache.Module(),
 		limiter.Module(),
 		dcron.Module(),

.idea/kubernetes-settings.xml

@@ -18,5 +18,6 @@ func Module() fx.Option {
 		fx.Provide(LoadEnvFn("RABBITMQ", RabbitMQConfig{})),
 		fx.Provide(LoadEnvFn("JWT", JWTConfig{})),
 		fx.Provide(LoadEnvFn("SERVER", ServerConfig{})),
+		fx.Provide(LoadEnvFn("ETCD", EtcdConfig{})),
 	)
 }

bootstrap/appentry.go

@@ -0,0 +1,14 @@
+package conf
+
+type EtcdConfig struct {
+	Endpoints []string `envconfig:"ETCD_ENDPOINTS" default:"localhost:2379"`
+	Username  string   `envconfig:"ETCD_USERNAME" default:""`
+	Password  string   `envconfig:"ETCD_PASSWORD" default:""`
+
+	Namespace string `envconfig:"ETCD_NAMESPACE" default:"nextpayt"` // Namespace for etcd keys
+
+	// TLS
+	CACert     string `envconfig:"ETCD_CA_CERT" default:""`     // CA certificate path
+	ClientCert string `envconfig:"ETCD_CLIENT_CERT" default:""` // Client certificate path
+	ClientKey  string `envconfig:"ETCD_CLIENT_KEY" default:""`  // Client key path
+}

conf/0module.go

@@ -1,48 +1,48 @@
 package constant
 
 const (
-	SettingAdminUserName     = "admin_user_name"
-	SettingAdminUserPassword = "admin_user_password"
-
-	SettingUserDashboardEndpoint  = "user_dashboard_endpoint"
-	SettingAdminDashboardEndpoint = "admin_dashboard_endpoint"
-	SettingCashierEndpoint        = "cashier_endpoint"
-	SettingClientAPIEndpoint      = "client_api_endpoint"
-	SettingCashierAPIEndpoint     = "cashier_api_endpoint"
-	SettingPublicAPIEndpoint      = "public_api_endpoint"
-	SettingExternalAPIEndpoint    = "external_api_endpoint"
-
-	SettingEmailSmtpHost             = "email_smtp_host"
-	SettingEmailSmtpPort             = "email_smtp_port"
-	SettingEmailSmtpUsername         = "email_smtp_username"
-	SettingEmailSmtpPassword         = "email_smtp_password"
-	SettingEmailSmtpEncryption       = "email_smtp_encryption"
-	SettingEmailSmtpSkipVerify       = "email_smtp_skip_verify"
-	SettingEmailFromName             = "email_from_name"
-	SettingEmailFromAddress          = "email_from_address"
-	SettingEmailSubjectPrefix        = "email_subject_prefix"
-	SettingEmailTemplateValidateCode = "email_template_validate_code"
-
-	SettingUserDefaultAllowLogin  = "user_default_allow_login"
-	SettingUserDefaultAllowPay    = "user_default_allow_pay"
-	SettingUserDefaultAllowSettle = "user_default_allow_settle"
-	SettingUserDefaultGroupId     = "user_default_group_id"
-
-	SettingApplicationAllowCreate = "application_allow_create"
-
-	SettingAuthRegisterEnabled = "auth_register_enabled"
-	SettingJWTExpire           = "jwt_expire"
-
-	SettingPaymentMinUSDAmount                 = "payment_min_usd_amount"
-	SettingPaymentMaxUSDAmount                 = "payment_max_usd_amount"
-	SettingPaymentOrderExpire                  = "payment_order_expire"
-	SettingPaymentEPaySubmitFrequencyPerHour   = "payment_frequency_epay_submit_per_hour"
-	SettingPaymentEPaySubmitFrequencyPerMinute = "payment_frequency_epay_submit_per_minute"
-	SettingPaymentEPaySubmitFrequencyPerSecond = "payment_frequency_epay_submit_per_second"
-	SettingPaymentRequestConsistentDomain      = "payment_request_consistent_domain"
-	SettingPaymentDefaultEPayCurrencyID        = "payment_default_epay_currency_id"
-
-	SettingCurrencyDefaultCurrencyID = "currency_default_currency_id"
-	SettingCurrencyAPIAPIKey         = "currency_api_api_key"
-	SettingCurrencyAPICacheDuration  = "currency_api_cache_duration"
+	SettingAdminUserName     = "manage:admin_user_name"
+	SettingAdminUserPassword = "manage:admin_user_password"
+
+	SettingUserDashboardEndpoint  = "endpoint:user_dashboard"
+	SettingAdminDashboardEndpoint = "endpoint:admin_dashboard"
+	SettingCashierEndpoint        = "endpoint:cashier"
+	SettingClientAPIEndpoint      = "endpoint:client_api"
+	SettingCashierAPIEndpoint     = "endpoint:cashier_api"
+	SettingPublicAPIEndpoint      = "endpoint:public_api"
+	SettingExternalAPIEndpoint    = "endpoint:external_api"
+
+	SettingEmailSmtpHost             = "email:smtp_host"
+	SettingEmailSmtpPort             = "email:smtp_port"
+	SettingEmailSmtpUsername         = "email:smtp_username"
+	SettingEmailSmtpPassword         = "email:smtp_password"
+	SettingEmailSmtpEncryption       = "email:smtp_encryption"
+	SettingEmailSmtpSkipVerify       = "email:smtp_skip_verify"
+	SettingEmailFromName             = "email:from_name"
+	SettingEmailFromAddress          = "email:from_address"
+	SettingEmailSubjectPrefix        = "email:subject_prefix"
+	SettingEmailTemplateValidateCode = "email:template_validate_code"
+
+	SettingUserDefaultAllowLogin  = "user:default:allow_login"
+	SettingUserDefaultAllowPay    = "user:default:allow_pay"
+	SettingUserDefaultAllowSettle = "user:default:allow_settle"
+	SettingUserDefaultGroupId     = "user:default:group_id"
+
+	SettingApplicationAllowCreate = "application:allow_create"
+
+	SettingAuthRegisterEnabled = "auth:register_enabled"
+	SettingJWTExpire           = "auth:jwt_expire"
+
+	SettingPaymentMinUSDAmount                 = "payment:min_usd_amount"
+	SettingPaymentMaxUSDAmount                 = "payment:max_usd_amount"
+	SettingPaymentOrderExpire                  = "payment:order_expire"
+	SettingPaymentEPaySubmitFrequencyPerHour   = "payment:frequency_epay:submit_per_hour"
+	SettingPaymentEPaySubmitFrequencyPerMinute = "payment:frequency_epay:submit_per_minute"
+	SettingPaymentEPaySubmitFrequencyPerSecond = "payment:frequency_epay:submit_per_second"
+	SettingPaymentRequestConsistentDomain      = "payment:request_consistent_domain"
+	SettingPaymentDefaultEPayCurrencyID        = "payment:default_epay_currency_id"
+
+	SettingCurrencyDefaultCurrencyID = "currency:default_currency_id"
+	SettingCurrencyAPIAPIKey         = "currency:api_api_key"
+	SettingCurrencyAPICacheDuration  = "currency:api_cache_duration"
 )

conf/etcd.go

@@ -23,7 +23,6 @@ func Module() fx.Option {
 		fx.Provide(repo.NewPaymentChannelPaymentGroupMany2ManyRepo),
 		fx.Provide(repo.NewPaymentGroupRepo),
 		fx.Provide(repo.NewPaymentTypeRepo),
-		fx.Provide(repo.NewSettingRepo),
 		fx.Provide(repo.NewSettlementRepo),
 		fx.Provide(repo.NewUnifiedOrderRepo),
 		fx.Provide(repo.NewUnifiedOrderAssociationRepo),

constant/setting.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"github.com/ahdark-services/nextpayt/internal/etcd"
+	"go.etcd.io/etcd/client/v3/namespace"
 	"os"
 
 	"github.com/joho/godotenv"
@@ -15,21 +17,17 @@ import (
 
 var ctx = context.Background()
 
-func main() {
-	if err := godotenv.Load(".env"); err != nil {
-		logrus.WithContext(ctx).WithError(err).Warn("load .env failed")
-	}
-
+func databaseMigrate() error {
 	var e conf.DatabaseConfig
 	if err := envconfig.Process("DB", &e); err != nil {
 		logrus.WithContext(ctx).WithError(err).Fatal("process env failed")
-		return
+		return err
 	}
 
 	db, err := instance.NewDBInstance(ctx, &e)
 	if err != nil {
 		logrus.WithContext(ctx).WithError(err).Fatal("new database instance failed")
-		return
+		return err
 	}
 
 	if err := db.WithContext(ctx).AutoMigrate(
@@ -53,29 +51,11 @@ func main() {
 		&dao.UserGroupPaymentAssociation{},
 	); err != nil {
 		logrus.WithContext(ctx).WithError(err).Fatal("auto migrate failed")
-		return
+		return err
 	}
 
 	logrus.WithContext(ctx).Info("auto migrate success")
 
-	// create default settings
-	for k, v := range dao.DefaultSettings {
-		// check name exists
-		if err := db.Model(&dao.Setting{}).Where("name = ?", k).First(&dao.Setting{}).Error; err == nil {
-			logrus.WithContext(ctx).WithField("name", k).Debug("setting is exists")
-			continue
-		}
-
-		// create default setting
-		if err := db.Create(&dao.Setting{
-			Name:  k,
-			Value: v,
-		}).Error; err != nil {
-			logrus.WithContext(ctx).WithError(err).Fatalf("create default setting failed: %s", k)
-			continue
-		}
-	}
-
 	// create default payment types if not exists
 	if err := db.Model(&dao.PaymentType{}).First(&dao.PaymentType{}).Error; err != nil {
 		logrus.WithContext(ctx).Info("create default payment types")
@@ -92,7 +72,7 @@ func main() {
 		logrus.WithContext(ctx).Info("create default organization group")
 		if err := db.Create(&dao.DefaultUserGroup).Error; err != nil {
 			logrus.WithContext(ctx).WithError(err).Fatalf("create default organization group failed: %s", dao.DefaultUserGroup.Name)
-			return
+			return err
 		}
 	}
 
@@ -102,12 +82,71 @@ func main() {
 		for _, currency := range dao.DefaultCurrencies {
 			if err := db.Create(&currency).Error; err != nil {
 				logrus.WithContext(ctx).WithError(err).Fatalf("create default currency failed: %s", currency.Name)
-				return
+				return err
 			}
 		}
 	}
 
 	logrus.WithContext(ctx).Info("create default data success")
 
+	return nil
+}
+
+func etcdMigrate() error {
+	var e conf.EtcdConfig
+	if err := envconfig.Process("ETCD", &e); err != nil {
+		logrus.WithContext(ctx).WithError(err).Fatal("process env failed")
+		return err
+	}
+
+	client, err := etcd.NewEtcdClient(ctx, &e)
+	if err != nil {
+		logrus.WithContext(ctx).WithError(err).Fatal("new etcd client failed")
+		return err
+	}
+	kv := namespace.NewKV(client, "settings:")
+	//client.Delete(ctx, "", clientv3.WithPrefix())
+
+	// create default settings
+	for k, v := range dao.DefaultSettings {
+		// check if exists
+		resp, err := kv.Get(ctx, k)
+		if err != nil {
+			logrus.WithContext(ctx).WithError(err).Fatalf("get setting failed: %s", k)
+			continue
+		}
+
+		if len(resp.Kvs) > 0 {
+			logrus.WithContext(ctx).Infof("setting exists: %s", k)
+			continue
+		}
+
+		// create
+		if _, err := kv.Put(ctx, k, v); err != nil {
+			logrus.WithContext(ctx).WithError(err).Fatalf("create setting failed: %s", k)
+			continue
+		}
+
+		logrus.WithContext(ctx).Infof("create setting success: %s", k)
+	}
+
+	return nil
+}
+
+func main() {
+	if err := godotenv.Load(".env"); err != nil {
+		logrus.WithContext(ctx).WithError(err).Warn("load .env failed")
+	}
+
+	if err := databaseMigrate(); err != nil {
+		logrus.WithContext(ctx).WithError(err).Fatal("database migrate failed")
+		return
+	}
+
+	if err := etcdMigrate(); err != nil {
+		logrus.WithContext(ctx).WithError(err).Fatal("etcd migrate failed")
+		return
+	}
+
 	os.Exit(0)
 }