Merge branch 'develop' into main-develop-sync

db/migrations/admin-migrations/2024-07-15.0-migrate-data-v1-to-v2-function.sql

@@ -36,7 +36,9 @@ BEGIN
         ALTER TABLE public.receiver_wallets
             ALTER COLUMN status DROP DEFAULT,
             ALTER COLUMN status TYPE %I.receiver_wallet_status
-                USING status::text::%I.receiver_wallet_status;
+                USING status::text::%I.receiver_wallet_status,
+            ALTER COLUMN stellar_memo_type TYPE %I.memo_type
+                USING stellar_memo_type::text::%I.memo_type;
         INSERT INTO %I.receiver_wallets SELECT * FROM public.receiver_wallets;
 
         ALTER TABLE public.receiver_verifications
@@ -72,7 +74,7 @@ BEGIN
                    schema_name, schema_name, schema_name, schema_name, schema_name, schema_name,
                    schema_name, schema_name, schema_name, schema_name, schema_name, schema_name,
                    schema_name, schema_name, schema_name, schema_name, schema_name, schema_name,
-                   schema_name, schema_name);
+                   schema_name, schema_name, schema_name, schema_name);
 
 
     -- Step 3: Import TSS data

db/migrations/sdp-migrations/2025-02-07.0-receiver-wallets-update-memo-type.sql

@@ -0,0 +1,44 @@
+-- This migration updates the receiver_wallets, circle_recipients and organizations tables to support the memo use cases.
+-- +migrate Up
+
+-- +migrate StatementBegin
+DO $$ BEGIN
+    CREATE TYPE memo_type AS ENUM (
+        'text',
+        'id',
+        'hash',
+        'return'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+-- +migrate StatementEnd
+
+ALTER TABLE receiver_wallets
+    ALTER COLUMN stellar_memo_type TYPE memo_type
+    USING CASE
+        WHEN stellar_memo_type IN ('text', 'id', 'hash', 'return') THEN stellar_memo_type::memo_type
+        ELSE 'text'::memo_type
+    END;
+
+ALTER TABLE circle_recipients
+    ADD COLUMN stellar_address TEXT,
+    ADD COLUMN stellar_memo TEXT;
+
+ALTER TABLE organizations
+    ADD COLUMN is_memo_tracing_enabled BOOLEAN DEFAULT TRUE;
+
+
+-- +migrate Down
+
+ALTER TABLE organizations
+    DROP COLUMN is_memo_tracing_enabled;
+
+ALTER TABLE circle_recipients
+    DROP COLUMN stellar_address,
+    DROP COLUMN stellar_memo;
+
+ALTER TABLE receiver_wallets
+    ALTER COLUMN stellar_memo_type TYPE TEXT USING stellar_memo_type::text;
+
+DROP TYPE IF EXISTS memo_type;

db/migrations/tss-migrations/2025-01-07.0-add-memo-columns.sql

@@ -0,0 +1,28 @@
+-- This migration adds the memo and memo_type (with custom type) columns to the submitter_transactions table.
+-- +migrate Up
+
+-- +migrate StatementBegin
+DO $$ BEGIN
+    CREATE TYPE memo_type AS ENUM (
+        'text',
+        'id',
+        'hash',
+        'return'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+-- +migrate StatementEnd
+
+ALTER TABLE submitter_transactions
+    ADD COLUMN memo TEXT,
+    ADD COLUMN memo_type memo_type;
+
+-- +migrate Down
+
+
+ALTER TABLE submitter_transactions
+    DROP COLUMN memo,
+    DROP COLUMN memo_type;
+
+DROP TYPE IF EXISTS memo_type;

dev/README.md

@@ -5,7 +5,7 @@
   - [Table of Contents](#table-of-contents)
   - [Introduction](#introduction)
   - [Quick Setup and Deployment](#quick-setup-and-deployment)
-    - [Docker](#docker)
+    - [Pre-requisites](#pre-requisites)
     - [Clone the repository:](#clone-the-repository)
     - [Update local DNS](#update-local-dns)
     - [Automated Stellar Account Creation and .env Configuration](#automated-stellar-account-creation-and-env-configuration)
@@ -15,12 +15,14 @@
   - [Additional Development Environment Details](#additional-development-environment-details)
     - [Stellar Accounts and .env File](#stellar-accounts-and-env-file)
     - [Building the SDP Docker Containers](#building-the-sdp-docker-containers)
+    - [Using Kafka for Event Handling](#using-kafka-for-event-handling)
     - [Remote Debugging](#remote-debugging)
-    - [Remote Debugging](#remote-debugging-1)
       - [Ensure Docker Containers are Running:](#ensure-docker-containers-are-running)
       - [Using VS Code:](#using-vs-code)
       - [Using IntelliJ GoLang:](#using-intellij-golang)
     - [Monitoring the SDP](#monitoring-the-sdp)
+      - [Start Prometheus and Grafana containers](#start-prometheus-and-grafana-containers)
+      - [Load the SDP Grafana Dashboard](#load-the-sdp-grafana-dashboard)
   - [Troubleshooting](#troubleshooting)
       - [Sample Tenant Management Postman collection](#sample-tenant-management-postman-collection)
       - [Distribution account out of funds](#distribution-account-out-of-funds)

internal/anchorplatform/object_schemas.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strings"
 	"time"
+
+	"github.com/stellar/stellar-disbursement-platform-backend/pkg/schema"
 )
 
 // GetTransactionsQueryParams are the query parameters that can be used in the `GET {PlatformAPIBaseURL}/transactions`
@@ -134,9 +136,9 @@ type APSep24TransactionPatchPostError struct {
 
 // APTransactionStatus is the body of the Stellar transaction stored in the Anchor Platform.
 type APStellarTransaction struct {
-	ID       string `json:"id"`
-	Memo     string `json:"memo,omitempty"`
-	MemoType string `json:"memo_type,omitempty"`
+	ID       string          `json:"id"`
+	Memo     string          `json:"memo,omitempty"`
+	MemoType schema.MemoType `json:"memo_type,omitempty"`
 	// CreatedAt time.Time `json:"created_at"`
 	// Envelope string `json:"envelope"`
 	// Payments  []APStellarPayment `json:"payments,omitempty"`

internal/circle/payment_request.go

@@ -39,6 +39,7 @@ type PaymentRequest struct {
 	SourceWalletID            string
 	RecipientID               string
 	DestinationStellarAddress string
+	DestinationStellarMemo    string
 	APIType                   APIType
 	Amount                    string
 	StellarAssetCode          string

internal/circle/recipient.go

@@ -15,6 +15,7 @@ import (
 type Recipient struct {
 	ID         string            `json:"id"`
 	Chain      string            `json:"chain"`
+	AddressTag string            `json:"addressTag"`
 	Address    string            `json:"address"`
 	Metadata   RecipientMetadata `json:"metadata"`
 	Status     string            `json:"status"`
@@ -32,6 +33,7 @@ type RecipientMetadata struct {
 type RecipientRequest struct {
 	IdempotencyKey string            `json:"idempotencyKey"`
 	Address        string            `json:"address"`
+	AddressTag     string            `json:"addressTag"`
 	Chain          string            `json:"chain"`
 	Metadata       RecipientMetadata `json:"metadata"`
 }

internal/circle/service.go

@@ -112,9 +112,10 @@ func (s *Service) SendTransfer(ctx context.Context, paymentRequest PaymentReques
 			ID:   paymentRequest.SourceWalletID,
 		},
 		Destination: TransferAccount{
-			Type:    TransferAccountTypeBlockchain,
-			Chain:   StellarChainCode,
-			Address: paymentRequest.DestinationStellarAddress,
+			Type:       TransferAccountTypeBlockchain,
+			Chain:      StellarChainCode,
+			Address:    paymentRequest.DestinationStellarAddress,
+			AddressTag: paymentRequest.DestinationStellarMemo,
 		},
 	})
 }

internal/data/assets.go

@@ -24,6 +24,22 @@ type Asset struct {
 	DeletedAt *time.Time `json:"deleted_at" csv:"-" db:"deleted_at"`
 }
 
+func AssetColumnNames(tableReference, resultAlias string, includeDates bool) string {
+	cols := []string{"id", "code"}
+	if includeDates {
+		cols = append(cols, "created_at", "updated_at", "deleted_at")
+	}
+
+	columns := SQLColumnConfig{
+		TableReference:  tableReference,
+		ResultAlias:     resultAlias,
+		RawColumns:      cols,
+		CoalesceColumns: []string{"issuer"},
+	}.Build()
+
+	return strings.Join(columns, ",\n")
+}
+
 // IsNative returns true if the asset is the native asset (XLM).
 func (a Asset) IsNative() bool {
 	return strings.TrimSpace(a.Issuer) == "" &&
@@ -53,16 +69,11 @@ type AssetModel struct {
 func (a *AssetModel) Get(ctx context.Context, id string) (*Asset, error) {
 	var asset Asset
 	query := `
-		SELECT 
-		    a.id, 
-		    a.code, 
-		    a.issuer,
-		    a.created_at,
-		    a.updated_at,
-		    a.deleted_at
-		FROM 
+		SELECT
+		    *
+		FROM
 		    assets a
-		WHERE 
+		WHERE
 		    a.id = $1
 		`
 
@@ -80,14 +91,9 @@ func (a *AssetModel) Get(ctx context.Context, id string) (*Asset, error) {
 func (a *AssetModel) GetByCodeAndIssuer(ctx context.Context, code, issuer string) (*Asset, error) {
 	var asset Asset
 	query := `
-		SELECT 
-		    a.id, 
-		    a.code, 
-		    a.issuer,
-		    a.created_at,
-		    a.updated_at,
-		    a.deleted_at
-		FROM 
+		SELECT
+		    *
+		FROM
 		    assets a
 		WHERE a.code = $1
 		AND a.issuer = $2
@@ -107,44 +113,43 @@ func (a *AssetModel) GetByCodeAndIssuer(ctx context.Context, code, issuer string
 func (a *AssetModel) GetByWalletID(ctx context.Context, walletID string) ([]Asset, error) {
 	assets := []Asset{}
 	query := `
-		SELECT 
+		SELECT
 		    a.*
-		FROM 
+		FROM
 		    assets a
-		JOIN 
+		JOIN
 		    wallets_assets wa ON a.id = wa.asset_id
-		WHERE 
-		    deleted_at IS NULL 
+		WHERE
+		    deleted_at IS NULL
 		    AND wa.wallet_id = $1
-		ORDER BY 
+		ORDER BY
 		    a.code ASC
 	`
 
 	err := a.dbConnectionPool.SelectContext(ctx, &assets, query, walletID)
 	if err != nil {
-		return nil, fmt.Errorf("querying assets: %w", err)
+		return nil, fmt.Errorf("selecting assets by wallet ID %s: %w", walletID, err)
 	}
 	return assets, nil
 }
 
 // GetAll returns all assets in the database.
 func (a *AssetModel) GetAll(ctx context.Context) ([]Asset, error) {
-	// TODO: We will want to filter out "deleted" assets at some point
 	assets := []Asset{}
 	query := `
-		SELECT 
-			a.*
-		FROM 
-			assets a
-		WHERE 
+		SELECT
+			*
+		FROM
+			assets
+		WHERE
 		    deleted_at IS NULL
-		ORDER BY 
-			a.code ASC
+		ORDER BY
+			code ASC
 	`
 
 	err := a.dbConnectionPool.SelectContext(ctx, &assets, query)
 	if err != nil {
-		return nil, fmt.Errorf("error querying assets: %w", err)
+		return nil, fmt.Errorf("selecting assets: %w", err)
 	}
 	return assets, nil
 }
@@ -188,7 +193,7 @@ func (a *AssetModel) GetOrCreate(ctx context.Context, code, issuer string) (*Ass
 	)
 	SELECT * FROM create_asset ca
 	UNION ALL
-	SELECT * FROM assets a 
+	SELECT * FROM assets a
 	WHERE a.code = $1
 	AND a.issuer = $2
 	`
@@ -240,7 +245,7 @@ func (a *AssetModel) GetAssetsPerReceiverWallet(ctx context.Context, receiverWal
 
 	var receiverWalletsAssets []ReceiverWalletAsset
 	query := `
-		WITH latest_payments_by_wallet AS ( 
+		WITH latest_payments_by_wallet AS (
 			-- Gets the latest payment by wallet with its asset
 			SELECT
 				p.id AS payment_id,
@@ -263,7 +268,7 @@ func (a *AssetModel) GetAssetsPerReceiverWallet(ctx context.Context, receiverWal
 				m.receiver_wallet_id,
 				m.wallet_id,
 				m.asset_id,
-				COUNT(*) AS total_invitation_sms_resent_attempts
+				COUNT(*) AS total_invitation_resent_attempts
 			FROM
 				messages m
 				INNER JOIN receiver_wallets rw ON rw.id = m.receiver_wallet_id AND rw.wallet_id = m.wallet_id
@@ -282,15 +287,11 @@ func (a *AssetModel) GetAssetsPerReceiverWallet(ctx context.Context, receiverWal
 			lpw.receiver_registration_message_template,
 			rw.id AS "receiver_wallet.id",
 			rw.invitation_sent_at AS "receiver_wallet.invitation_sent_at",
-			COALESCE(mrsi.total_invitation_sms_resent_attempts, 0) AS "receiver_wallet.total_invitation_sms_resent_attempts",
+			COALESCE(mrsi.total_invitation_resent_attempts, 0) AS "receiver_wallet.total_invitation_resent_attempts",
 			r.id AS "receiver_wallet.receiver.id",
 			COALESCE(r.phone_number, '') AS "receiver_wallet.receiver.phone_number",
 			COALESCE(r.email, '') AS "receiver_wallet.receiver.email",
-			a.id AS "asset.id",
-			a.code AS "asset.code",
-			a.issuer AS "asset.issuer",
-			a.created_at AS "asset.created_at",
-			a.updated_at AS "asset.updated_at"
+			` + AssetColumnNames("a", "asset", true) + `
 		FROM
 			assets a
 			INNER JOIN latest_payments_by_wallet lpw ON lpw.asset_id = a.id