Merge pull request #525 from LerianStudio/feature/MIDAZ-517

Feature/MIDAZ-517
LerianStudio · Feb 18, 2025 · f34afa8 · f34afa8
2 parents 5eacb6d + a03dcdd
commit f34afa8
Show file tree

Hide file tree

Showing 30 changed files with 539 additions and 152 deletions.
diff --git a/components/audit/docker-compose.yml b/components/audit/docker-compose.yml
@@ -15,6 +15,9 @@ services:
       - audit_network
       - infra_network
       - auth_network
+    depends_on:
+      - midaz-audit-server
+      - midaz-audit-signer
 
   midaz-audit-db:
     container_name: midaz-audit-db

diff --git a/components/infra/.env.example b/components/infra/.env.example
@@ -1,10 +1,15 @@
 # INFRA
 
+# PG BOUNCER
+DB_PGBOUNCER_PORT=5706
+
 # DB POSTGRESQL PRIMARY AND REPLICA
 DB_HOST=midaz-postgres-primary
 DB_USER=midaz
 DB_PASSWORD=lerian
 DB_PORT=5701
+MAX_CONNECTIONS=3000
+SHARED_BUFFERS=1GB
 
 DB_REPLICA_HOST=midaz-postgres-replica
 DB_REPLICA_USER=midaz

diff --git a/components/infra/docker-compose.yml b/components/infra/docker-compose.yml
@@ -75,6 +75,8 @@ services:
       -c max_wal_senders=10
       -c max_replication_slots=10
       -c hot_standby_feedback=on
+      -c max_connections=${MAX_CONNECTIONS}
+      -c shared_buffers=${SHARED_BUFFERS}
     volumes:
       - postgres-data:/var/lib/postgresql/data
       - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
@@ -96,10 +98,18 @@ services:
           echo 'Waiting for midaz-postgres-primary to connect...'
           sleep 1s
         done
+      
         echo 'Backup done..., starting midaz-postgres-replica...'
         chmod 0700 /var/lib/postgresql/data
+      
         # Ensure the port is set to use for the replica
         sed -i 's/^#port.*/port = ${DB_REPLICA_PORT}/' /var/lib/postgresql/data/postgresql.conf
+      
+        # Define database max conn
+        sed -i 's/^#*max_connections.*/max_connections = ${MAX_CONNECTIONS}/' /var/lib/postgresql/data/postgresql.conf      
+      
+        # Define database shared buffers
+        sed -i 's/^#*shared_buffers.*/shared_buffers = ${SHARED_BUFFERS}/' /var/lib/postgresql/data/postgresql.conf      
       fi
       exec postgres -c config_file=/var/lib/postgresql/data/postgresql.conf
       "
@@ -112,6 +122,23 @@ services:
       midaz-postgres-primary:
         condition: service_healthy
 
+  midaz-pgbouncer:
+    image: edoburu/pgbouncer
+    container_name: midaz-pgbouncer
+    restart: always
+    ports:
+      - ${DB_PGBOUNCER_PORT}:${DB_PGBOUNCER_PORT}
+    volumes:
+      - ./pgbouncer/pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini:ro
+      - ./pgbouncer/userlist.txt:/etc/pgbouncer/userlist.txt:ro
+    environment:
+      - DATABASES_HOST=localhost
+    networks:
+      - infra_network
+    depends_on:
+      - midaz-postgres-primary
+      - midaz-postgres-replica
+
   midaz-otel-lgtm:
     container_name: midaz-otel-lgtm
     image: grafana/otel-lgtm:latest

diff --git a/components/infra/pgbouncer/pgbouncer.ini b/components/infra/pgbouncer/pgbouncer.ini
@@ -0,0 +1,21 @@
+#pgbouncer.ini
+
+[databases]
+transaction_primary = host=midaz-postgres-primary port=5701 dbname=transaction
+transaction_replica = host=midaz-postgres-replica port=5702 dbname=transaction
+
+onboarding_primary = host=midaz-postgres-primary port=5701 dbname=onboarding
+onboarding_replica = host=midaz-postgres-replica port=5702 dbname=onboarding
+
+[pgbouncer]
+listen_addr = 0.0.0.0
+listen_port = 5706
+auth_type = md5
+auth_file = /etc/pgbouncer/userlist.txt
+pool_mode = transaction
+admin_users = postgres
+ignore_startup_parameters = extra_float_digits
+
+max_client_conn = 3000
+default_pool_size = 2500
+server_idle_timeout = 30
diff --git a/components/infra/pgbouncer/userlist.txt b/components/infra/pgbouncer/userlist.txt
@@ -0,0 +1 @@
+"midaz" "lerian"
diff --git a/components/infra/rabbitmq/etc/definitions.json b/components/infra/rabbitmq/etc/definitions.json
@@ -70,6 +70,11 @@
       "name": "transaction_balance_queue",
       "vhost": "/",
       "durable": true
+    },
+    {
+      "name": "balance_retry_queue",
+      "vhost": "/",
+      "durable": true
     }
   ],
   "exchanges": [
@@ -84,6 +89,12 @@
       "vhost": "/",
       "type": "direct",
       "durable": true
+    },
+    {
+      "name": "balance_retry_queue_exchange",
+      "vhost": "/",
+      "type": "direct",
+      "durable": true
     }
   ],
   "bindings": [
@@ -100,6 +111,13 @@
       "destination": "transaction_balance_queue",
       "destination_type": "queue",
       "routing_key": "transaction_balance_key"
+    },
+    {
+      "source": "balance_retry_queue_exchange",
+      "vhost": "/",
+      "destination": "balance_retry_queue",
+      "destination_type": "queue",
+      "routing_key": "balance_retry_queue_key"
     }
   ]
 }
diff --git a/components/onboarding/.env.example b/components/onboarding/.env.example
@@ -9,17 +9,34 @@ SERVER_PORT=3000
 SERVER_ADDRESS=:${SERVER_PORT}
 
 # DB POSTGRESQL PRIMARY AND REPLICA
-DB_HOST=midaz-postgres-primary
+#DB_HOST=midaz-postgres-primary
+#DB_USER=midaz
+#DB_NAME=onboarding
+#DB_PASSWORD=lerian
+#DB_PORT=5701
+
+#DB_REPLICA_HOST=midaz-postgres-replica
+#DB_REPLICA_USER=midaz
+#DB_REPLICA_NAME=onboarding
+#DB_REPLICA_PASSWORD=lerian
+#DB_REPLICA_PORT=5702
+
+DB_MAX_OPEN_CONNS=3000
+DB_MAX_IDLE_CONNS=3000
+
+# DB PGBOUNCE PRIMARY
+DB_HOST=midaz-pgbouncer
 DB_USER=midaz
-DB_NAME=onboarding
+DB_NAME=onboarding_primary
 DB_PASSWORD=lerian
-DB_PORT=5701
+DB_PORT=5706
 
-DB_REPLICA_HOST=midaz-postgres-replica
+# DB PGBOUNCE REPLICA
+DB_REPLICA_HOST=midaz-pgbouncer
 DB_REPLICA_USER=midaz
-DB_REPLICA_NAME=onboarding
+DB_REPLICA_NAME=onboarding_replica
 DB_REPLICA_PASSWORD=lerian
-DB_REPLICA_PORT=5702
+DB_REPLICA_PORT=5706
 
 # MONGO DB
 #MONGO_URI=mongodb+srv

diff --git a/components/transaction/.env.example b/components/transaction/.env.example
@@ -9,19 +9,34 @@ APP_CONTEXT=/transaction/v1
 SERVER_PORT=3001
 SERVER_ADDRESS=:${SERVER_PORT}
 
-# DB POSTGRESQL PRIMARY
-DB_HOST=midaz-postgres-primary
+# DB POSTGRESQL PRIMARY AND REPLICA
+#DB_HOST=midaz-postgres-primary
+#DB_USER=midaz
+#DB_NAME=transaction
+#DB_PASSWORD=lerian
+#DB_PORT=5701
+#
+#DB_REPLICA_HOST=midaz-postgres-replica
+#DB_REPLICA_USER=midaz
+#DB_REPLICA_NAME=transaction
+#DB_REPLICA_PASSWORD=lerian
+#DB_REPLICA_PORT=5702
+
+DB_MAX_OPEN_CONNS=3000
+DB_MAX_IDLE_CONNS=3000
+
+# DB PGBOUCE PRIMARY AND REPLICA
+DB_HOST=midaz-pgbouncer
 DB_USER=midaz
-DB_NAME=transaction
+DB_NAME=transaction_primary
 DB_PASSWORD=lerian
-DB_PORT=5701
+DB_PORT=5706
 
-# DB POSTGRESQL REPLICA
-DB_REPLICA_HOST=midaz-postgres-replica
+DB_REPLICA_HOST=midaz-pgbouncer
 DB_REPLICA_USER=midaz
-DB_REPLICA_NAME=transaction
+DB_REPLICA_NAME=transaction_replica
 DB_REPLICA_PASSWORD=lerian
-DB_REPLICA_PORT=5702
+DB_REPLICA_PORT=5706
 
 # MONGO DB
 #MONGO_URI=mongodb+srv
@@ -69,6 +84,9 @@ RABBITMQ_DEFAULT_PASS=lerian
 RABBITMQ_AUDIT_EXCHANGE=audit_exchange
 RABBITMQ_AUDIT_KEY=audit_key
 RABBITMQ_QUEUE=transaction_balance_queue
+RABBITMQ_BALANCE_RETRY_EXCHANGE=balance_retry_queue_exchange
+RABBITMQ_BALANCE_RETRY_KEY=balance_retry_queue_key
+RABBITMQ_BALANCE_RETRY_QUEUE=balance_retry_queue
 
 # SWAGGER
 SWAGGER_TITLE=Transaction API
@@ -81,7 +99,7 @@ SWAGGER_LEFT_DELIMITER={{
 SWAGGER_RIGHT_DELIMITER=}}
 
 # AUDIT
-AUDIT_LOG_ENABLED=true
+AUDIT_LOG_ENABLED=false
 
 # PAGINATION
 MAX_PAGINATION_LIMIT=100