Merge branch 'PAGOPA-1488-renewal-debt-positions' of

https://github.com/pagopa/pagopa-debt-position.git into
PAGOPA-1488-renewal-debt-positions

Conflicts:
	src/main/java/it/gov/pagopa/debtposition/scheduler/HistoricizationScheduler.java
	src/main/resources/application-h2.properties
	src/test/resources/application.properties

src/main/java/it/gov/pagopa/debtposition/scheduler/HistoricizationScheduler.java

@@ -39,7 +39,7 @@
 
 @Component
 @Slf4j
-@ConditionalOnProperty(name = "cron.job.schedule.historicization.enabled", matchIfMissing = true)
+@ConditionalOnProperty(name = "cron.job.schedule.history.enabled", matchIfMissing = true)
 @NoArgsConstructor
 public class HistoricizationScheduler {
 
@@ -52,24 +52,24 @@ public class HistoricizationScheduler {
 
 
     // extraction params
-    @Value("${cron.job.schedule.extraction.history.query:SELECT pp FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1}")
+    @Value("${cron.job.schedule.history.query:SELECT pp FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1}")
     private String extractionQuery;
-    @Value("${cron.job.schedule.extraction.history.query.interval:365}")
+    @Value("${cron.job.schedule.history.query.interval:365}")
     private short extractionInterval;
     // extraction params: pagination mode
-    @Value("${cron.job.schedule.extraction.history.query.paginated:true}")
+    @Value("${cron.job.schedule.history.paginated:true}")
     private boolean paginationMode;
-    @Value("${cron.job.schedule.extraction.history.query.count:SELECT count(pp.id) FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1}")
+    @Value("${cron.job.schedule.history.query.count:SELECT count(pp.id) FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1}")
     private String countExtractionQuery;
-    @Value("${cron.job.schedule.extraction.history.query.page.size:100000}")
+    @Value("${cron.job.schedule.history.query.page.size:100000}")
     private int pageSize;
 
     // azure storage params
     @Value("${azure.archive.storage.connection}")
     private String archiveStorageConnection;
-    @Value("${azure.archive.storage.table.po:pagopadweugpsarchivesapaymentoptiontable}")
+    @Value("${azure.archive.storage.table.po:paymentoptiontable}")
     private String archiveStoragePOTable;
-    @Value("${azure.archive.storage.table.pp:pagopadweugpsarchivesapaymentpositiontable}")
+    @Value("${azure.archive.storage.table.pp:paymentpositiontable}")
     private String archiveStoragePPTable;
 
     @Autowired
@@ -82,8 +82,9 @@ public HistoricizationScheduler(PaymentPositionRepository paymentPositionReposit
 		this.paymentPositionRepository = paymentPositionRepository;
 	}
 
-    @Scheduled(cron = "${cron.job.schedule.expression.historicization.debt.positions}")
-    @SchedulerLock(name = "HistoricizationScheduler_manageDebtPositionsToHistoricize", lockAtMostFor = "180m", lockAtLeastFor = "15m")
+    @Scheduled(cron = "${cron.job.schedule.history.trigger}")
+    @SchedulerLock(name = "HistoricizationScheduler_manageDebtPositionsToHistoricize", lockAtMostFor = "${cron.job.schedule.history.shedlock.lockatmostfor}", 
+    lockAtLeastFor = "${cron.job.schedule.history.shedlock.lockatleastfor}")
     @Async
     @Transactional
     public void manageDebtPositionsToHistoricize() throws JsonProcessingException, TableServiceException {
@@ -112,7 +113,8 @@ public void manageDebtPositionsToHistoricize() throws JsonProcessingException, T
     		// archived debt positions are removed 
         	paymentPositionRepository.deleteAll(ppList);
         	log.info(String.format(LOG_BASE_HEADER_INFO, CRON_JOB, METHOD, "deleted n. "+ppList.size()+" archived debt positions"));
-    	}    	
+    	} 
+    	log.info(String.format(LOG_BASE_HEADER_INFO, CRON_JOB, METHOD, "Finished at " + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(LocalDateTime.now())));
     }
 
     public EntityManager getEntityManager() {
@@ -126,7 +128,7 @@ public TableClient getTableClient(String connectionString, String tableName) {
 			    .buildClient();
     }
 
-	public void saveToPOTable(String organizationFiscalCode, PaymentPosition pp, PaymentOption po) {
+	public void upsertPOTable(String organizationFiscalCode, PaymentPosition pp, PaymentOption po) {
 	    TableClient tableClient = this.getTableClient(archiveStorageConnection, archiveStoragePOTable);
 		TableEntity tableEntity = new TableEntity(organizationFiscalCode, po.getIuv());
 		try {
@@ -148,7 +150,7 @@ public void saveToPOTable(String organizationFiscalCode, PaymentPosition pp, Pay
 		}
 	}
 
-	public void saveToPPTable(String organizationFiscalCode, PaymentPosition pp, ObjectMapper objectMapper) throws JsonProcessingException {
+	public void upsertPPTable(String organizationFiscalCode, PaymentPosition pp, ObjectMapper objectMapper) throws JsonProcessingException {
 		TableClient tableClient = this.getTableClient(archiveStorageConnection, archiveStoragePPTable);
 		TableEntity tableEntity = new TableEntity(organizationFiscalCode, pp.getIupd());
 		try {
@@ -181,10 +183,10 @@ private void archivesDebtPositions(List<PaymentPosition> ppList) throws JsonProc
             for (PaymentPosition pp: organizationPpList) {
             	pp.getPaymentOption().forEach(po -> 
             		// write on azure table storage to persist the PO debt position info
-                	this.saveToPOTable(entry.getKey(), pp, po)
+                	this.upsertPOTable(entry.getKey(), pp, po)
             	);
             	// write on azure table storage to persist the PP debt position info and json
-            	this.saveToPPTable(entry.getKey(), pp, objectMapper);
+            	this.upsertPPTable(entry.getKey(), pp, objectMapper);
             }
             log.debug(String.format(LOG_BASE_HEADER_INFO, CRON_JOB, "archivesDebtPositions", "historicized n. "+organizationPpList.size()+" debt positions for the organization fiscal code: " +entry.getKey()));
         }

src/main/resources/application-h2.properties

@@ -36,20 +36,24 @@ cron.job.schedule.enabled=false
 cron.job.schedule.expression.valid.status=*/35 * * * * *
 cron.job.schedule.expression.expired.status=*/35 * * * * *
 
-cron.job.schedule.historicization.enabled=true
-cron.job.schedule.expression.historicization.debt.positions=*/55 * * * * *
-cron.job.schedule.extraction.history.query =SELECT pp FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1
-cron.job.schedule.extraction.history.query.paginated = true
+cron.job.schedule.history.enabled=true
+cron.job.schedule.history.trigger=*/55 * * * * *
+cron.job.schedule.history.query =SELECT pp FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1
+cron.job.schedule.history.paginated = true
 # The number of records extracted each time the query runs. The parameter is used only if pagination mode is true (default 100.000)
-cron.job.schedule.extraction.history.query.page.size = 5
+cron.job.schedule.history.query.page.size = 5
 # The query to get the number of total records to be extracted in paginated mode. The parameter is used only if pagination mode is true
-cron.job.schedule.extraction.history.query.count = SELECT count(pp.id) FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1
+cron.job.schedule.history.query.count = SELECT count(pp.id) FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1
 # The time, in days, to be subtracted from the current date to decide how old, the debt positions to be historicized, must be (default 365 days)
-cron.job.schedule.extraction.history.query.interval=0
+cron.job.schedule.history.query.interval=0
+
+cron.job.schedule.history.shedlock.defaultlockatmostfor = 10s
+cron.job.schedule.history.shedlock.lockatmostfor = 10s
+cron.job.schedule.history.shedlock.lockatleastfor = 1s
 
 azure.archive.storage.connection=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
-azure.archive.storage.table.po=pagopadweugpsarchivesapaymentoptiontable
-azure.archive.storage.table.pp=pagopadweugpsarchivesapaymentpositiontable
+azure.archive.storage.table.po=paymentoptiontable
+azure.archive.storage.table.pp=paymentpositiontable
 
 
 # Flyway settings 

src/main/resources/application-local.properties

@@ -34,6 +34,25 @@ cron.job.schedule.enabled=true
 cron.job.schedule.expression.valid.status=*/35 * * * * *
 cron.job.schedule.expression.expired.status=*/35 * * * * *
 
+cron.job.schedule.history.enabled=true
+cron.job.schedule.history.trigger=*/55 * * * * *
+cron.job.schedule.history.query =SELECT pp FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1
+cron.job.schedule.history.paginated = true
+# The number of records extracted each time the query runs. The parameter is used only if pagination mode is true (default 100.000)
+cron.job.schedule.history.query.page.size = 5
+# The query to get the number of total records to be extracted in paginated mode. The parameter is used only if pagination mode is true
+cron.job.schedule.history.query.count = SELECT count(pp.id) FROM PaymentPosition pp WHERE pp.status IN ('PAID', 'REPORTED', 'INVALID', 'EXPIRED') AND pp.lastUpdatedDate < ?1
+# The time, in days, to be subtracted from the current date to decide how old, the debt positions to be historicized, must be (default 365 days)
+cron.job.schedule.history.query.interval=0
+
+cron.job.schedule.history.shedlock.defaultlockatmostfor = 10s
+cron.job.schedule.history.shedlock.lockatmostfor = 10s
+cron.job.schedule.history.shedlock.lockatleastfor = 1s
+
+azure.archive.storage.connection=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
+azure.archive.storage.table.po=paymentoptiontable
+azure.archive.storage.table.pp=paymentpositiontable
+
 
 # Flyway settings 
 spring.flyway.schemas=apd

src/main/resources/application.properties

@@ -29,20 +29,49 @@ spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialec
 spring.jpa.hibernate.ddl-auto = ${SPRING_JPA_HIBERNATE_DDL_AUTO}
 spring.jpa.hibernate.show-sql = ${SPRING_JPA_HIBERNATE_SHOW_SQL}
 
-# Scheduling configuration
+# Scheduling configuration: change status
 cron.job.schedule.enabled = ${CRON_JOB_SCHEDULE_ENABLED}
 cron.job.schedule.expression.valid.status = ${CRON_JOB_SCHEDULE_EXPRESSION_TO_VALID}
 cron.job.schedule.expression.expired.status = ${CRON_JOB_SCHEDULE_EXPRESSION_TO_EXPIRED}
+# Scheduling configuration: archive debt positions
+cron.job.schedule.history.enabled = ${CRON_JOB_SCHEDULE_HISTORY_ENABLED}
+cron.job.schedule.history.trigger = ${CRON_JOB_SCHEDULE_HISTORY_TRIGGER}
+cron.job.schedule.history.query = ${CRON_JOB_SCHEDULE_HISTORY_QUERY}
+cron.job.schedule.history.paginated = ${CRON_JOB_SCHEDULE_HISTORY_PAGINATED}
+# The number of records extracted each time the query runs. The parameter is used only if pagination mode is true (default 100.000)
+cron.job.schedule.history.query.page.size = ${CRON_JOB_SCHEDULE_HISTORY_QUERY_PAGE_SIZE}
+# The query to get the number of total records to be extracted in paginated mode. The parameter is used only if pagination mode is true
+cron.job.schedule.history.query.count = ${CRON_JOB_SCHEDULE_HISTORY_QUERY_COUNT}
+# The time, in days, to be subtracted from the current date to decide how old, the debt positions to be historicized, must be (default 365 days)
+cron.job.schedule.history.query.interval = ${CRON_JOB_SCHEDULE_HISTORY_QUERY_INTERVAL}
+
+cron.job.schedule.history.shedlock.defaultlockatmostfor = ${CRON_JOB_SCHEDULE_HISTORY_SHEDLOCK_DEFAULTLOCKATMOSTFOR}
+cron.job.schedule.history.shedlock.lockatmostfor = ${CRON_JOB_SCHEDULE_HISTORY_SHEDLOCK_LOCKATMOSTFOR}
+cron.job.schedule.history.shedlock.lockatleastfor = ${CRON_JOB_SCHEDULE_HISTORY_SHEDLOCK_LOCKATLEASTFOR}
+
+azure.archive.storage.connection=${GPD_ARCHIVE_SA_CONNECTION_STRING}
+azure.archive.storage.table.po=${GPD_ARCHIVE_SA_PO_TABLE}
+azure.archive.storage.table.pp=${GPD_ARCHIVE_SA_PP_TABLE}
 
-cron.job.schedule.historicization.enabled = ${CRON_JOB_SCHEDULE_HISTORICIZATION_ENABLED}
-cron.job.schedule.expression.historicization.debt.positions = ${CRON_JOB_SCHEDULE_EXPRESSION_HISTORICIZATION_DEBT_POS}
-cron.job.schedule.extraction.history.query = ${CRON_JOB_SCHEDULE_EXTRACTION_HISTORY_QUERY}
+
+cron.job.schedule.history.enabled = ${CRON_JOB_SCHEDULE_HISTORY_ENABLED}
+cron.job.schedule.history.trigger = ${CRON_JOB_SCHEDULE_HISTORY_TRIGGER}
+cron.job.schedule.history.query = ${CRON_JOB_SCHEDULE_HISTORY_QUERY}
+cron.job.schedule.history.paginated = ${CRON_JOB_SCHEDULE_HISTORY_PAGINATED}
+# The number of records extracted each time the query runs. The parameter is used only if pagination mode is true (default 100.000)
+cron.job.schedule.history.query.page.size = ${CRON_JOB_SCHEDULE_HISTORY_QUERY_PAGE_SIZE}
+# The query to get the number of total records to be extracted in paginated mode. The parameter is used only if pagination mode is true
+cron.job.schedule.history.query.count = ${CRON_JOB_SCHEDULE_HISTORY_QUERY_COUNT}
 # The time, in days, to be subtracted from the current date to decide how old, the debt positions to be historicized, must be (default 365 days)
-cron.job.schedule.extraction.history.query.interval = ${CRON_JOB_SCHEDULE_EXTRACTION_HISTORY_QUERY_INTERVAL}
+cron.job.schedule.history.query.interval= ${CRON_JOB_SCHEDULE_HISTORY_QUERY_INTERVAL}
+
+cron.job.schedule.history.shedlock.defaultlockatmostfor = ${CRON_JOB_SCHEDULE_HISTORY_SHEDLOCK_DEFAULTLOCKATMOSTFOR}
+cron.job.schedule.history.shedlock.lockatmostfor = ${CRON_JOB_SCHEDULE_HISTORY_SHEDLOCK_LOCKATMOSTFOR}
+cron.job.schedule.history.shedlock.lockatleastfor = ${CRON_JOB_SCHEDULE_HISTORY_SHEDLOCK_LOCKATLEASTFOR}
 
-azure.archive.storage.connection=${ARCHIVE_SA_CONNECTION_STRING}
-azure.archive.storage.table.po=${ARCHIVE_SA_PO_TABLE}
-azure.archive.storage.table.pp=${ARCHIVE_SA_PP_TABLE}
+azure.archive.storage.connection=${GPD_ARCHIVE_SA_CONNECTION_STRING}
+azure.archive.storage.table.po=${GPD_ARCHIVE_SA_PO_TABLE}
+azure.archive.storage.table.pp=${GPD_ARCHIVE_SA_PP_TABLE}
 
 
 # Max num. of days for the recovery of debt positions

src/main/resources/db/migration/V011__ADD_SHEDLOCK.sql

@@ -0,0 +1,9 @@
+-- see: https://www.springcloud.io/post/2022-07/shedlock/#gsc.tab=0 
+
+CREATE TABLE shedlock (
+  name VARCHAR(64),
+  lock_until TIMESTAMP(3) NULL,
+  locked_at TIMESTAMP(3) NULL,
+  locked_by VARCHAR(255),
+  PRIMARY KEY (name)
+);