From 682354e850e46cbc9aaff3f5c8d029014c9a2df5 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 5 Aug 2024 13:58:04 +0200 Subject: [PATCH 01/50] postCheckpointEndpointV2 --- .../absa/atum/server/api/http/Endpoints.scala | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 792d68a25..f98f73150 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -25,6 +25,7 @@ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import sttp.tapir.{PublicEndpoint, endpoint} +import za.co.absa.atum.server.api.http.ApiPaths.V2Paths trait Endpoints extends BaseEndpoints { @@ -36,13 +37,23 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[CheckpointDTO]) } - protected val createCheckpointEndpointV2 - : PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { +// protected val createCheckpointEndpointV2 +// : PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { +// apiV2.post +// .in(CreateCheckpoint) +// .in(jsonBody[CheckpointDTO]) +// .out(statusCode(StatusCode.Created)) +// .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) +// } + + protected val postCheckpointEndpointV2 + : PublicEndpoint[(Long, CheckpointDTO), ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String), Any] = { apiV2.post - .in(CreateCheckpoint) + .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints) .in(jsonBody[CheckpointDTO]) .out(statusCode(StatusCode.Created)) .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) + .out(header[String]("Location")) } protected val createPartitioningEndpointV1 From b72ebfadea1af27b4a4f824ce1fe562c4fcd4b88 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 11:29:07 +0200 Subject: [PATCH 02/50] postCheckpointV2 --- .../runs/V1.9.2__write_checkpoint_v2.sql | 87 ++++++ .../WriteCheckpointV2IntegrationTests.scala | 264 ++++++++++++++++++ .../scala/za/co/absa/atum/server/Main.scala | 1 + .../api/controller/BaseController.scala | 28 +- .../api/controller/CheckpointController.scala | 5 +- .../controller/CheckpointControllerImpl.scala | 19 +- .../runs/functions/WriteCheckpointV2.scala | 55 ++++ .../atum/server/api/exception/AppError.scala | 10 +- .../atum/server/api/http/BaseEndpoints.scala | 14 +- .../absa/atum/server/api/http/Endpoints.scala | 12 +- .../co/absa/atum/server/api/http/Routes.scala | 20 +- .../api/repository/BaseRepository.scala | 19 +- .../api/repository/CheckpointRepository.scala | 2 +- .../repository/CheckpointRepositoryImpl.scala | 18 +- .../atum/server/api/service/BaseService.scala | 11 +- .../api/service/CheckpointService.scala | 2 +- .../api/service/CheckpointServiceImpl.scala | 7 + .../server/api/service/FlowServiceImpl.scala | 4 +- .../api/service/PartitioningServiceImpl.scala | 4 +- .../server/model/WriteCheckpointV2Args.scala | 21 ++ .../CreateCheckpointEndpointUnitTests.scala | 8 +- 21 files changed, 555 insertions(+), 56 deletions(-) create mode 100644 database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql create mode 100644 database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala create mode 100644 server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql new file mode 100644 index 000000000..ba5373b0b --- /dev/null +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql @@ -0,0 +1,87 @@ +CREATE OR REPLACE FUNCTION runs.write_checkpoint_v2( + IN i_partitioning_id BIGINT, + IN i_id_checkpoint UUID, + IN i_checkpoint_name TEXT, + IN i_process_start_time TIMESTAMP WITH TIME ZONE, + IN i_process_end_time TIMESTAMP WITH TIME ZONE, + IN i_measurements JSONB[], + IN i_measured_by_atum_agent BOOLEAN, + IN i_by_user TEXT, + OUT status INTEGER, + OUT status_text TEXT +) RETURNS record AS +$$ + ------------------------------------------------------------------------------- +-- +-- Function: runs.write_checkpoint_v2(10) +-- Creates a checkpoint and adds all the measurements that it consists of +-- +-- Parameters: +-- i_partitioning_id - ID of the partitioning the measure belongs to +-- i_id_checkpoint - reference to the checkpoint this measure belongs into +-- i_checkpoint_name - name of the checkpoint +-- i_process_start_time - the start of processing (measuring) of the checkpoint +-- i_process_end_time - the end of the processing (measuring) of the checkpoint +-- i_measurements - array of JSON objects of the following format (values of the keys are examples only) +-- { +-- "measure": { +-- "measureName": "count", +-- "measuredColumns": ["a","b"] +-- }, +-- "result": { +-- whatever here +-- } +-- } +-- i_measured_by_atum_agent - flag it the checkpoint was measured by Atum or data provided by user +-- i_by_user - user behind the change +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- +-- Status codes: +-- 11 - Checkpoint created +-- 31 - Conflict, checkpoint already present +-- +------------------------------------------------------------------------------- +BEGIN + + PERFORM 1 + FROM runs.checkpoints CP + WHERE CP.id_checkpoint = i_id_checkpoint; + + IF found THEN + status := 31; + status_text := 'Checkpoint already present'; + RETURN; + END IF; + + INSERT INTO runs.checkpoints (id_checkpoint, fk_partitioning, + checkpoint_name, measured_by_atum_agent, + process_start_time, process_end_time, created_by) + VALUES (i_id_checkpoint, i_partitioning_id, + i_checkpoint_name, i_measured_by_atum_agent, + i_process_start_time, i_process_end_time, i_by_user); + + -- maybe could use `jsonb_populate_record` function to be little bit more effective + PERFORM runs._write_measurement( + i_id_checkpoint, + i_partitioning_id, + UN.measurement->'measure'->>'measureName', + jsonb_array_to_text_array(UN.measurement->'measure'->'measuredColumns'), + UN.measurement->'result', + i_by_user + ) + FROM ( + SELECT unnest(i_measurements) AS measurement + ) UN; + + status := 11; + status_text := 'Checkpoint created'; + RETURN; +END; +$$ + LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +ALTER FUNCTION runs.write_checkpoint(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.write_checkpoint(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; \ No newline at end of file diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala new file mode 100644 index 000000000..cb386287d --- /dev/null +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala @@ -0,0 +1,264 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.database.runs + +import za.co.absa.balta.DBTestSuite +import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.setter.CustomDBType + +import java.time.OffsetDateTime +import java.util.UUID + +class WriteCheckpointV2IntegrationTests extends DBTestSuite { + + private val partitioning = JsonBString( + """ + |{ + | "version": 1, + | "keys": ["key1", "key3", "key2", "key4"], + | "keysToValues": { + | "key1": "valueX", + | "key2": "valueY", + | "key3": "valueZ", + | "key4": "valueA" + | } + |} + |""".stripMargin + ) + + test("Write new checkpoint without data") { + val uuid = UUID.randomUUID + val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") + val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") + + + table("runs.partitionings").insert( + add("partitioning", partitioning) + .add("created_by", "John von Neumann") + ) + //DBTable's insert doesn't return the values yet correctly + val fkPartitioning: Long = table("runs.partitionings").fieldValue("partitioning", partitioning, "id_partitioning").get.get + + table("runs.measure_definitions").insert( + add("fk_partitioning", fkPartitioning) + .add("measure_name", "avg") + .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) + .add("created_by", "Aristoteles") + ) + + assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) + + function("runs.write_checkpoint_v2") + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_id_checkpoint", uuid) + .setParam("i_checkpoint_name", "Empty path") + .setParam("i_process_start_time", startTime) + .setParam("i_process_end_time", endTime) + .setParam("i_measurements", CustomDBType("{}", "JSONB[]")) + .setParam("i_measured_by_atum_agent", true) + .setParam("i_by_user", "J. Robert Oppenheimer") + .execute { queryResult => + assert(queryResult.hasNext) + val row = queryResult.next() + assert(row.getInt("status").contains(11)) + assert(row.getString("status_text").contains("Checkpoint created")) + } + + assert(table("runs.measure_definitions").count(add("fk_partitioning", fkPartitioning)) == 1) + assert(table("runs.measurements").count(add("fk_checkpoint", uuid)) == 0) + assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 1) + table("runs.checkpoints").where(add("fk_partitioning", fkPartitioning)) {resultSet => + val row = resultSet.next() + assert(row.getString("checkpoint_name").contains("Empty path")) + assert(row.getOffsetDateTime("process_start_time").contains(startTime)) + assert(row.getOffsetDateTime("process_end_time").contains(endTime)) + assert(row.getBoolean("measured_by_atum_agent").contains(true)) + assert(row.getString("created_by").contains("J. Robert Oppenheimer")) + assert(row.getOffsetDateTime("created_at").contains(now())) + } + + } + + test("Write new checkpoint"){ + val uuid = UUID.randomUUID + val user = "Franz Kafka" + val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") + val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") + val measurements = + """ + |{ + | "{ + | \"measure\": { + | \"measureName\": \"count\", + | \"measuredColumns\": [] + | }, + | \"result\":{ + | \"value\":\"3\", + | \"type\":\"int\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"col1\"] + | }, + | \"result\":{ + | \"value\":\"3.14\", + | \"type\":\"double\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"a\",\"b\"] + | }, + | \"result\":{ + | \"value\":\"2.71\", + | \"type\":\"double\" + | } + | }" + |} + |""".stripMargin + + table("runs.partitionings").insert( + add("partitioning", partitioning) + .add("created_by", user) + ) + //DBTable's insert doesn't return the values yet correctly + val fkPartitioning: Long = table("runs.partitionings").fieldValue("partitioning", partitioning, "id_partitioning").get.get + + table("runs.measure_definitions").insert( + add("fk_partitioning", fkPartitioning) + .add("measure_name", "avg") + .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) + .add("created_by", "Aristoteles") + ) + + assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) + + function("runs.write_checkpoint_v2") + .setParam("i_partitioning", fkPartitioning) + .setParam("i_id_checkpoint", uuid) + .setParam("i_checkpoint_name", "Happy path") + .setParam("i_process_start_time", startTime) + .setParam("i_process_end_time", endTime) + .setParam("i_measurements", CustomDBType(measurements, "JSONB[]")) + .setParam("i_measured_by_atum_agent", false) + .setParam("i_by_user", user) + .execute { queryResult => + assert(queryResult.hasNext) + val row = queryResult.next() + assert(row.getInt("status").contains(11)) + assert(row.getString("status_text").contains("Checkpoint created")) + } + + assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 1) + assert(table("runs.measure_definitions").count(add("fk_partitioning", fkPartitioning)) == 3) + assert(table("runs.measurements").count(add("fk_checkpoint", uuid)) == 3) + table("runs.checkpoints").where(add("fk_partitioning", fkPartitioning)) { resultSet => + val row = resultSet.next() + assert(row.getString("checkpoint_name").contains("Happy path")) + assert(row.getOffsetDateTime("process_start_time").contains(startTime)) + assert(row.getOffsetDateTime("process_end_time").contains(endTime)) + assert(row.getBoolean("measured_by_atum_agent").contains(false)) + assert(row.getString("created_by").contains(user)) + assert(row.getOffsetDateTime("created_at").contains(now())) + } + + val measureDefinitionIds = table("runs.measure_definitions") + .where(add("fk_partitioning", fkPartitioning),"id_measure_definition") { resultSet => + val row1 = resultSet.next() + val result1: Vector[Long] = Vector(row1.getLong("id_measure_definition").get) + assert(row1.getString("measure_name").contains("avg")) + assert(row1.getArray[String]("measured_columns").map(_.toList).contains(List("col1"))) + assert(row1.getString("created_by").contains("Aristoteles")) + assert(row1.getOffsetDateTime("created_at").contains(now())) + val row2 = resultSet.next() + val result2: Vector[Long] = result1 :+ row2.getLong("id_measure_definition").get + assert(row2.getString("measure_name").contains("count")) + assert(row2.getArray[String]("measured_columns").map(_.toList).contains(List.empty)) + assert(row2.getString("created_by").contains(user)) + assert(row2.getOffsetDateTime("created_at").contains(now())) + val row3 = resultSet.next() + val result3: Vector[Long] = result2 :+ row3.getLong("id_measure_definition").get + assert(row3.getString("measure_name").contains("avg")) + assert(row3.getArray[String]("measured_columns").map(_.toList).contains(List("a", "b"))) + assert(row3.getString("created_by").contains(user)) + assert(row3.getOffsetDateTime("created_at").contains(now())) + result3 + } + table("runs.measurements").where(add("fk_checkpoint", uuid), "id_measurement") { resultSet => + val row1 = resultSet.next() + // because measure definition of `count` was created only after the manual enter of the `avg`, it's actually only + // second in the list + assert(row1.getLong("fk_measure_definition").contains(measureDefinitionIds(1))) + assert(row1.getJsonB("measurement_value").contains(JsonBString("""{"type": "int", "value": "3"}"""))) + val row2 = resultSet.next() + assert(row2.getLong("fk_measure_definition").contains(measureDefinitionIds(0))) + assert(row2.getJsonB("measurement_value").contains(JsonBString("""{"type": "double", "value": "3.14"}"""))) + val row3 = resultSet.next() + assert(row3.getLong("fk_measure_definition").contains(measureDefinitionIds(2))) + assert(row3.getJsonB("measurement_value").contains(JsonBString("""{"type": "double", "value": "2.71"}"""))) + } + } + + test("Checkpoint already exists") { + val uuid = UUID.randomUUID + val origAuthor = "John von Neumann" + table("runs.partitionings").insert( + add("partitioning", partitioning) + .add("created_by", origAuthor) + ) + + //DBTable's insert doesn't return the values yet correctly + val fkPartitioning: Long = table("runs.partitionings").fieldValue("partitioning", partitioning, "id_partitioning").get.get + + table("runs.checkpoints").insert( + add("id_checkpoint", uuid) + .add("fk_partitioning", fkPartitioning) + .add("checkpoint_name", "I came before") + .add("process_start_time", now()) + .add("process_end_time", now()) + .add("measured_by_atum_agent", false) + .add("created_by", origAuthor) + ) + + function("runs.write_checkpoint_v2") + .setParam("i_partitioning", fkPartitioning) + .setParam("i_id_checkpoint", uuid) + .setParam("i_checkpoint_name", "Won't go in") + .setParam("i_process_start_time", now()) + .setParamNull("i_process_end_time") + .setParamNull("i_measurements") + .setParam("i_measured_by_atum_agent", true) + .setParam("i_by_user", "J. Robert Oppenheimer") + .execute { queryResult => + assert(queryResult.hasNext) + val row = queryResult.next() + assert(row.getInt("status").contains(31)) + assert(row.getString("status_text").contains("Checkpoint already present")) + } + + table("runs.checkpoints").where(add("id_checkpoint", uuid)){queryResult => + val row = queryResult.next() + assert(row.getString("checkpoint_name").contains("I came before")) + assert(row.getBoolean("measured_by_atum_agent").contains(false)) + assert(row.getString("created_by").contains(origAuthor)) + } + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 19ab07fb8..5a07279b6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -56,6 +56,7 @@ object Main extends ZIOAppDefault with Server { CreateOrUpdateAdditionalData.layer, GetPartitioningCheckpoints.layer, WriteCheckpoint.layer, + WriteCheckpointV2.layer, GetFlowCheckpoints.layer, PostgresDatabaseProvider.layer, TransactorProvider.layer, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 88c133487..1a97b6b9b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -16,8 +16,9 @@ package za.co.absa.atum.server.api.controller -import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.atum.server.model.{ErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.api.exception.{ConflictServiceError, GeneralServiceError, ServiceError} +import za.co.absa.atum.server.config.SslConfig +import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ @@ -29,8 +30,9 @@ trait BaseController { ): IO[ErrorResponse, B] = { serviceCall - .mapError { serviceError: ServiceError => - InternalServerErrorResponse(serviceError.message) + .mapError { + case ConflictServiceError(message) => ConflictErrorResponse(message) + case GeneralServiceError(message) => InternalServerErrorResponse(message) } .flatMap { result => ZIO.succeed(onSuccessFnc(result)) @@ -49,4 +51,22 @@ trait BaseController { ): IO[ErrorResponse, MultiSuccessResponse[A]] = { effect.map(MultiSuccessResponse(_)) } + + protected def createResourceUri(parts: Seq[String]): IO[ErrorResponse, String] = { + for { + hostname <- System.env("HOSTNAME") + .orElseFail(InternalServerErrorResponse("Failed to get hostname")) + .flatMap { + case Some(value) => ZIO.succeed(value) + case None => ZIO.fail(InternalServerErrorResponse("Failed to get hostname")) + } + sslConfig <- ZIO.config[SslConfig](SslConfig.config) + .orElseFail(InternalServerErrorResponse("Failed to get SSL config")) + } yield { + val protocol = if (sslConfig.enabled) "https" else "http" + val port = if (sslConfig.enabled) 8443 else 8080 + val path = parts.mkString("/") + s"$protocol://$hostname:$port/$path" + } + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index 6b547c3cb..d4bb2229d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -27,8 +27,9 @@ trait CheckpointController { def createCheckpointV1(checkpointDTO: CheckpointDTO): IO[ErrorResponse, CheckpointDTO] - def createCheckpointV2( + def postCheckpointV2( + partitioningId: Long, checkpointDTO: CheckpointDTO - ): IO[ErrorResponse, SingleSuccessResponse[CheckpointDTO]] + ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String)] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index b64c825a7..44fc93372 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -17,6 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse @@ -33,12 +34,22 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che ) } - override def createCheckpointV2( + override def postCheckpointV2( + partitioningId: Long, checkpointDTO: CheckpointDTO - ): IO[ErrorResponse, SingleSuccessResponse[CheckpointDTO]] = { - mapToSingleSuccessResponse(createCheckpointV1(checkpointDTO)) + ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String)] = { + for { + response <- mapToSingleSuccessResponse( + serviceCall[Unit, CheckpointDTO]( + checkpointService.saveCheckpointV2(partitioningId, checkpointDTO), + _ => checkpointDTO + ) + ) + uri <- createResourceUri( + Seq(V2Paths.Partitionings, partitioningId.toString, V2Paths.Checkpoints, checkpointDTO.id.toString) + ) + } yield (response, uri) } - } object CheckpointControllerImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala new file mode 100644 index 000000000..f440ec6f7 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.database.runs.functions + +import doobie.implicits.toSqlInterpolator +import za.co.absa.atum.server.model.WriteCheckpointV2Args +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling +import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.atum.server.api.database.runs.Runs +import zio._ +import io.circe.syntax._ + +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut +import doobie.postgres.implicits._ + +class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => + Seq( + fr"${args.partitioningId}", + fr"${args.checkpointDTO.id}", + fr"${args.checkpointDTO.name}", + fr"${args.checkpointDTO.processStartTime}", + fr"${args.checkpointDTO.processEndTime}", + fr"${args.checkpointDTO.measurements.toList.map(_.asJson)}", + fr"${args.checkpointDTO.measuredByAtumAgent}", + fr"${args.checkpointDTO.author}" + ) + ) + with StandardStatusHandling + +object WriteCheckpointV2 { + val layer: URLayer[PostgresDatabaseProvider, WriteCheckpointV2] = ZLayer { + for { + dbProvider <- ZIO.service[PostgresDatabaseProvider] + } yield new WriteCheckpointV2()(Runs, dbProvider.dbEngine) + } +} + diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala index c4b129b94..106d647f2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala @@ -20,5 +20,11 @@ sealed trait AppError extends Throwable { def message: String } -case class DatabaseError(message: String) extends AppError -case class ServiceError(message: String) extends AppError +sealed trait DatabaseError extends AppError +case class GeneralDatabaseError(message: String) extends DatabaseError +case class ConflictDatabaseError(message: String) extends DatabaseError + +sealed trait ServiceError extends AppError + +case class GeneralServiceError(message: String) extends ServiceError +case class ConflictServiceError(message: String) extends ServiceError diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index 3ef8edfb1..04b3c228a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -19,12 +19,7 @@ package za.co.absa.atum.server.api.http import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody -import za.co.absa.atum.server.model.{ - BadRequestResponse, - ErrorResponse, - GeneralErrorResponse, - InternalServerErrorResponse -} +import za.co.absa.atum.server.model.{BadRequestResponse, ConflictErrorResponse, ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import sttp.tapir.typelevel.MatchType import sttp.tapir.ztapir._ import sttp.tapir.{EndpointOutput, PublicEndpoint} @@ -36,6 +31,13 @@ trait BaseEndpoints { implicit val uuidMatchType: MatchType[UUID] = (a: Any) => a.isInstanceOf[UUID] + protected val conflictErrorOneOfVariant: EndpointOutput.OneOfVariant[ConflictErrorResponse] = { + oneOfVariantFromMatchType( + StatusCode.Conflict, + jsonBody[ConflictErrorResponse] + ) + } + private val badRequestOneOfVariant: EndpointOutput.OneOfVariant[BadRequestResponse] = { oneOfVariantFromMatchType( StatusCode.BadRequest, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index f98f73150..78800ae37 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -25,7 +25,7 @@ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import sttp.tapir.{PublicEndpoint, endpoint} -import za.co.absa.atum.server.api.http.ApiPaths.V2Paths +import za.co.absa.atum.server.api.http.ApiPaths.{V1Paths, V2Paths} trait Endpoints extends BaseEndpoints { @@ -37,15 +37,6 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[CheckpointDTO]) } -// protected val createCheckpointEndpointV2 -// : PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { -// apiV2.post -// .in(CreateCheckpoint) -// .in(jsonBody[CheckpointDTO]) -// .out(statusCode(StatusCode.Created)) -// .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) -// } - protected val postCheckpointEndpointV2 : PublicEndpoint[(Long, CheckpointDTO), ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String), Any] = { apiV2.post @@ -54,6 +45,7 @@ trait Endpoints extends BaseEndpoints { .out(statusCode(StatusCode.Created)) .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) .out(header[String]("Location")) + .errorOutVariantPrepend(conflictErrorOneOfVariant) } protected val createPartitioningEndpointV1 diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 1da88efce..af229a453 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -19,14 +19,19 @@ package za.co.absa.atum.server.api.http import cats.syntax.semigroupk._ import org.http4s.HttpRoutes import sttp.tapir.PublicEndpoint +import sttp.tapir.model.ServerRequest import sttp.tapir.server.http4s.Http4sServerInterpreter import sttp.tapir.server.http4s.ztapir.ZHttp4sServerInterpreter import sttp.tapir.server.interceptor.metrics.MetricsRequestInterceptor import sttp.tapir.swagger.bundle.SwaggerInterpreter import sttp.tapir.ztapir._ +import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.Constants.{SwaggerApiName, SwaggerApiVersion} -import za.co.absa.atum.server.api.controller.{CheckpointController, PartitioningController, FlowController} +import za.co.absa.atum.server.api.controller.{CheckpointController, FlowController, PartitioningController} +import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.config.{HttpMonitoringConfig, JvmMonitoringConfig} +import za.co.absa.atum.server.model.ErrorResponse +import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.interop.catz._ import zio.metrics.connectors.prometheus.PrometheusPublisher @@ -39,7 +44,16 @@ trait Routes extends Endpoints with ServerOptions { } val endpoints = List( createServerEndpoint(createCheckpointEndpointV1, CheckpointController.createCheckpointV1), - createServerEndpoint(createCheckpointEndpointV2, CheckpointController.createCheckpointV2), + createServerEndpoint[ + (Long, CheckpointDTO), + ErrorResponse, + (SingleSuccessResponse[CheckpointDTO], String) + ]( + postCheckpointEndpointV2, + { case (partitioningId: Long, checkpointDTO: CheckpointDTO) => + CheckpointController.postCheckpointV2(partitioningId, checkpointDTO) + } + ), createServerEndpoint(createPartitioningEndpointV1, PartitioningController.createPartitioningIfNotExistsV1), createServerEndpoint(createPartitioningEndpointV2, PartitioningController.createPartitioningIfNotExistsV2), createServerEndpoint( @@ -59,7 +73,7 @@ trait Routes extends Endpoints with ServerOptions { private def createSwaggerRoutes: HttpRoutes[HttpEnv.F] = { val endpoints = List( createCheckpointEndpointV1, - createCheckpointEndpointV2, + postCheckpointEndpointV2, createPartitioningEndpointV1, createPartitioningEndpointV2, createOrUpdateAdditionalDataEndpointV2, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 9f9764f91..c9ce33570 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -16,8 +16,8 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.db.fadb.exceptions.StatusException +import za.co.absa.atum.server.api.exception.{ConflictDatabaseError, DatabaseError, GeneralDatabaseError} +import za.co.absa.db.fadb.exceptions.{DataConflictException, StatusException} import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} import zio._ @@ -32,7 +32,7 @@ trait BaseRepository { case Left(statusException: StatusException) => ZIO.logError( s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}), ${statusException.status.statusText}" + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" ) case Right(_) => ZIO.logDebug(s"Operation '$operationName' succeeded in database") } @@ -40,12 +40,15 @@ trait BaseRepository { private def defaultErrorHandler(operationName: String): PartialFunction[Throwable, DatabaseError] = { case statusException: StatusException => - DatabaseError( - s"Exception caused by operation: '$operationName': " + - s"(${statusException.status.statusCode}) ${statusException.status.statusText}" - ) + val message = s"Exception caused by operation: '$operationName': " + + s"(${statusException.status.statusCode}) ${statusException.status.statusText}" + + statusException match { + case DataConflictException(_) => ConflictDatabaseError(message) + case _ => GeneralDatabaseError(message) + } case error => - DatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") + GeneralDatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") } def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): IO[DatabaseError, R] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index 59c33d1b6..a2237a98b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -24,5 +24,5 @@ import zio.macros.accessible @accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] - + def writeCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index e63d92c9f..9d117c663 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,23 +17,33 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint +import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.model.WriteCheckpointV2Args import zio._ import zio.interop.catz.asyncInstance -class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint) extends CheckpointRepository with BaseRepository { +class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint, writeCheckpointV2Fn: WriteCheckpointV2) + extends CheckpointRepository + with BaseRepository { override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") } + override def writeCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { + dbSingleResultCallWithStatus( + writeCheckpointV2Fn(WriteCheckpointV2Args(partitioningId, checkpointDTO)), + "writeCheckpoint" + ) + } } object CheckpointRepositoryImpl { - val layer: URLayer[WriteCheckpoint, CheckpointRepository] = ZLayer { + val layer: URLayer[WriteCheckpoint with WriteCheckpointV2, CheckpointRepository] = ZLayer { for { writeCheckpoint <- ZIO.service[WriteCheckpoint] - } yield new CheckpointRepositoryImpl(writeCheckpoint) + writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] + } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala index 680dcc22d..0ca695299 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala @@ -16,16 +16,21 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} +import za.co.absa.atum.server.api.exception._ import zio._ trait BaseService { def repositoryCall[R](repositoryCall: IO[DatabaseError, R], operationName: String): IO[ServiceError, R] = { repositoryCall - .mapError { case DatabaseError(message) => - ServiceError(s"Failed to perform '$operationName': $message") + .mapError { + case ConflictDatabaseError(message) => ConflictServiceError(createMessage(operationName, message)) + case GeneralDatabaseError(message) => GeneralServiceError(createMessage(operationName, message)) } } + private def createMessage(operationName: String, message: String): String = { + s"Failed to perform '$operationName': $message" + } + } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index a38811890..ce026ff05 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -24,5 +24,5 @@ import zio.macros.accessible @accessible trait CheckpointService { def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] - + def saveCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index aae123ea4..f8e772c2c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -30,6 +30,13 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends ) } + override def saveCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] = { + repositoryCall( + checkpointRepository.writeCheckpointV2(partitioningId, checkpointDTO), + "saveCheckpoint" + ) + } + } object CheckpointServiceImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala index 5f788b86c..f768a3262 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.repository.FlowRepository import za.co.absa.atum.server.model.CheckpointFromDB import zio._ @@ -33,7 +33,7 @@ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with B checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { checkpointFromDB => ZIO .fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) - .mapError(error => ServiceError(error.getMessage)) + .mapError(error => GeneralServiceError(error.getMessage)) } } yield checkpointDTOs } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index a4978718d..e4617879f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.repository.PartitioningRepository import za.co.absa.atum.server.model.CheckpointFromDB import zio._ @@ -65,7 +65,7 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) checkpointDTOs <- ZIO.foreach(checkpointsFromDB) { checkpointFromDB => ZIO .fromEither(CheckpointFromDB.toCheckpointDTO(checkpointQueryDTO.partitioning, checkpointFromDB)) - .mapError(error => ServiceError(error.getMessage)) + .mapError(error => GeneralServiceError(error.getMessage)) } } yield checkpointDTOs diff --git a/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala b/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala new file mode 100644 index 000000000..1a7a80828 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.model + +import za.co.absa.atum.model.dto.CheckpointDTO + +case class WriteCheckpointV2Args(partitioningId: Long, checkpointDTO: CheckpointDTO) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 0621d7ca4..11d5eeec6 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -36,17 +36,17 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w private val checkpointControllerMock = mock(classOf[CheckpointController]) - when(checkpointControllerMock.createCheckpointV2(checkpointDTO1)) + when(checkpointControllerMock.postCheckpointV2(checkpointDTO1)) .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointDTO1, uuid))) - when(checkpointControllerMock.createCheckpointV2(checkpointDTO2)) + when(checkpointControllerMock.postCheckpointV2(checkpointDTO2)) .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) - when(checkpointControllerMock.createCheckpointV2(checkpointDTO3)) + when(checkpointControllerMock.postCheckpointV2(checkpointDTO3)) .thenReturn(ZIO.fail(InternalServerErrorResponse("error"))) private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) private val createCheckpointServerEndpoint = createCheckpointEndpointV2 - .zServerLogic(CheckpointController.createCheckpointV2) + .zServerLogic(CheckpointController.postCheckpointV2) def spec: Spec[TestEnvironment with Scope, Any] = { val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) From 7b7819c2834baed65334c22d06ccc8d699bae7ea Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 11:36:43 +0200 Subject: [PATCH 03/50] write_checkpoint_v2 --- .../src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql index ba5373b0b..420d132b4 100644 --- a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql @@ -83,5 +83,5 @@ END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.write_checkpoint(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.write_checkpoint(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; \ No newline at end of file +ALTER FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; \ No newline at end of file From b4604c4bfec49c61d64b06b9272b346df328cf27 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 11:44:04 +0200 Subject: [PATCH 04/50] fix existing tests --- .../repository/CheckpointRepositoryUnitTests.scala | 13 +++++++------ .../PartitioningRepositoryUnitTests.scala | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index e7af6ecf1..a3da98e49 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,8 +17,8 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint -import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError} import za.co.absa.atum.server.api.TestData import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.FunctionStatus @@ -26,19 +26,20 @@ import zio._ import zio.interop.catz.asyncInstance import zio.test.Assertion.{failsWithA, isUnit} import zio.test._ - import za.co.absa.db.fadb.status.Row object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) + private val writeCheckpointMockV2: WriteCheckpointV2 = mock(classOf[WriteCheckpointV2]) when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(writeCheckpointMock.apply(checkpointDTO2)) - .thenReturn(ZIO.fail(DatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) + private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointMockV2) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -53,14 +54,14 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { for { result <- CheckpointRepository.writeCheckpoint(checkpointDTO2).exit } yield assertTrue( - result == Exit.fail(DatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null")) + result == Exit.fail(GeneralDatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null")) ) }, test("Returns expected DatabaseError") { assertZIO(CheckpointRepository.writeCheckpoint(checkpointDTO3).exit)(failsWithA[DatabaseError]) } ) - ).provide(CheckpointRepositoryImpl.layer, writeCheckpointMockLayer) + ).provide(CheckpointRepositoryImpl.layer, writeCheckpointMockLayer, writeCheckpointV2MockLayer) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index 1c054613f..b03c80236 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions._ -import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError} import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ @@ -63,7 +63,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { Seq(Row(FunctionStatus(0, "success"), measureFromDB1), Row(FunctionStatus(0, "success"), measureFromDB2)) ) ) - when(getPartitioningMeasuresMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) + when(getPartitioningMeasuresMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val getPartitioningMeasuresMockLayer = ZLayer.succeed(getPartitioningMeasuresMock) @@ -72,7 +72,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getPartitioningAdditionalDataMock.apply(partitioningDTO1)) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(0, "success"), AdditionalDataFromDB(Some("key"), Some("value")))))) - when(getPartitioningAdditionalDataMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) + when(getPartitioningAdditionalDataMock.apply(partitioningDTO2)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val getPartitioningAdditionalDataMockLayer = ZLayer.succeed(getPartitioningAdditionalDataMock) @@ -82,7 +82,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO1)) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(0, "success"), checkpointFromDB1)))) when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO3)).thenReturn(ZIO.right(Seq.empty)) - when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO2)).thenReturn(ZIO.fail(DatabaseError("boom!"))) + when(getPartitioningCheckpointsMock.apply(checkpointQueryDTO2)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val getPartitioningCheckpointsMockLayer = ZLayer.succeed(getPartitioningCheckpointsMock) @@ -100,7 +100,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { result <- PartitioningRepository.createPartitioningIfNotExists(partitioningSubmitDTO2).exit } yield assertTrue( result == Exit.fail( - DatabaseError( + GeneralDatabaseError( "Exception caused by operation: 'createPartitioningIfNotExists': (50) error in Partitioning data" ) ) @@ -123,7 +123,7 @@ object PartitioningRepositoryUnitTests extends ZIOSpecDefault with TestData { result <- PartitioningRepository.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit } yield assertTrue( result == Exit.fail( - DatabaseError("Exception caused by operation: 'createOrUpdateAdditionalData': (50) error in AD data") + GeneralDatabaseError("Exception caused by operation: 'createOrUpdateAdditionalData': (50) error in AD data") ) ) }, From 4b89a3aed1ff958f423fea0bc417902946be2491 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 12:02:58 +0200 Subject: [PATCH 05/50] fixes related to error model --- .../CheckpointControllerUnitTests.scala | 14 +- .../controller/FlowControllerUnitTests.scala | 4 +- .../PartitioningControllerUnitTests.scala | 8 +- .../CreateCheckpointEndpointUnitTests.scala | 156 +++++++++--------- .../service/CheckpointServiceUnitTests.scala | 10 +- .../api/service/FlowServiceUnitTests.scala | 4 +- .../PartitioningServiceUnitTests.scala | 24 +-- 7 files changed, 110 insertions(+), 110 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 7554742eb..2e167be8d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.{ConflictServiceError, GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.server.model.InternalServerErrorResponse import zio.test.Assertion.failsWithA @@ -29,11 +29,11 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { private val checkpointServiceMock = mock(classOf[CheckpointService]) - when(checkpointServiceMock.saveCheckpoint(checkpointDTO1)).thenReturn(ZIO.succeed(())) + when(checkpointServiceMock.saveCheckpoint(checkpointDTO1)).thenReturn(ZIO.unit) when(checkpointServiceMock.saveCheckpoint(checkpointDTO2)) - .thenReturn(ZIO.fail(ServiceError("error in data"))) + .thenReturn(ZIO.fail(ConflictServiceError("error in data"))) when(checkpointServiceMock.saveCheckpoint(checkpointDTO3)) - .thenReturn(ZIO.fail(ServiceError("boom!"))) + .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) @@ -46,14 +46,14 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { result <- CheckpointController.createCheckpointV1(checkpointDTO1) } yield assertTrue(result == checkpointDTO1) }, - test("Returns expected InternalServerErrorResponse") { + test("Returns expected ConflictServiceError") { assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)( - failsWithA[InternalServerErrorResponse] + failsWithA[ConflictServiceError] ) }, test("Returns expected GeneralErrorResponse") { assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)( - failsWithA[InternalServerErrorResponse] + failsWithA[GeneralServiceError] ) } ) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala index 4cf152128..a338ea38a 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.service.FlowService import za.co.absa.atum.server.model.InternalServerErrorResponse import zio._ @@ -28,7 +28,7 @@ import zio.test._ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { private val flowServiceMock = mock(classOf[FlowService]) when(flowServiceMock.getFlowCheckpoints(checkpointQueryDTO1)) - .thenReturn(ZIO.fail(ServiceError("boom!"))) + .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) when(flowServiceMock.getFlowCheckpoints(checkpointQueryDTO2)) .thenReturn(ZIO.succeed(Seq(checkpointDTO2))) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 5a78b5012..397e40a32 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.service.PartitioningService import za.co.absa.atum.server.model.InternalServerErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse @@ -33,7 +33,7 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { when(partitioningServiceMock.createPartitioningIfNotExists(partitioningSubmitDTO1)) .thenReturn(ZIO.succeed(())) when(partitioningServiceMock.createPartitioningIfNotExists(partitioningSubmitDTO2)) - .thenReturn(ZIO.fail(ServiceError("boom!"))) + .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) when(partitioningServiceMock.getPartitioningMeasures(partitioningDTO1)) .thenReturn(ZIO.succeed(Seq(measureDTO1, measureDTO2))) @@ -44,14 +44,14 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { when(partitioningServiceMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)) .thenReturn(ZIO.succeed(())) when(partitioningServiceMock.createOrUpdateAdditionalData(additionalDataSubmitDTO2)) - .thenReturn(ZIO.fail(ServiceError("boom!"))) + .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) when(partitioningServiceMock.getPartitioningCheckpoints(checkpointQueryDTO1)) .thenReturn(ZIO.succeed(Seq(checkpointDTO1, checkpointDTO2))) when(partitioningServiceMock.getPartitioningCheckpoints(checkpointQueryDTO2)) .thenReturn(ZIO.succeed(Seq.empty)) when(partitioningServiceMock.getPartitioningCheckpoints(checkpointQueryDTO3)) - .thenReturn(ZIO.fail(ServiceError("boom!"))) + .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) private val partitioningServiceMockLayer = ZLayer.succeed(partitioningServiceMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 11d5eeec6..5e57fe2f7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -14,81 +14,81 @@ * limitations under the License. */ -package za.co.absa.atum.server.api.http - -import org.mockito.Mockito.{mock, when} -import sttp.client3.testing.SttpBackendStub -import sttp.client3.{UriContext, basicRequest} -import sttp.client3.circe._ -import sttp.model.StatusCode -import sttp.tapir.server.stub.TapirStubInterpreter -import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} -import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.controller.CheckpointController -import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse -import zio._ -import zio.test.Assertion.equalTo -import zio.test._ - -object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { - - private val checkpointControllerMock = mock(classOf[CheckpointController]) - - when(checkpointControllerMock.postCheckpointV2(checkpointDTO1)) - .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointDTO1, uuid))) - when(checkpointControllerMock.postCheckpointV2(checkpointDTO2)) - .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) - when(checkpointControllerMock.postCheckpointV2(checkpointDTO3)) - .thenReturn(ZIO.fail(InternalServerErrorResponse("error"))) - - private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) - - private val createCheckpointServerEndpoint = createCheckpointEndpointV2 - .zServerLogic(CheckpointController.postCheckpointV2) - - def spec: Spec[TestEnvironment with Scope, Any] = { - val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) - .whenServerEndpoint(createCheckpointServerEndpoint) - .thenRunLogic() - .backend() - - val request = basicRequest - .post(uri"https://test.com/api/v2/create-checkpoint") - .response(asJson[SingleSuccessResponse[CheckpointDTO]]) - - suite("CreateCheckpointEndpointSuite")( - test("Returns expected CheckpointDTO") { - val response = request - .body(checkpointDTO1) - .send(backendStub) - - val body = response.map(_.body) - val statusCode = response.map(_.code) - - assertZIO(body <&> statusCode)(equalTo(Right(SingleSuccessResponse(checkpointDTO1, uuid)), StatusCode.Created)) - }, - test("Returns expected BadRequest") { - val response = request - .body(checkpointDTO2) - .send(backendStub) - - val statusCode = response.map(_.code) - - assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) - }, - test("Returns expected InternalServerError") { - val response = request - .body(checkpointDTO3) - .send(backendStub) - - val statusCode = response.map(_.code) - - assertZIO(statusCode)(equalTo(StatusCode.InternalServerError)) - } - ) - }.provide( - checkpointControllerMockLayer - ) -} +//package za.co.absa.atum.server.api.http +// +//import org.mockito.Mockito.{mock, when} +//import sttp.client3.testing.SttpBackendStub +//import sttp.client3.{UriContext, basicRequest} +//import sttp.client3.circe._ +//import sttp.model.StatusCode +//import sttp.tapir.server.stub.TapirStubInterpreter +//import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} +//import za.co.absa.atum.model.dto.CheckpointDTO +//import za.co.absa.atum.server.api.TestData +//import za.co.absa.atum.server.api.controller.CheckpointController +//import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} +//import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse +//import zio._ +//import zio.test.Assertion.equalTo +//import zio.test._ +// +//object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { +// +// private val checkpointControllerMock = mock(classOf[CheckpointController]) +// +// when(checkpointControllerMock.postCheckpointV2(checkpointDTO1)) +// .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointDTO1, uuid))) +// when(checkpointControllerMock.postCheckpointV2(checkpointDTO2)) +// .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) +// when(checkpointControllerMock.postCheckpointV2(checkpointDTO3)) +// .thenReturn(ZIO.fail(InternalServerErrorResponse("error"))) +// +// private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) +// +// private val createCheckpointServerEndpoint = createCheckpointEndpointV2 +// .zServerLogic(CheckpointController.postCheckpointV2) +// +// def spec: Spec[TestEnvironment with Scope, Any] = { +// val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) +// .whenServerEndpoint(createCheckpointServerEndpoint) +// .thenRunLogic() +// .backend() +// +// val request = basicRequest +// .post(uri"https://test.com/api/v2/create-checkpoint") +// .response(asJson[SingleSuccessResponse[CheckpointDTO]]) +// +// suite("CreateCheckpointEndpointSuite")( +// test("Returns expected CheckpointDTO") { +// val response = request +// .body(checkpointDTO1) +// .send(backendStub) +// +// val body = response.map(_.body) +// val statusCode = response.map(_.code) +// +// assertZIO(body <&> statusCode)(equalTo(Right(SingleSuccessResponse(checkpointDTO1, uuid)), StatusCode.Created)) +// }, +// test("Returns expected BadRequest") { +// val response = request +// .body(checkpointDTO2) +// .send(backendStub) +// +// val statusCode = response.map(_.code) +// +// assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) +// }, +// test("Returns expected InternalServerError") { +// val response = request +// .body(checkpointDTO3) +// .send(backendStub) +// +// val statusCode = response.map(_.code) +// +// assertZIO(statusCode)(equalTo(StatusCode.InternalServerError)) +// } +// ) +// }.provide( +// checkpointControllerMockLayer +// ) +//} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index f73c21c8c..9e671aeb8 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} +import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError, GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.repository.CheckpointRepository import zio.test.Assertion.failsWithA import zio.test._ @@ -28,9 +28,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { private val checkpointRepositoryMock = mock(classOf[CheckpointRepository]) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.succeed(())) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)).thenReturn(ZIO.fail(DatabaseError("error in data"))) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(DatabaseError("boom!"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.unit) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)).thenReturn(ZIO.fail(GeneralDatabaseError("error in data"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -46,7 +46,7 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- CheckpointService.saveCheckpoint(checkpointDTO2).exit - } yield assertTrue(result == Exit.fail(ServiceError("Failed to perform 'saveCheckpoint': error in data"))) + } yield assertTrue(result == Exit.fail(GeneralServiceError("Failed to perform 'saveCheckpoint': error in data"))) }, test("Returns expected ServiceError") { assertZIO(CheckpointService.saveCheckpoint(checkpointDTO3).exit)(failsWithA[ServiceError]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala index da1bfba87..ad2740207 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} +import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError, ServiceError} import za.co.absa.atum.server.api.repository.FlowRepository import zio._ import zio.test.Assertion.failsWithA @@ -27,7 +27,7 @@ import zio.test._ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { private val flowRepositoryMock = mock(classOf[FlowRepository]) - when(flowRepositoryMock.getFlowCheckpoints(checkpointQueryDTO1)).thenReturn(ZIO.fail(DatabaseError("boom!"))) + when(flowRepositoryMock.getFlowCheckpoints(checkpointQueryDTO1)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) when(flowRepositoryMock.getFlowCheckpoints(checkpointQueryDTO2)) .thenReturn(ZIO.succeed(Seq(checkpointFromDB2))) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 34da663c0..48e90407f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{DatabaseError, ServiceError} +import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError, GeneralServiceError, ServiceError} import za.co.absa.atum.server.api.repository.PartitioningRepository import zio.test.Assertion.failsWithA import zio.test._ @@ -28,32 +28,32 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { private val partitioningRepositoryMock = mock(classOf[PartitioningRepository]) - when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO1)).thenReturn(ZIO.succeed(())) + when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO1)).thenReturn(ZIO.unit) when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO2)) - .thenReturn(ZIO.fail(DatabaseError("error in data"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("error in data"))) when(partitioningRepositoryMock.createPartitioningIfNotExists(partitioningSubmitDTO3)) - .thenReturn(ZIO.fail(DatabaseError("boom!"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) - when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)).thenReturn(ZIO.succeed(())) + when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)).thenReturn(ZIO.unit) when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO2)) - .thenReturn(ZIO.fail(DatabaseError("error in AD data"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("error in AD data"))) when(partitioningRepositoryMock.createOrUpdateAdditionalData(additionalDataSubmitDTO3)) - .thenReturn(ZIO.fail(DatabaseError("boom!"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) when(partitioningRepositoryMock.getPartitioningMeasures(partitioningDTO1)) .thenReturn(ZIO.succeed(Seq(measureDTO1, measureDTO2))) when(partitioningRepositoryMock.getPartitioningMeasures(partitioningDTO2)) - .thenReturn(ZIO.fail(DatabaseError("boom!"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) when(partitioningRepositoryMock.getPartitioningAdditionalData(partitioningDTO1)) .thenReturn(ZIO.succeed(additionalDataDTO1)) when(partitioningRepositoryMock.getPartitioningAdditionalData(partitioningDTO2)) - .thenReturn(ZIO.fail(DatabaseError("boom!"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) when(partitioningRepositoryMock.getPartitioningCheckpoints(checkpointQueryDTO1)) .thenReturn(ZIO.succeed(Seq(checkpointFromDB1, checkpointFromDB2))) when(partitioningRepositoryMock.getPartitioningCheckpoints(checkpointQueryDTO2)) - .thenReturn(ZIO.fail(DatabaseError("boom!"))) + .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val partitioningRepositoryMockLayer = ZLayer.succeed(partitioningRepositoryMock) @@ -70,7 +70,7 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { for { result <- PartitioningService.createPartitioningIfNotExists(partitioningSubmitDTO2).exit } yield assertTrue( - result == Exit.fail(ServiceError("Failed to perform 'createPartitioningIfNotExists': error in data")) + result == Exit.fail(GeneralServiceError("Failed to perform 'createPartitioningIfNotExists': error in data")) ) }, test("Returns expected ServiceError") { @@ -89,7 +89,7 @@ object PartitioningServiceUnitTests extends ZIOSpecDefault with TestData { for { result <- PartitioningService.createOrUpdateAdditionalData(additionalDataSubmitDTO2).exit } yield assertTrue( - result == Exit.fail(ServiceError("Failed to perform 'createOrUpdateAdditionalData': error in AD data")) + result == Exit.fail(GeneralServiceError("Failed to perform 'createOrUpdateAdditionalData': error in AD data")) ) }, test("Returns expected ServiceError") { From 47dea3158f2e1410a94194e7417ca474634dbad7 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 13:01:01 +0200 Subject: [PATCH 06/50] fixes related to error model --- .../CheckpointControllerUnitTests.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 2e167be8d..237f32b5a 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -18,9 +18,9 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{ConflictServiceError, GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.{ConflictServiceError, GeneralServiceError} import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.InternalServerErrorResponse +import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse} import zio.test.Assertion.failsWithA import zio._ import zio.test._ @@ -31,7 +31,7 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { when(checkpointServiceMock.saveCheckpoint(checkpointDTO1)).thenReturn(ZIO.unit) when(checkpointServiceMock.saveCheckpoint(checkpointDTO2)) - .thenReturn(ZIO.fail(ConflictServiceError("error in data"))) + .thenReturn(ZIO.fail(GeneralServiceError("error in data"))) when(checkpointServiceMock.saveCheckpoint(checkpointDTO3)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) @@ -47,13 +47,13 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { } yield assertTrue(result == checkpointDTO1) }, test("Returns expected ConflictServiceError") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)( - failsWithA[ConflictServiceError] + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)( + failsWithA[InternalServerErrorResponse] ) }, - test("Returns expected GeneralErrorResponse") { - assertZIO(CheckpointController.createCheckpointV1(checkpointDTO2).exit)( - failsWithA[GeneralServiceError] + test("Returns expected ConflictServiceError") { + assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)( + failsWithA[InternalServerErrorResponse] ) } ) From cc4af62b4792acd2de853d54d107c427f2b39fb0 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 13:06:02 +0200 Subject: [PATCH 07/50] param name fix --- .../database/runs/WriteCheckpointV2IntegrationTests.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala index cb386287d..85d4da20d 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala @@ -151,7 +151,7 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) function("runs.write_checkpoint_v2") - .setParam("i_partitioning", fkPartitioning) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Happy path") .setParam("i_process_start_time", startTime) @@ -238,7 +238,7 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { ) function("runs.write_checkpoint_v2") - .setParam("i_partitioning", fkPartitioning) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) From bb6331ece92cfbdfa84b402d795a73e4f2b10094 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 13:25:21 +0200 Subject: [PATCH 08/50] exceptions imports --- .../runs/V1.9.2__write_checkpoint_v2.sql | 15 +++++++++++ .../api/controller/BaseController.scala | 3 ++- .../atum/server/api/exception/AppError.scala | 11 +------- .../server/api/exception/DatabaseError.scala | 26 +++++++++++++++++++ .../server/api/exception/ServiceError.scala | 26 +++++++++++++++++++ .../api/repository/BaseRepository.scala | 3 ++- .../atum/server/api/service/BaseService.scala | 2 ++ .../server/api/service/FlowServiceImpl.scala | 3 ++- .../api/service/PartitioningServiceImpl.scala | 3 ++- 9 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala create mode 100644 server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql index 420d132b4..979b8ba83 100644 --- a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + CREATE OR REPLACE FUNCTION runs.write_checkpoint_v2( IN i_partitioning_id BIGINT, IN i_id_checkpoint UUID, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 1a97b6b9b..ab0e36fe1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -16,7 +16,8 @@ package za.co.absa.atum.server.api.controller -import za.co.absa.atum.server.api.exception.{ConflictServiceError, GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.config.SslConfig import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala index 106d647f2..e33f02cd4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala @@ -16,15 +16,6 @@ package za.co.absa.atum.server.api.exception -sealed trait AppError extends Throwable { +abstract class AppError extends Throwable { def message: String } - -sealed trait DatabaseError extends AppError -case class GeneralDatabaseError(message: String) extends DatabaseError -case class ConflictDatabaseError(message: String) extends DatabaseError - -sealed trait ServiceError extends AppError - -case class GeneralServiceError(message: String) extends ServiceError -case class ConflictServiceError(message: String) extends ServiceError diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala new file mode 100644 index 000000000..7ef1cee7e --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.exception + +sealed trait DatabaseError extends AppError + +object DatabaseError { + + case class GeneralDatabaseError(message: String) extends DatabaseError + case class ConflictDatabaseError(message: String) extends DatabaseError + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala new file mode 100644 index 000000000..3d36a28d2 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.exception + +sealed trait ServiceError extends AppError + +object ServiceError { + + case class GeneralServiceError(message: String) extends ServiceError + case class ConflictServiceError(message: String) extends ServiceError + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index c9ce33570..0bb9500ab 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -16,7 +16,8 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.server.api.exception.{ConflictDatabaseError, DatabaseError, GeneralDatabaseError} +import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.exceptions.{DataConflictException, StatusException} import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala index 0ca695299..33ca81d02 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala @@ -16,6 +16,8 @@ package za.co.absa.atum.server.api.service +import za.co.absa.atum.server.api.exception.DatabaseError._ +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.exception._ import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala index f768a3262..497d1c4d8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/FlowServiceImpl.scala @@ -17,7 +17,8 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.repository.FlowRepository import za.co.absa.atum.server.model.CheckpointFromDB import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index e4617879f..3066878c3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -17,7 +17,8 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.repository.PartitioningRepository import za.co.absa.atum.server.model.CheckpointFromDB import zio._ From 19e638258fc0f31a44b99bb644b263ca2d8e7db8 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 6 Aug 2024 13:32:29 +0200 Subject: [PATCH 09/50] exceptions imports --- .../api/controller/CheckpointControllerUnitTests.scala | 4 ++-- .../server/api/controller/FlowControllerUnitTests.scala | 2 +- .../api/controller/PartitioningControllerUnitTests.scala | 6 +++--- .../api/repository/CheckpointRepositoryUnitTests.scala | 4 ++-- .../api/repository/PartitioningRepositoryUnitTests.scala | 3 ++- .../server/api/service/CheckpointServiceUnitTests.scala | 4 +++- .../absa/atum/server/api/service/FlowServiceUnitTests.scala | 3 ++- .../server/api/service/PartitioningServiceUnitTests.scala | 4 +++- 8 files changed, 18 insertions(+), 12 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 237f32b5a..8bfb520a2 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -18,9 +18,9 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{ConflictServiceError, GeneralServiceError} +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.InternalServerErrorResponse import zio.test.Assertion.failsWithA import zio._ import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala index a338ea38a..4d2f90fe3 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.service.FlowService import za.co.absa.atum.server.model.InternalServerErrorResponse import zio._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 397e40a32..30ac2ab0b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.ServiceError.GeneralServiceError import za.co.absa.atum.server.api.service.PartitioningService import za.co.absa.atum.server.model.InternalServerErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse @@ -31,7 +31,7 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { private val partitioningServiceMock = mock(classOf[PartitioningService]) when(partitioningServiceMock.createPartitioningIfNotExists(partitioningSubmitDTO1)) - .thenReturn(ZIO.succeed(())) + .thenReturn(ZIO.unit) when(partitioningServiceMock.createPartitioningIfNotExists(partitioningSubmitDTO2)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) @@ -42,7 +42,7 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { .thenReturn(ZIO.succeed(Map.empty)) when(partitioningServiceMock.createOrUpdateAdditionalData(additionalDataSubmitDTO1)) - .thenReturn(ZIO.succeed(())) + .thenReturn(ZIO.unit) when(partitioningServiceMock.createOrUpdateAdditionalData(additionalDataSubmitDTO2)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index a3da98e49..538fea208 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -18,9 +18,9 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} -import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError} +import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData -import za.co.absa.db.fadb.exceptions.ErrorInDataException +import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala index b03c80236..c1174e3a2 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryUnitTests.scala @@ -19,7 +19,8 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions._ -import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError} +import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.exceptions.ErrorInDataException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index 9e671aeb8..df6cf0a7d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -18,7 +18,9 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError, GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.DatabaseError._ +import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.repository.CheckpointRepository import zio.test.Assertion.failsWithA import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala index ad2740207..a3c856fec 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/FlowServiceUnitTests.scala @@ -18,7 +18,8 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError, ServiceError} +import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError +import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.FlowRepository import zio._ import zio.test.Assertion.failsWithA diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala index 48e90407f..0c97e1d7e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/PartitioningServiceUnitTests.scala @@ -18,7 +18,9 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.exception.{DatabaseError, GeneralDatabaseError, GeneralServiceError, ServiceError} +import za.co.absa.atum.server.api.exception.DatabaseError._ +import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.repository.PartitioningRepository import zio.test.Assertion.failsWithA import zio.test._ From a3ffb44d660272159b727fee3e5598587b20a377 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 7 Aug 2024 09:58:41 +0200 Subject: [PATCH 10/50] PostCheckpointEndpointV2UnitTests --- .../za/co/absa/atum/server/Constants.scala | 1 + .../atum/server/api/http/BaseEndpoints.scala | 10 +- .../absa/atum/server/api/http/Endpoints.scala | 4 +- .../za/co/absa/atum/server/api/TestData.scala | 1 + .../api/http/BaseEndpointsUnitTests.scala | 85 -------------- .../CreateCheckpointEndpointUnitTests.scala | 94 --------------- .../PostCheckpointEndpointV2UnitTests.scala | 110 ++++++++++++++++++ 7 files changed, 115 insertions(+), 190 deletions(-) delete mode 100644 server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala delete mode 100644 server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala create mode 100644 server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/Constants.scala b/server/src/main/scala/za/co/absa/atum/server/Constants.scala index 6b200ae05..a2e6aa42c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Constants.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Constants.scala @@ -16,6 +16,7 @@ package za.co.absa.atum.server +// TODO: to be removed when v2 endpoints are implemented, replaced by ApiPaths object object Constants { object Endpoints { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index 04b3c228a..aac2a5e7b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.http import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody -import za.co.absa.atum.server.model.{BadRequestResponse, ConflictErrorResponse, ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model._ import sttp.tapir.typelevel.MatchType import sttp.tapir.ztapir._ import sttp.tapir.{EndpointOutput, PublicEndpoint} @@ -77,12 +77,4 @@ trait BaseEndpoints { baseEndpoint.in(Api / V2) } - def pathToAPIv1CompatibleFormat(apiURLPath: String): String = { - // this is basically kebab-case/snake_case to camelCase - val inputParts = apiURLPath.split("[_-]") - - // Capitalize the first letter of each part except the first one (lowercase always) - inputParts.head.toLowerCase + inputParts.tail.map(_.capitalize).mkString("") - } - } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 78800ae37..5c54774c1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -31,7 +31,7 @@ trait Endpoints extends BaseEndpoints { protected val createCheckpointEndpointV1: PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { apiV1.post - .in(pathToAPIv1CompatibleFormat(CreateCheckpoint)) + .in(V1Paths.CreateCheckpoint) .in(jsonBody[CheckpointDTO]) .out(statusCode(StatusCode.Created)) .out(jsonBody[CheckpointDTO]) @@ -51,7 +51,7 @@ trait Endpoints extends BaseEndpoints { protected val createPartitioningEndpointV1 : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { apiV1.post - .in(pathToAPIv1CompatibleFormat(CreatePartitioning)) + .in(V1Paths.CreatePartitioning) .in(jsonBody[PartitioningSubmitDTO]) .out(statusCode(StatusCode.Ok)) .out(jsonBody[AtumContextDTO]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 5e58ee4ac..2963abf3b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -191,6 +191,7 @@ trait TestData { ) protected val checkpointDTO3: CheckpointDTO = checkpointDTO1.copy(id = UUID.randomUUID()) + protected val checkpointDTO4: CheckpointDTO = checkpointDTO1.copy(id = UUID.randomUUID()) // Checkpoint From DB protected val checkpointFromDB1: CheckpointFromDB = CheckpointFromDB( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala deleted file mode 100644 index e8798974b..000000000 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/BaseEndpointsUnitTests.scala +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.api.http - -import org.scalatest.flatspec.AnyFlatSpec - -class BaseEndpointsUnitTests extends AnyFlatSpec { - - object BaseEndpointsForTests extends BaseEndpoints - - "pathToAPIv1CompatibleFormat" should "successfully handle empty input" in { - val input = "" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "" - assert(actual == expected) - } - - "pathToAPIv1CompatibleFormat" should - "successfully convert our standard API path format to format compatible with API V1 (kebab)" in { - - val input = "create-checkpoint" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheckpoint" - assert(actual == expected) - } - - "pathToAPIv1CompatibleFormat" should - "successfully convert our standard API path format to format compatible with API V1 (kebab2)" in { - - val input = "create-check-point2" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheckPoint2" - assert(actual == expected) - } - - "pathToAPIv1CompatibleFormat" should - "successfully convert our standard API path format to format compatible with API V1 (kebab3)" in { - - val input = "Create-check-" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheck" - assert(actual == expected) - } - - "pathToAPIv1CompatibleFormat" should - "successfully convert our standard API path format to format compatible with API V1 (snake)" in { - - val input = "_create_check_point" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "CreateCheckPoint" - assert(actual == expected) - } - - "pathToAPIv1CompatibleFormat" should - "successfully convert our standard API path format to format compatible with API V1 (kebab and snake)" in { - - val input = "Create-check_Point" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createCheckPoint" - assert(actual == expected) - } - - "pathToAPIv1CompatibleFormat" should - "successfully convert our standard API path format to format compatible with API V1 (one word)" in { - - val input = "createcheckpoint" - val actual = BaseEndpointsForTests.pathToAPIv1CompatibleFormat(input) - val expected = "createcheckpoint" - assert(actual == expected) - } -} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala deleted file mode 100644 index 5e57fe2f7..000000000 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//package za.co.absa.atum.server.api.http -// -//import org.mockito.Mockito.{mock, when} -//import sttp.client3.testing.SttpBackendStub -//import sttp.client3.{UriContext, basicRequest} -//import sttp.client3.circe._ -//import sttp.model.StatusCode -//import sttp.tapir.server.stub.TapirStubInterpreter -//import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} -//import za.co.absa.atum.model.dto.CheckpointDTO -//import za.co.absa.atum.server.api.TestData -//import za.co.absa.atum.server.api.controller.CheckpointController -//import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -//import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse -//import zio._ -//import zio.test.Assertion.equalTo -//import zio.test._ -// -//object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { -// -// private val checkpointControllerMock = mock(classOf[CheckpointController]) -// -// when(checkpointControllerMock.postCheckpointV2(checkpointDTO1)) -// .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointDTO1, uuid))) -// when(checkpointControllerMock.postCheckpointV2(checkpointDTO2)) -// .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) -// when(checkpointControllerMock.postCheckpointV2(checkpointDTO3)) -// .thenReturn(ZIO.fail(InternalServerErrorResponse("error"))) -// -// private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) -// -// private val createCheckpointServerEndpoint = createCheckpointEndpointV2 -// .zServerLogic(CheckpointController.postCheckpointV2) -// -// def spec: Spec[TestEnvironment with Scope, Any] = { -// val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) -// .whenServerEndpoint(createCheckpointServerEndpoint) -// .thenRunLogic() -// .backend() -// -// val request = basicRequest -// .post(uri"https://test.com/api/v2/create-checkpoint") -// .response(asJson[SingleSuccessResponse[CheckpointDTO]]) -// -// suite("CreateCheckpointEndpointSuite")( -// test("Returns expected CheckpointDTO") { -// val response = request -// .body(checkpointDTO1) -// .send(backendStub) -// -// val body = response.map(_.body) -// val statusCode = response.map(_.code) -// -// assertZIO(body <&> statusCode)(equalTo(Right(SingleSuccessResponse(checkpointDTO1, uuid)), StatusCode.Created)) -// }, -// test("Returns expected BadRequest") { -// val response = request -// .body(checkpointDTO2) -// .send(backendStub) -// -// val statusCode = response.map(_.code) -// -// assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) -// }, -// test("Returns expected InternalServerError") { -// val response = request -// .body(checkpointDTO3) -// .send(backendStub) -// -// val statusCode = response.map(_.code) -// -// assertZIO(statusCode)(equalTo(StatusCode.InternalServerError)) -// } -// ) -// }.provide( -// checkpointControllerMockLayer -// ) -//} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala new file mode 100644 index 000000000..f2791578a --- /dev/null +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.http + +import org.mockito.Mockito.{mock, when} +import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ +import sttp.model.StatusCode +import sttp.tapir.server.stub.TapirStubInterpreter +import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} +import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.server.api.TestData +import za.co.absa.atum.server.api.controller.CheckpointController +import za.co.absa.atum.server.model.{ConflictErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse +import zio._ +import zio.test.Assertion.equalTo +import zio.test._ + +object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints with TestData { + + private val checkpointControllerMock = mock(classOf[CheckpointController]) + + when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO1)) + .thenReturn(ZIO.succeed((SingleSuccessResponse(checkpointDTO1, uuid), "some location"))) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO2)) + .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO3)) + .thenReturn(ZIO.fail(InternalServerErrorResponse("error"))) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO4)) + .thenReturn(ZIO.fail(ConflictErrorResponse("error"))) + + private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) + + private val postCheckpointServerEndpointV2 = postCheckpointEndpointV2 + .zServerLogic({ case (partitioningId: Long, checkpointDTO: CheckpointDTO) => + CheckpointController.postCheckpointV2(partitioningId, checkpointDTO) + }) + + def spec: Spec[TestEnvironment with Scope, Any] = { + val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) + .whenServerEndpoint(postCheckpointServerEndpointV2) + .thenRunLogic() + .backend() + + val request = basicRequest + .post(uri"https://test.com/api/v2/partitionings/1/checkpoints") + .response(asJson[SingleSuccessResponse[CheckpointDTO]]) + + suite("CreateCheckpointEndpointSuite")( + test("Returns expected CheckpointDTO") { + val response = request + .body(checkpointDTO1) + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + val header = response.map(_.header("Location")) + + assertZIO(body <&> statusCode <&> header)( + equalTo(Right(SingleSuccessResponse(checkpointDTO1, uuid)), StatusCode.Created, Some("some location")) + ) + }, + test("Returns expected BadRequest") { + val response = request + .body(checkpointDTO2) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected InternalServerError") { + val response = request + .body(checkpointDTO3) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.InternalServerError)) + }, + test("Returns expected ConflictError") { + val response = request + .body(checkpointDTO4) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.Conflict)) + } + ) + }.provide( + checkpointControllerMockLayer + ) +} From 928823d5783fa982f004f0ae4e0db15d68ed1bc4 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 7 Aug 2024 18:05:11 +0200 Subject: [PATCH 11/50] tests --- .../api/controller/BaseController.scala | 1 + server/src/test/resources/reference.conf | 22 ++++++++- .../CheckpointControllerUnitTests.scala | 45 ++++++++++++++++- .../WriteCheckpointIntegrationTests.scala | 1 - .../WriteCheckpointV2IntegrationTests.scala | 48 +++++++++++++++++++ .../CheckpointRepositoryUnitTests.scala | 30 +++++++++++- .../service/CheckpointServiceUnitTests.scala | 34 +++++++++++-- 7 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index ab0e36fe1..ad33cf4a9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -56,6 +56,7 @@ trait BaseController { protected def createResourceUri(parts: Seq[String]): IO[ErrorResponse, String] = { for { hostname <- System.env("HOSTNAME") + // fails the request if the hostname is not found, we need to make sure that the hostname is always available .orElseFail(InternalServerErrorResponse("Failed to get hostname")) .flatMap { case Some(value) => ZIO.succeed(value) diff --git a/server/src/test/resources/reference.conf b/server/src/test/resources/reference.conf index c4fc78a1d..05d4df973 100644 --- a/server/src/test/resources/reference.conf +++ b/server/src/test/resources/reference.conf @@ -1,5 +1,5 @@ { - postgres { + postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver serverName=localhost @@ -11,4 +11,24 @@ # maximum number of connections that HikariCP will keep in the pool, including both idle and in-use connections maxPoolSize=10 } + aws { + region = "af-south-1" + dbPasswordSecretName = "serviceUserSecretKey" + } + ssl { + enabled=false + keyStorePassword=password + keyStorePath="/path/to/your/cert" + } + monitoring { + # monitoring of http communication + http { + enabled=true + } + # monitoring of jvm and zio + jvm { + enabled=true + intervalInSeconds=5 + } + } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 8bfb520a2..40b211191 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -17,15 +17,20 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} +import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError._ +import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.InternalServerErrorResponse +import za.co.absa.atum.server.config.SslConfig +import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.test.Assertion.failsWithA import zio._ import zio.test._ -object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { +object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { private val checkpointServiceMock = mock(classOf[CheckpointService]) @@ -35,6 +40,14 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { when(checkpointServiceMock.saveCheckpoint(checkpointDTO3)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) + private val partitioningId = 1L + + when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointDTO1)).thenReturn(ZIO.unit) + when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointDTO2)) + .thenReturn(ZIO.fail(GeneralServiceError("error in data"))) + when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointDTO3)) + .thenReturn(ZIO.fail(ConflictServiceError("boom!"))) + private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -56,6 +69,34 @@ object CheckpointControllerUnitTests extends ZIOSpecDefault with TestData { failsWithA[InternalServerErrorResponse] ) } + ), + suite("PostCheckpointV2Suite")( + test("Returns expected CheckpointDTO") { + val host = "testHost" + for { + _ <- TestSystem.putEnv("HOSTNAME", host) + sslConfig <- ZIO.config[SslConfig](SslConfig.config) + result <- CheckpointController.postCheckpointV2(partitioningId, checkpointDTO1) + protocol = if (sslConfig.enabled) "https" else "http" + port = if (sslConfig.enabled) 8443 else 8080 + path = s"${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}/${checkpointDTO1.id}" + expectedUri = s"$protocol://$host:$port/$path" + } yield assertTrue( + result._1.isInstanceOf[SingleSuccessResponse[CheckpointDTO]] + && result._1.data == checkpointDTO1 + && result._2 == expectedUri + ) + }, + test("Returns expected ConflictServiceError") { + assertZIO(CheckpointController.postCheckpointV2(1L, checkpointDTO2).exit)( + failsWithA[InternalServerErrorResponse] + ) + }, + test("Returns expected ConflictServiceError") { + assertZIO(CheckpointController.postCheckpointV2(1L, checkpointDTO3).exit)( + failsWithA[ConflictErrorResponse] + ) + } ) ).provide( CheckpointControllerImpl.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 2f1b8ffe5..68436230c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -37,7 +37,6 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { suite("WriteCheckpointSuite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { - val checkpointDTO = CheckpointDTO( id = UUID.randomUUID(), name = "name", diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala new file mode 100644 index 000000000..9403b44b5 --- /dev/null +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -0,0 +1,48 @@ +package za.co.absa.atum.server.api.database.runs.functions + +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.server.ConfigProviderTest +import za.co.absa.atum.server.api.TestTransactorProvider +import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.atum.server.model.WriteCheckpointV2Args +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.{FunctionStatus, Row} +import zio._ +import zio.interop.catz.asyncInstance +import zio.test._ + +import java.time.ZonedDateTime +import java.util.UUID + +object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { + + override def spec: Spec[TestEnvironment with Scope, Any] = { + + suite("WriteCheckpointSuite")( + test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { + val checkpointDTO = CheckpointDTO( + id = UUID.randomUUID(), + name = "name", + author = "author", + partitioning = Seq(PartitionDTO("key4", "value4")), + processStartTime = ZonedDateTime.now(), + processEndTime = Option(ZonedDateTime.now()), + measurements = Set( + MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue))) + ) + ) + for { + writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] + result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointDTO)) + } yield assertTrue(result == Right(Row(FunctionStatus(11, "Checkpoint created"),()))) + } + ).provide( + WriteCheckpointV2.layer, + PostgresDatabaseProvider.layer, + TestTransactorProvider.layerWithRollback + ) + } + +} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 538fea208..b2fd3f4f3 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -21,10 +21,12 @@ import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, Writ import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.DatabaseError._ +import za.co.absa.atum.server.model.WriteCheckpointV2Args +import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance -import zio.test.Assertion.{failsWithA, isUnit} +import zio.test.Assertion.failsWithA import zio.test._ import za.co.absa.db.fadb.status.Row @@ -38,6 +40,15 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { .thenReturn(ZIO.fail(GeneralDatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) + private val partitioningId = 1L + + when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointDTO1))) + .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) + when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointDTO2))) + .thenReturn(ZIO.left(DataConflictException(FunctionStatus(31, "conflict in data")))) + when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointDTO3))) + .thenReturn(ZIO.fail(new Exception("boom!"))) + private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointMockV2) @@ -60,6 +71,23 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { test("Returns expected DatabaseError") { assertZIO(CheckpointRepository.writeCheckpoint(checkpointDTO3).exit)(failsWithA[DatabaseError]) } + ), + suite("WriteCheckpointV2Suite")( + test("Returns an expected Unit") { + for { + result <- CheckpointRepository.writeCheckpointV2(partitioningId, checkpointDTO1) + } yield assertTrue(result == ()) + }, + test("Fails with an expected ConflictDatabaseError") { + assertZIO(CheckpointRepository.writeCheckpointV2(partitioningId, checkpointDTO2).exit)( + failsWithA[ConflictDatabaseError] + ) + }, + test("Fails with an expected GeneralDatabaseError") { + assertZIO(CheckpointRepository.writeCheckpointV2(partitioningId, checkpointDTO3).exit)( + failsWithA[GeneralDatabaseError] + ) + } ) ).provide(CheckpointRepositoryImpl.layer, writeCheckpointMockLayer, writeCheckpointV2MockLayer) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index df6cf0a7d..ab896fa33 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -31,9 +31,18 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { private val checkpointRepositoryMock = mock(classOf[CheckpointRepository]) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.unit) - when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)).thenReturn(ZIO.fail(GeneralDatabaseError("error in data"))) + when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)) + .thenReturn(ZIO.fail(GeneralDatabaseError("error in data"))) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) + private val partitioningId = 1L + + when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointDTO1)).thenReturn(ZIO.unit) + when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointDTO2)) + .thenReturn(ZIO.fail(ConflictDatabaseError("conflict in data"))) + when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointDTO3)) + .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) + private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -48,10 +57,29 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { test("Returns expected Left with StatusException") { for { result <- CheckpointService.saveCheckpoint(checkpointDTO2).exit - } yield assertTrue(result == Exit.fail(GeneralServiceError("Failed to perform 'saveCheckpoint': error in data"))) + } yield assertTrue( + result == Exit.fail(GeneralServiceError("Failed to perform 'saveCheckpoint': error in data")) + ) }, test("Returns expected ServiceError") { - assertZIO(CheckpointService.saveCheckpoint(checkpointDTO3).exit)(failsWithA[ServiceError]) + assertZIO(CheckpointService.saveCheckpoint(checkpointDTO3).exit)(failsWithA[GeneralServiceError]) + } + ), + suite("SaveCheckpointV2Suite")( + test("Returns an expected Unit") { + for { + result <- CheckpointService.saveCheckpointV2(partitioningId, checkpointDTO1) + } yield assertTrue(result == ()) + }, + test("Fails with an expected ConflictServiceError") { + assertZIO(CheckpointService.saveCheckpointV2(partitioningId, checkpointDTO2).exit)( + failsWithA[ConflictServiceError] + ) + }, + test("Fails with an expected GeneralServiceError") { + assertZIO(CheckpointService.saveCheckpointV2(partitioningId, checkpointDTO3).exit)( + failsWithA[GeneralServiceError] + ) } ) ).provide( From bf472238fbb64acd341cbc1fc0096fd3b8f13729 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 7 Aug 2024 18:05:32 +0200 Subject: [PATCH 12/50] tests --- .../runs/functions/WriteCheckpointV2IntegrationTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index 9403b44b5..032d2f20a 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -20,7 +20,7 @@ object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("WriteCheckpointSuite")( + suite("WriteCheckpointV2Suite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { val checkpointDTO = CheckpointDTO( id = UUID.randomUUID(), From c4245ebe15631acef6633ca4d565452e491a94e6 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 7 Aug 2024 18:15:44 +0200 Subject: [PATCH 13/50] remove unused import --- .../runs/functions/WriteCheckpointV2IntegrationTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index 032d2f20a..ce469c2f8 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -7,7 +7,6 @@ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.model.WriteCheckpointV2Args -import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ import zio.interop.catz.asyncInstance From 51332729cddc08837d9a3d3a4f006e7c660a67ee Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 14:23:59 +0200 Subject: [PATCH 14/50] get checkpoint --- ...V1.9.3__get_partitioning_checkpoint_v2.sql | 104 ++++++++++++++++++ .../api/controller/BaseController.scala | 3 +- .../api/controller/CheckpointController.scala | 5 + .../controller/CheckpointControllerImpl.scala | 12 ++ .../runs/functions/GetCheckpointV2.scala | 45 ++++++++ .../server/api/exception/DatabaseError.scala | 1 + .../server/api/exception/ServiceError.scala | 1 + .../atum/server/api/http/BaseEndpoints.scala | 7 ++ .../absa/atum/server/api/http/Endpoints.scala | 9 ++ .../co/absa/atum/server/api/http/Routes.scala | 7 ++ .../api/repository/BaseRepository.scala | 7 +- .../api/repository/CheckpointRepository.scala | 1 + .../repository/CheckpointRepositoryImpl.scala | 18 ++- .../atum/server/api/service/BaseService.scala | 1 + .../api/service/CheckpointService.scala | 1 + .../api/service/CheckpointServiceImpl.scala | 14 +++ .../server/model/GetCheckpointV2Args.scala | 22 ++++ 17 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql create mode 100644 server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala diff --git a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql new file mode 100644 index 000000000..47cb9f5b9 --- /dev/null +++ b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql @@ -0,0 +1,104 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoint_v2( + IN i_partitioning_id BIGINT, + IN i_checkpoint_id UUID, + OUT status INTEGER, + OUT status_text TEXT, + OUT id_checkpoint UUID, + OUT checkpoint_name TEXT, + OUT author TEXT, + OUT measured_by_atum_agent BOOLEAN, + OUT measure_name TEXT, + OUT measured_columns TEXT[], + OUT measurement_value JSONB, + OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE +) + RETURNS record AS +$$ + ------------------------------------------------------------------------------- +-- +-- Function: runs.get_partitioning_checkpoint_v2(BIGINT, UUID) +-- Retrieves a single checkpoint (measures and their measurement details) related to a +-- given partitioning and checkpoint ID. +-- +-- Parameters: +-- i_partitioning_id - ID of the partitioning +-- i_checkpoint_id - ID of the checkpoint +-- +-- Returns: +-- status - Status code +-- status_text - Status message +-- id_checkpoint - ID of the checkpoint +-- checkpoint_name - Name of the checkpoint +-- author - Author of the checkpoint +-- measuredByAtumAgent - Flag indicating whether the checkpoint was measured by ATUM agent +-- measure_name - Name of the measure +-- measure_columns - Columns of the measure +-- measurement_value - Value of the measurement +-- checkpoint_start_time - Time of the checkpoint +-- checkpoint_end_time - End time of the checkpoint computation +-- +-- Status codes: +-- 11 - OK +-- 41 - Partitioning or Checkpoint not found +-- +------------------------------------------------------------------------------- +BEGIN + IF NOT EXISTS (SELECT 1 FROM runs.partitionings WHERE id_partitioning = i_partitioning_id) THEN + status := 41; + status_text := 'Partitioning not found'; + RETURN; + END IF; + + RETURN QUERY + SELECT + 11 AS status, + 'Ok' AS status_text, + C.id_checkpoint, + C.checkpoint_name, + C.created_by AS author, + C.measured_by_atum_agent, + md.measure_name, + md.measured_columns, + M.measurement_value, + C.process_start_time AS checkpoint_start_time, + C.process_end_time AS checkpoint_end_time + FROM + runs.checkpoints C + JOIN + runs.measurements M ON C.id_checkpoint = M.fk_checkpoint + JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + WHERE + C.fk_partitioning = i_partitioning_id + AND + C.id_checkpoint = i_checkpoint_id + LIMIT 1; + + IF NOT FOUND THEN + status := 41; + status_text := 'Checkpoint not found'; + END IF; +END; +$$ + + LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +ALTER FUNCTION runs.get_partitioning_checkpoint_v2(BIGINT, UUID) OWNER TO atum_owner; + +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoint_v2(BIGINT, UUID) TO atum_owner; \ No newline at end of file diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index ad33cf4a9..950a5a00d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.config.SslConfig -import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ @@ -33,6 +33,7 @@ trait BaseController { serviceCall .mapError { case ConflictServiceError(message) => ConflictErrorResponse(message) + case NotFoundServiceError(message) => NotFoundErrorResponse(message) case GeneralServiceError(message) => InternalServerErrorResponse(message) } .flatMap { result => diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index d4bb2229d..61e50836d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -32,4 +32,9 @@ trait CheckpointController { checkpointDTO: CheckpointDTO ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String)] + def getPartitioningCheckpointV2( + partitioningId: Long, + checkpointId: String + ): IO[ErrorResponse, SingleSuccessResponse[CheckpointDTO]] + } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 44fc93372..60b56714f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -50,6 +50,18 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che ) } yield (response, uri) } + + override def getPartitioningCheckpointV2( + partitioningId: Long, + checkpointId: String + ): IO[ErrorResponse, SingleSuccessResponse[CheckpointDTO]] = { + mapToSingleSuccessResponse( + serviceCall[CheckpointDTO, CheckpointDTO]( + checkpointService.getCheckpointV2(partitioningId, checkpointId), + identity + ) + ) + } } object CheckpointControllerImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala new file mode 100644 index 000000000..eee81f283 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.database.runs.functions + +import doobie.implicits.toSqlInterpolator +import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.atum.server.api.database.runs.Runs +import za.co.absa.atum.server.model.{CheckpointFromDB, GetCheckpointV2Args} +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieEngine +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus +import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling +import zio._ + +class GetCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieSingleResultFunctionWithStatus[GetCheckpointV2Args, CheckpointFromDB, Task](input => + Seq( + fr"${input.partitioningId}", + fr"${input.checkpointId}" + ) + ) + with StandardStatusHandling + +object GetCheckpointV2 { + val layer: URLayer[PostgresDatabaseProvider, GetCheckpointV2] = ZLayer { + for { + dbProvider <- ZIO.service[PostgresDatabaseProvider] + } yield new GetCheckpointV2()(Runs, dbProvider.dbEngine) + } +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala index 7ef1cee7e..cebc9c674 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/DatabaseError.scala @@ -22,5 +22,6 @@ object DatabaseError { case class GeneralDatabaseError(message: String) extends DatabaseError case class ConflictDatabaseError(message: String) extends DatabaseError + case class NotFoundDatabaseError(message: String) extends DatabaseError } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala index 3d36a28d2..5d2f746b2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/ServiceError.scala @@ -22,5 +22,6 @@ object ServiceError { case class GeneralServiceError(message: String) extends ServiceError case class ConflictServiceError(message: String) extends ServiceError + case class NotFoundServiceError(message: String) extends ServiceError } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index aac2a5e7b..15919c240 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -38,6 +38,13 @@ trait BaseEndpoints { ) } + protected val notFoundErrorOneOfVariant: EndpointOutput.OneOfVariant[NotFoundErrorResponse] = { + oneOfVariantFromMatchType( + StatusCode.NotFound, + jsonBody[NotFoundErrorResponse] + ) + } + private val badRequestOneOfVariant: EndpointOutput.OneOfVariant[BadRequestResponse] = { oneOfVariantFromMatchType( StatusCode.BadRequest, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 5c54774c1..26af51dc1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -75,6 +75,15 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AdditionalDataSubmitDTO]]) } + protected val getPartitioningCheckpointEndpointV2 + : PublicEndpoint[(Long, String), ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { + apiV2.get + .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints / path[String]("checkpointId")) + .out(statusCode(StatusCode.Ok)) + .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) + .errorOutVariantPrepend(notFoundErrorOneOfVariant) + } + protected val getPartitioningCheckpointsEndpointV2 : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.get diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index af229a453..36004347c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -60,6 +60,13 @@ trait Routes extends Endpoints with ServerOptions { createOrUpdateAdditionalDataEndpointV2, PartitioningController.createOrUpdateAdditionalDataV2 ), + createServerEndpoint( + getPartitioningCheckpointEndpointV2, + { + case(partitioningId: Long, checkpointId: String) => + CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointId) + } + ), createServerEndpoint(getPartitioningCheckpointsEndpointV2, PartitioningController.getPartitioningCheckpointsV2), createServerEndpoint(getFlowCheckpointsEndpointV2, FlowController.getFlowCheckpointsV2), createServerEndpoint(healthEndpoint, (_: Unit) => ZIO.unit) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 0bb9500ab..866d32d8a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.db.fadb.exceptions.{DataConflictException, StatusException} +import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException, StatusException} import za.co.absa.db.fadb.status.{FailedOrRow, FailedOrRows} import zio._ @@ -46,13 +46,14 @@ trait BaseRepository { statusException match { case DataConflictException(_) => ConflictDatabaseError(message) + case DataNotFoundException(_) => NotFoundDatabaseError(message) case _ => GeneralDatabaseError(message) } case error => GeneralDatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") } - def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): IO[DatabaseError, R] = { + protected def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): IO[DatabaseError, R] = { logAndReturn(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) @@ -64,7 +65,7 @@ trait BaseRepository { .tapError(error => ZIO.logError(s"Operation '$operationName' failed: ${error.message}")) } - def dbMultipleResultCallWithAggregatedStatus[R]( + protected def dbMultipleResultCallWithAggregatedStatus[R]( dbFuncCall: Task[FailedOrRows[R]], operationName: String ): IO[DatabaseError, Seq[R]] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index a2237a98b..b887d11b8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -25,4 +25,5 @@ import zio.macros.accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] def writeCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] + def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointDTO] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 9d117c663..693500902 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,14 +17,17 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.WriteCheckpointV2Args import zio._ import zio.interop.catz.asyncInstance -class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint, writeCheckpointV2Fn: WriteCheckpointV2) - extends CheckpointRepository +class CheckpointRepositoryImpl( + writeCheckpointFn: WriteCheckpoint, + writeCheckpointV2Fn: WriteCheckpointV2, + getCheckpointV2Fn: GetCheckpointV2 +) extends CheckpointRepository with BaseRepository { override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { @@ -37,13 +40,18 @@ class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint, writeCheckpoi "writeCheckpoint" ) } + + override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointDTO] = { + dbSingleResultCallWithStatus(getCheckpointV2Fn(partitioningId, checkpointId), "getCheckpoint") + } } object CheckpointRepositoryImpl { - val layer: URLayer[WriteCheckpoint with WriteCheckpointV2, CheckpointRepository] = ZLayer { + val layer: URLayer[WriteCheckpoint with WriteCheckpointV2 with GetCheckpointV2, CheckpointRepository] = ZLayer { for { writeCheckpoint <- ZIO.service[WriteCheckpoint] writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] - } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2) + getCheckpointV2 <- ZIO.service[GetCheckpointV2] + } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2, getCheckpointV2) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala index 33ca81d02..3c7c98eec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/BaseService.scala @@ -27,6 +27,7 @@ trait BaseService { repositoryCall .mapError { case ConflictDatabaseError(message) => ConflictServiceError(createMessage(operationName, message)) + case NotFoundDatabaseError(message) => NotFoundServiceError(createMessage(operationName, message)) case GeneralDatabaseError(message) => GeneralServiceError(createMessage(operationName, message)) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index ce026ff05..89831db53 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -25,4 +25,5 @@ import zio.macros.accessible trait CheckpointService { def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] def saveCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] + def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointDTO] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index f8e772c2c..b3ae07b70 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -18,7 +18,9 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.ServiceError +import za.co.absa.atum.server.api.exception.ServiceError.GeneralServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository +import za.co.absa.atum.server.model.CheckpointFromDB import zio._ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { @@ -37,6 +39,18 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends ) } + override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointDTO] = { + for { + checkpointFromDB <- repositoryCall( + checkpointRepository.getCheckpointV2(partitioningId, checkpointId), + "getCheckpoint" + ) + // why CheckpointDTO contains partitioningDTO??? + checkpointDTO <- ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointFromDB)) + .mapError(error => GeneralServiceError(error.getMessage)) + } yield checkpointDTO + + } } object CheckpointServiceImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala b/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala new file mode 100644 index 000000000..9019fa495 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.model + +case class GetCheckpointV2Args( + partitioningId: Long, + checkpointId: String +) From b4ef95708944f654ef498be183ddd5ee8949f046 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 15:25:17 +0200 Subject: [PATCH 15/50] checkpointv2dto --- .../absa/atum/model/dto/CheckpointV2DTO.scala | 39 ++++++++++++++++++ .../scala/za/co/absa/atum/server/Main.scala | 1 + .../api/controller/CheckpointController.scala | 4 +- .../controller/CheckpointControllerImpl.scala | 6 +-- .../runs/functions/GetCheckpointV2.scala | 4 +- .../absa/atum/server/api/http/Endpoints.scala | 4 +- .../co/absa/atum/server/api/http/Routes.scala | 8 +++- .../api/repository/CheckpointRepository.scala | 3 +- .../repository/CheckpointRepositoryImpl.scala | 6 +-- .../api/service/CheckpointService.scala | 4 +- .../api/service/CheckpointServiceImpl.scala | 7 ++-- .../atum/server/model/CheckpointFromDB.scala | 41 ++++++++++++++++++- .../CheckpointRepositoryUnitTests.scala | 11 ++++- 13 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala new file mode 100644 index 000000000..e214878b7 --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.dto + +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} + +import java.time.ZonedDateTime +import java.util.UUID + +case class CheckpointV2DTO( + id: UUID, + name: String, + author: String, + measuredByAtumAgent: Boolean = false, + partitioningId: Long, + processStartTime: ZonedDateTime, + processEndTime: Option[ZonedDateTime], + measurements: Set[MeasurementDTO] +) + +object CheckpointV2DTO { + implicit val decodeCheckpointDTO: Decoder[CheckpointV2DTO] = deriveDecoder + implicit val encodeCheckpointDTO: Encoder[CheckpointV2DTO] = deriveEncoder +} diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 5a07279b6..1d0c61c7f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -57,6 +57,7 @@ object Main extends ZIOAppDefault with Server { GetPartitioningCheckpoints.layer, WriteCheckpoint.layer, WriteCheckpointV2.layer, + GetCheckpointV2.layer, GetFlowCheckpoints.layer, PostgresDatabaseProvider.layer, TransactorProvider.layer, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index 61e50836d..15026468f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.controller -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.IO @@ -35,6 +35,6 @@ trait CheckpointController { def getPartitioningCheckpointV2( partitioningId: Long, checkpointId: String - ): IO[ErrorResponse, SingleSuccessResponse[CheckpointDTO]] + ): IO[ErrorResponse, SingleSuccessResponse[CheckpointV2DTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 60b56714f..40f48b1ba 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.controller -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.server.model.ErrorResponse @@ -54,9 +54,9 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che override def getPartitioningCheckpointV2( partitioningId: Long, checkpointId: String - ): IO[ErrorResponse, SingleSuccessResponse[CheckpointDTO]] = { + ): IO[ErrorResponse, SingleSuccessResponse[CheckpointV2DTO]] = { mapToSingleSuccessResponse( - serviceCall[CheckpointDTO, CheckpointDTO]( + serviceCall[CheckpointV2DTO, CheckpointV2DTO]( checkpointService.getCheckpointV2(partitioningId, checkpointId), identity ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala index eee81f283..d68002c9c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator -import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import za.co.absa.atum.server.model.{CheckpointFromDB, GetCheckpointV2Args} @@ -26,6 +25,9 @@ import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ +import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import doobie.postgres.implicits._ +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet class GetCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[GetCheckpointV2Args, CheckpointFromDB, Task](input => diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 26af51dc1..0eaf32c88 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -76,11 +76,11 @@ trait Endpoints extends BaseEndpoints { } protected val getPartitioningCheckpointEndpointV2 - : PublicEndpoint[(Long, String), ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { + : PublicEndpoint[(Long, String), ErrorResponse, SingleSuccessResponse[CheckpointV2DTO], Any] = { apiV2.get .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints / path[String]("checkpointId")) .out(statusCode(StatusCode.Ok)) - .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) + .out(jsonBody[SingleSuccessResponse[CheckpointV2DTO]]) .errorOutVariantPrepend(notFoundErrorOneOfVariant) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 36004347c..7b3e5ab28 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -25,7 +25,7 @@ import sttp.tapir.server.http4s.ztapir.ZHttp4sServerInterpreter import sttp.tapir.server.interceptor.metrics.MetricsRequestInterceptor import sttp.tapir.swagger.bundle.SwaggerInterpreter import sttp.tapir.ztapir._ -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.Constants.{SwaggerApiName, SwaggerApiVersion} import za.co.absa.atum.server.api.controller.{CheckpointController, FlowController, PartitioningController} import za.co.absa.atum.server.api.http.ApiPaths.V2Paths @@ -60,7 +60,11 @@ trait Routes extends Endpoints with ServerOptions { createOrUpdateAdditionalDataEndpointV2, PartitioningController.createOrUpdateAdditionalDataV2 ), - createServerEndpoint( + createServerEndpoint[ + (Long, String), + ErrorResponse, + SingleSuccessResponse[CheckpointV2DTO] + ]( getPartitioningCheckpointEndpointV2, { case(partitioningId: Long, checkpointId: String) => diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index b887d11b8..c285d85e0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -18,6 +18,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.exception.DatabaseError +import za.co.absa.atum.server.model.CheckpointFromDB import zio._ import zio.macros.accessible @@ -25,5 +26,5 @@ import zio.macros.accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] def writeCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] - def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointDTO] + def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointFromDB] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 693500902..60e4fb49e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.atum.server.model.WriteCheckpointV2Args +import za.co.absa.atum.server.model.{CheckpointFromDB, GetCheckpointV2Args, WriteCheckpointV2Args} import zio._ import zio.interop.catz.asyncInstance @@ -41,8 +41,8 @@ class CheckpointRepositoryImpl( ) } - override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointDTO] = { - dbSingleResultCallWithStatus(getCheckpointV2Fn(partitioningId, checkpointId), "getCheckpoint") + override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointFromDB] = { + dbSingleResultCallWithStatus(getCheckpointV2Fn(GetCheckpointV2Args(partitioningId, checkpointId)), "getCheckpoint") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index 89831db53..2536f37bd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.ServiceError import zio.IO import zio.macros.accessible @@ -25,5 +25,5 @@ import zio.macros.accessible trait CheckpointService { def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] def saveCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] - def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointDTO] + def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointV2DTO] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index b3ae07b70..f9ae3c4c4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError.GeneralServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository @@ -39,14 +39,13 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends ) } - override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointDTO] = { + override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointV2DTO] = { for { checkpointFromDB <- repositoryCall( checkpointRepository.getCheckpointV2(partitioningId, checkpointId), "getCheckpoint" ) - // why CheckpointDTO contains partitioningDTO??? - checkpointDTO <- ZIO.fromEither(CheckpointFromDB.toCheckpointDTO(checkpointFromDB)) + checkpointDTO <- ZIO.fromEither(CheckpointFromDB.toCheckpointV2DTO(partitioningId, checkpointFromDB)) .mapError(error => GeneralServiceError(error.getMessage)) } yield checkpointDTO diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 0eba1e01c..137aca2ff 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -16,7 +16,14 @@ package za.co.absa.atum.server.model -import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} +import za.co.absa.atum.model.dto.{ + CheckpointDTO, + CheckpointV2DTO, + MeasureDTO, + MeasureResultDTO, + MeasurementDTO, + PartitioningDTO +} import io.circe.{DecodingFailure, Json} import java.time.ZonedDateTime @@ -70,4 +77,36 @@ object CheckpointFromDB { } } + def toCheckpointV2DTO( + partitioningId: Long, + checkpointQueryResult: CheckpointFromDB + ): Either[DecodingFailure, CheckpointV2DTO] = { + val measureResultOrErr = checkpointQueryResult.measurementValue.get.as[MeasureResultDTO] + + measureResultOrErr match { + case Left(err) => Left(err) + case Right(measureResult) => + Right( + CheckpointV2DTO( + id = checkpointQueryResult.idCheckpoint.get, + name = checkpointQueryResult.checkpointName.get, + author = checkpointQueryResult.author.get, + measuredByAtumAgent = checkpointQueryResult.measuredByAtumAgent.get, + partitioningId = partitioningId, + processStartTime = checkpointQueryResult.checkpointStartTime.get, + processEndTime = checkpointQueryResult.checkpointEndTime, + measurements = Set( + MeasurementDTO( + measure = MeasureDTO( + measureName = checkpointQueryResult.measureName.get, + measuredColumns = checkpointQueryResult.measuredColumns.get + ), + result = measureResult + ) + ) + ) + ) + } + } + } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index b2fd3f4f3..a96d6ef1f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.DatabaseError._ @@ -34,6 +34,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) private val writeCheckpointMockV2: WriteCheckpointV2 = mock(classOf[WriteCheckpointV2]) + private val getCheckpointMockV2: GetCheckpointV2 = mock(classOf[GetCheckpointV2]) when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(writeCheckpointMock.apply(checkpointDTO2)) @@ -51,6 +52,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointMockV2) + private val getCheckpointV2MockLayer = ZLayer.succeed(getCheckpointMockV2) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -89,7 +91,12 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ) - ).provide(CheckpointRepositoryImpl.layer, writeCheckpointMockLayer, writeCheckpointV2MockLayer) + ).provide( + CheckpointRepositoryImpl.layer, + writeCheckpointMockLayer, + writeCheckpointV2MockLayer, + getCheckpointV2MockLayer + ) } From 69169ba8280775ab3977d089ed4a53b915b7759b Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 15:42:37 +0200 Subject: [PATCH 16/50] checkpointv2dto --- .../absa/atum/model/dto/CheckpointV2DTO.scala | 39 +++++++++++++++++++ .../api/controller/CheckpointController.scala | 6 +-- .../controller/CheckpointControllerImpl.scala | 14 +++---- .../runs/functions/WriteCheckpointV2.scala | 14 +++---- .../absa/atum/server/api/http/Endpoints.scala | 6 +-- .../co/absa/atum/server/api/http/Routes.scala | 10 ++--- .../api/repository/CheckpointRepository.scala | 4 +- .../repository/CheckpointRepositoryImpl.scala | 6 +-- .../api/service/CheckpointService.scala | 4 +- .../api/service/CheckpointServiceImpl.scala | 6 +-- .../atum/server/model/CheckpointFromDB.scala | 34 +++++++++++++++- .../server/model/WriteCheckpointV2Args.scala | 4 +- .../za/co/absa/atum/server/api/TestData.scala | 24 ++++++++++++ .../CheckpointControllerUnitTests.scala | 20 +++++----- .../WriteCheckpointV2IntegrationTests.scala | 7 ++-- .../PostCheckpointEndpointV2UnitTests.scala | 28 ++++++------- .../CheckpointRepositoryUnitTests.scala | 12 +++--- .../service/CheckpointServiceUnitTests.scala | 13 +++---- 18 files changed, 172 insertions(+), 79 deletions(-) create mode 100644 model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala new file mode 100644 index 000000000..47020810c --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.dto + +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} + +import java.time.ZonedDateTime +import java.util.UUID + +case class CheckpointV2DTO( + id: UUID, + name: String, + author: String, + measuredByAtumAgent: Boolean = false, +// partitioningId: Long, + processStartTime: ZonedDateTime, + processEndTime: Option[ZonedDateTime], + measurements: Set[MeasurementDTO] + ) + +object CheckpointV2DTO { + implicit val decodeCheckpointDTO: Decoder[CheckpointV2DTO] = deriveDecoder + implicit val encodeCheckpointDTO: Encoder[CheckpointV2DTO] = deriveEncoder +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index d4bb2229d..d22e2e0b9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.controller -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.IO @@ -29,7 +29,7 @@ trait CheckpointController { def postCheckpointV2( partitioningId: Long, - checkpointDTO: CheckpointDTO - ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String)] + checkpointV2DTO: CheckpointV2DTO + ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String)] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 44fc93372..55b8fd58f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.controller -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.server.model.ErrorResponse @@ -36,17 +36,17 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che override def postCheckpointV2( partitioningId: Long, - checkpointDTO: CheckpointDTO - ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String)] = { + checkpointV2DTO: CheckpointV2DTO + ): IO[ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String)] = { for { response <- mapToSingleSuccessResponse( - serviceCall[Unit, CheckpointDTO]( - checkpointService.saveCheckpointV2(partitioningId, checkpointDTO), - _ => checkpointDTO + serviceCall[Unit, CheckpointV2DTO]( + checkpointService.saveCheckpointV2(partitioningId, checkpointV2DTO), + _ => checkpointV2DTO ) ) uri <- createResourceUri( - Seq(V2Paths.Partitionings, partitioningId.toString, V2Paths.Checkpoints, checkpointDTO.id.toString) + Seq(V2Paths.Partitionings, partitioningId.toString, V2Paths.Checkpoints, checkpointV2DTO.id.toString) ) } yield (response, uri) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala index f440ec6f7..c1990f0c4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala @@ -34,13 +34,13 @@ class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => Seq( fr"${args.partitioningId}", - fr"${args.checkpointDTO.id}", - fr"${args.checkpointDTO.name}", - fr"${args.checkpointDTO.processStartTime}", - fr"${args.checkpointDTO.processEndTime}", - fr"${args.checkpointDTO.measurements.toList.map(_.asJson)}", - fr"${args.checkpointDTO.measuredByAtumAgent}", - fr"${args.checkpointDTO.author}" + fr"${args.checkpointV2DTO.id}", + fr"${args.checkpointV2DTO.name}", + fr"${args.checkpointV2DTO.processStartTime}", + fr"${args.checkpointV2DTO.processEndTime}", + fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", + fr"${args.checkpointV2DTO.measuredByAtumAgent}", + fr"${args.checkpointV2DTO.author}" ) ) with StandardStatusHandling diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 5c54774c1..19b25716a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -38,12 +38,12 @@ trait Endpoints extends BaseEndpoints { } protected val postCheckpointEndpointV2 - : PublicEndpoint[(Long, CheckpointDTO), ErrorResponse, (SingleSuccessResponse[CheckpointDTO], String), Any] = { + : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { apiV2.post .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints) - .in(jsonBody[CheckpointDTO]) + .in(jsonBody[CheckpointV2DTO]) .out(statusCode(StatusCode.Created)) - .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) + .out(jsonBody[SingleSuccessResponse[CheckpointV2DTO]]) .out(header[String]("Location")) .errorOutVariantPrepend(conflictErrorOneOfVariant) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index af229a453..d026abd30 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -25,7 +25,7 @@ import sttp.tapir.server.http4s.ztapir.ZHttp4sServerInterpreter import sttp.tapir.server.interceptor.metrics.MetricsRequestInterceptor import sttp.tapir.swagger.bundle.SwaggerInterpreter import sttp.tapir.ztapir._ -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.Constants.{SwaggerApiName, SwaggerApiVersion} import za.co.absa.atum.server.api.controller.{CheckpointController, FlowController, PartitioningController} import za.co.absa.atum.server.api.http.ApiPaths.V2Paths @@ -45,13 +45,13 @@ trait Routes extends Endpoints with ServerOptions { val endpoints = List( createServerEndpoint(createCheckpointEndpointV1, CheckpointController.createCheckpointV1), createServerEndpoint[ - (Long, CheckpointDTO), + (Long, CheckpointV2DTO), ErrorResponse, - (SingleSuccessResponse[CheckpointDTO], String) + (SingleSuccessResponse[CheckpointV2DTO], String) ]( postCheckpointEndpointV2, - { case (partitioningId: Long, checkpointDTO: CheckpointDTO) => - CheckpointController.postCheckpointV2(partitioningId, checkpointDTO) + { case (partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) => + CheckpointController.postCheckpointV2(partitioningId, checkpointV2DTO) } ), createServerEndpoint(createPartitioningEndpointV1, PartitioningController.createPartitioningIfNotExistsV1), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index a2237a98b..a0e51264d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.DatabaseError import zio._ import zio.macros.accessible @@ -24,5 +24,5 @@ import zio.macros.accessible @accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] - def writeCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] + def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 9d117c663..dd9646c14 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.WriteCheckpointV2Args @@ -31,9 +31,9 @@ class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint, writeCheckpoi dbSingleResultCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") } - override def writeCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { + override def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus( - writeCheckpointV2Fn(WriteCheckpointV2Args(partitioningId, checkpointDTO)), + writeCheckpointV2Fn(WriteCheckpointV2Args(partitioningId, checkpointV2DTO)), "writeCheckpoint" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index ce026ff05..0d41d9dd3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.ServiceError import zio.IO import zio.macros.accessible @@ -24,5 +24,5 @@ import zio.macros.accessible @accessible trait CheckpointService { def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] - def saveCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] + def saveCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[ServiceError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index f8e772c2c..600b037ea 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.service -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository import zio._ @@ -30,9 +30,9 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends ) } - override def saveCheckpointV2(partitioningId: Long, checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] = { + override def saveCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[ServiceError, Unit] = { repositoryCall( - checkpointRepository.writeCheckpointV2(partitioningId, checkpointDTO), + checkpointRepository.writeCheckpointV2(partitioningId, checkpointV2DTO), "saveCheckpoint" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 0eba1e01c..95fc049f1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.model -import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import io.circe.{DecodingFailure, Json} import java.time.ZonedDateTime @@ -70,4 +70,36 @@ object CheckpointFromDB { } } + def toCheckpointV2DTO( +// partitioningId: Long, + checkpointQueryResult: CheckpointFromDB + ): Either[DecodingFailure, CheckpointV2DTO] = { + val measureResultOrErr = checkpointQueryResult.measurementValue.get.as[MeasureResultDTO] + + measureResultOrErr match { + case Left(err) => Left(err) + case Right(measureResult) => + Right( + CheckpointV2DTO( + id = checkpointQueryResult.idCheckpoint.get, + name = checkpointQueryResult.checkpointName.get, + author = checkpointQueryResult.author.get, + measuredByAtumAgent = checkpointQueryResult.measuredByAtumAgent.get, +// partitioningId = partitioningId, + processStartTime = checkpointQueryResult.checkpointStartTime.get, + processEndTime = checkpointQueryResult.checkpointEndTime, + measurements = Set( + MeasurementDTO( + measure = MeasureDTO( + measureName = checkpointQueryResult.measureName.get, + measuredColumns = checkpointQueryResult.measuredColumns.get + ), + result = measureResult + ) + ) + ) + ) + } + } + } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala b/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala index 1a7a80828..3ce505ef6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala @@ -16,6 +16,6 @@ package za.co.absa.atum.server.model -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.CheckpointV2DTO -case class WriteCheckpointV2Args(partitioningId: Long, checkpointDTO: CheckpointDTO) +case class WriteCheckpointV2Args(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 2963abf3b..5ce0fe048 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -193,6 +193,30 @@ trait TestData { protected val checkpointDTO3: CheckpointDTO = checkpointDTO1.copy(id = UUID.randomUUID()) protected val checkpointDTO4: CheckpointDTO = checkpointDTO1.copy(id = UUID.randomUUID()) + // Checkpoint V2 DTO + protected val checkpointV2DTO1: CheckpointV2DTO = CheckpointV2DTO( + id = UUID.randomUUID(), + name = checkpointQueryDTO1.checkpointName.get, + author = "author", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.now(), + processEndTime = Some(ZonedDateTime.now()), + measurements = measurementsDTO1.toSet + ) + + protected val checkpointV2DTO2: CheckpointV2DTO = CheckpointV2DTO( + id = UUID.randomUUID(), + name = checkpointQueryDTO2.checkpointName.get, + author = "author2", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.now(), + processEndTime = Some(ZonedDateTime.now()), + measurements = measurementsDTO2.toSet + ) + + protected val checkpointV2DTO3: CheckpointV2DTO = checkpointV2DTO1.copy(id = UUID.randomUUID()) + protected val checkpointV2DTO4: CheckpointV2DTO = checkpointV2DTO1.copy(id = UUID.randomUUID()) + // Checkpoint From DB protected val checkpointFromDB1: CheckpointFromDB = CheckpointFromDB( idCheckpoint = Some(checkpointDTO1.id), diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 40b211191..93f364d49 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError._ @@ -42,10 +42,10 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { private val partitioningId = 1L - when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointDTO1)).thenReturn(ZIO.unit) - when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointDTO2)) + when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointV2DTO1)).thenReturn(ZIO.unit) + when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointV2DTO2)) .thenReturn(ZIO.fail(GeneralServiceError("error in data"))) - when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointDTO3)) + when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointV2DTO3)) .thenReturn(ZIO.fail(ConflictServiceError("boom!"))) private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) @@ -76,24 +76,24 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { for { _ <- TestSystem.putEnv("HOSTNAME", host) sslConfig <- ZIO.config[SslConfig](SslConfig.config) - result <- CheckpointController.postCheckpointV2(partitioningId, checkpointDTO1) + result <- CheckpointController.postCheckpointV2(partitioningId, checkpointV2DTO1) protocol = if (sslConfig.enabled) "https" else "http" port = if (sslConfig.enabled) 8443 else 8080 - path = s"${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}/${checkpointDTO1.id}" + path = s"${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}/${checkpointV2DTO1.id}" expectedUri = s"$protocol://$host:$port/$path" } yield assertTrue( - result._1.isInstanceOf[SingleSuccessResponse[CheckpointDTO]] - && result._1.data == checkpointDTO1 + result._1.isInstanceOf[SingleSuccessResponse[CheckpointV2DTO]] + && result._1.data == checkpointV2DTO1 && result._2 == expectedUri ) }, test("Returns expected ConflictServiceError") { - assertZIO(CheckpointController.postCheckpointV2(1L, checkpointDTO2).exit)( + assertZIO(CheckpointController.postCheckpointV2(1L, checkpointV2DTO2).exit)( failsWithA[InternalServerErrorResponse] ) }, test("Returns expected ConflictServiceError") { - assertZIO(CheckpointController.postCheckpointV2(1L, checkpointDTO3).exit)( + assertZIO(CheckpointController.postCheckpointV2(1L, checkpointV2DTO3).exit)( failsWithA[ConflictErrorResponse] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index ce469c2f8..7b220b1d0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -1,7 +1,7 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} +import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider @@ -21,11 +21,10 @@ object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { suite("WriteCheckpointV2Suite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { - val checkpointDTO = CheckpointDTO( + val checkpointV2DTO = CheckpointV2DTO( id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = Set( @@ -34,7 +33,7 @@ object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { ) for { writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] - result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointDTO)) + result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointV2DTO)) } yield assertTrue(result == Right(Row(FunctionStatus(11, "Checkpoint created"),()))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala index f2791578a..760a1081e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala @@ -23,7 +23,7 @@ import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{ConflictErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} @@ -36,20 +36,20 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w private val checkpointControllerMock = mock(classOf[CheckpointController]) - when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO1)) - .thenReturn(ZIO.succeed((SingleSuccessResponse(checkpointDTO1, uuid), "some location"))) - when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO2)) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO1)) + .thenReturn(ZIO.succeed((SingleSuccessResponse(checkpointV2DTO1, uuid), "some location"))) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO2)) .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) - when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO3)) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO3)) .thenReturn(ZIO.fail(InternalServerErrorResponse("error"))) - when(checkpointControllerMock.postCheckpointV2(1L, checkpointDTO4)) + when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO4)) .thenReturn(ZIO.fail(ConflictErrorResponse("error"))) private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) private val postCheckpointServerEndpointV2 = postCheckpointEndpointV2 - .zServerLogic({ case (partitioningId: Long, checkpointDTO: CheckpointDTO) => - CheckpointController.postCheckpointV2(partitioningId, checkpointDTO) + .zServerLogic({ case (partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) => + CheckpointController.postCheckpointV2(partitioningId, checkpointV2DTO) }) def spec: Spec[TestEnvironment with Scope, Any] = { @@ -60,12 +60,12 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w val request = basicRequest .post(uri"https://test.com/api/v2/partitionings/1/checkpoints") - .response(asJson[SingleSuccessResponse[CheckpointDTO]]) + .response(asJson[SingleSuccessResponse[CheckpointV2DTO]]) suite("CreateCheckpointEndpointSuite")( test("Returns expected CheckpointDTO") { val response = request - .body(checkpointDTO1) + .body(checkpointV2DTO1) .send(backendStub) val body = response.map(_.body) @@ -73,12 +73,12 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w val header = response.map(_.header("Location")) assertZIO(body <&> statusCode <&> header)( - equalTo(Right(SingleSuccessResponse(checkpointDTO1, uuid)), StatusCode.Created, Some("some location")) + equalTo(Right(SingleSuccessResponse(checkpointV2DTO1, uuid)), StatusCode.Created, Some("some location")) ) }, test("Returns expected BadRequest") { val response = request - .body(checkpointDTO2) + .body(checkpointV2DTO2) .send(backendStub) val statusCode = response.map(_.code) @@ -87,7 +87,7 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected InternalServerError") { val response = request - .body(checkpointDTO3) + .body(checkpointV2DTO3) .send(backendStub) val statusCode = response.map(_.code) @@ -96,7 +96,7 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected ConflictError") { val response = request - .body(checkpointDTO4) + .body(checkpointV2DTO4) .send(backendStub) val statusCode = response.map(_.code) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index b2fd3f4f3..a7cea896e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -42,11 +42,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val partitioningId = 1L - when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointDTO1))) + when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO1))) .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) - when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointDTO2))) + when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO2))) .thenReturn(ZIO.left(DataConflictException(FunctionStatus(31, "conflict in data")))) - when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointDTO3))) + when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) @@ -75,16 +75,16 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("WriteCheckpointV2Suite")( test("Returns an expected Unit") { for { - result <- CheckpointRepository.writeCheckpointV2(partitioningId, checkpointDTO1) + result <- CheckpointRepository.writeCheckpointV2(partitioningId, checkpointV2DTO1) } yield assertTrue(result == ()) }, test("Fails with an expected ConflictDatabaseError") { - assertZIO(CheckpointRepository.writeCheckpointV2(partitioningId, checkpointDTO2).exit)( + assertZIO(CheckpointRepository.writeCheckpointV2(partitioningId, checkpointV2DTO2).exit)( failsWithA[ConflictDatabaseError] ) }, test("Fails with an expected GeneralDatabaseError") { - assertZIO(CheckpointRepository.writeCheckpointV2(partitioningId, checkpointDTO3).exit)( + assertZIO(CheckpointRepository.writeCheckpointV2(partitioningId, checkpointV2DTO3).exit)( failsWithA[GeneralDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index ab896fa33..79d9e7127 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -19,7 +19,6 @@ package za.co.absa.atum.server.api.service import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.repository.CheckpointRepository import zio.test.Assertion.failsWithA @@ -37,10 +36,10 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { private val partitioningId = 1L - when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointDTO1)).thenReturn(ZIO.unit) - when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointDTO2)) + when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointV2DTO1)).thenReturn(ZIO.unit) + when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointV2DTO2)) .thenReturn(ZIO.fail(ConflictDatabaseError("conflict in data"))) - when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointDTO3)) + when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointV2DTO3)) .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -68,16 +67,16 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { suite("SaveCheckpointV2Suite")( test("Returns an expected Unit") { for { - result <- CheckpointService.saveCheckpointV2(partitioningId, checkpointDTO1) + result <- CheckpointService.saveCheckpointV2(partitioningId, checkpointV2DTO1) } yield assertTrue(result == ()) }, test("Fails with an expected ConflictServiceError") { - assertZIO(CheckpointService.saveCheckpointV2(partitioningId, checkpointDTO2).exit)( + assertZIO(CheckpointService.saveCheckpointV2(partitioningId, checkpointV2DTO2).exit)( failsWithA[ConflictServiceError] ) }, test("Fails with an expected GeneralServiceError") { - assertZIO(CheckpointService.saveCheckpointV2(partitioningId, checkpointDTO3).exit)( + assertZIO(CheckpointService.saveCheckpointV2(partitioningId, checkpointV2DTO3).exit)( failsWithA[GeneralServiceError] ) } From 519790ae51890f9449410b2f82c10b07e524ae27 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 15:58:03 +0200 Subject: [PATCH 17/50] checkpointv2dto --- .../runs/V1.9.2__write_checkpoint_v2.sql | 7 +++++++ .../WriteCheckpointV2IntegrationTests.scala | 21 +++++++++++++++++++ .../absa/atum/model/dto/CheckpointV2DTO.scala | 1 - 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql index 979b8ba83..4feb584d0 100644 --- a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql @@ -57,9 +57,16 @@ $$ -- Status codes: -- 11 - Checkpoint created -- 31 - Conflict, checkpoint already present +-- 32 - Partitioning not found -- ------------------------------------------------------------------------------- BEGIN + -- Check if partitioning exists + IF NOT EXISTS (SELECT 1 FROM runs.partitionings WHERE id_partitioning = i_partitioning_id) THEN + status := 32; + status_text := 'Partitioning not found'; + RETURN; + END IF; PERFORM 1 FROM runs.checkpoints CP diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala index 85d4da20d..28f59aba3 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala @@ -261,4 +261,25 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { } } + test("Partitioning of the checkpoint does not exist") { + val uuid = UUID.randomUUID + val count = table("runs.checkpoints").count() + function("runs.write_checkpoint") + .setParam("i_partitioning", 123456789L) + .setParam("i_id_checkpoint", uuid) + .setParam("i_checkpoint_name", "Won't go in") + .setParam("i_process_start_time", now()) + .setParamNull("i_process_end_time") + .setParamNull("i_measurements") + .setParam("i_measured_by_atum_agent", true) + .setParam("i_by_user", "J. Robert Oppenheimer") + .execute { queryResult => + assert(queryResult.hasNext) + val row = queryResult.next() + assert(row.getInt("status").contains(32)) + assert(row.getString("status_text").contains("Partitioning not found")) + } + assert(table("runs.checkpoints").count() == count) + } + } diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala index 47020810c..0ae4ad3fa 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -27,7 +27,6 @@ case class CheckpointV2DTO( name: String, author: String, measuredByAtumAgent: Boolean = false, -// partitioningId: Long, processStartTime: ZonedDateTime, processEndTime: Option[ZonedDateTime], measurements: Set[MeasurementDTO] From 6e3588bbba24234b3d5c50bccb7dee87804a58d2 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 16:01:41 +0200 Subject: [PATCH 18/50] fix --- .../atum/database/runs/WriteCheckpointV2IntegrationTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala index 28f59aba3..7fb6f1c35 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala @@ -264,7 +264,7 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { test("Partitioning of the checkpoint does not exist") { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() - function("runs.write_checkpoint") + function("runs.write_checkpoint_v2") .setParam("i_partitioning", 123456789L) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") From 61d587b63cb41a16c2e2efce538c5a62a4c51248 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 16:08:51 +0200 Subject: [PATCH 19/50] fix --- .../atum/database/runs/WriteCheckpointV2IntegrationTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala index 7fb6f1c35..0db5399fd 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala @@ -265,7 +265,7 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() function("runs.write_checkpoint_v2") - .setParam("i_partitioning", 123456789L) + .setParam("i_partitioning", 0L) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) From a46e3aa55b941f24f198a67ce986b8baa287a341 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 16:23:12 +0200 Subject: [PATCH 20/50] fix --- .../WriteCheckpointV2IntegrationTests.scala | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala index 0db5399fd..2ebdb3d29 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala @@ -40,6 +40,42 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { |""".stripMargin ) + private val measurements = + """ + |{ + | "{ + | \"measure\": { + | \"measureName\": \"count\", + | \"measuredColumns\": [] + | }, + | \"result\":{ + | \"value\":\"3\", + | \"type\":\"int\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"col1\"] + | }, + | \"result\":{ + | \"value\":\"3.14\", + | \"type\":\"double\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"a\",\"b\"] + | }, + | \"result\":{ + | \"value\":\"2.71\", + | \"type\":\"double\" + | } + | }" + |} + |""".stripMargin + test("Write new checkpoint without data") { val uuid = UUID.randomUUID val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") @@ -98,41 +134,7 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { val user = "Franz Kafka" val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") - val measurements = - """ - |{ - | "{ - | \"measure\": { - | \"measureName\": \"count\", - | \"measuredColumns\": [] - | }, - | \"result\":{ - | \"value\":\"3\", - | \"type\":\"int\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"col1\"] - | }, - | \"result\":{ - | \"value\":\"3.14\", - | \"type\":\"double\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"a\",\"b\"] - | }, - | \"result\":{ - | \"value\":\"2.71\", - | \"type\":\"double\" - | } - | }" - |} - |""".stripMargin + table("runs.partitionings").insert( add("partitioning", partitioning) @@ -265,7 +267,7 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() function("runs.write_checkpoint_v2") - .setParam("i_partitioning", 0L) + .setParam("i_partitioning_id", 0L) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) From 31652e5aceb50b0488681e7f94c6682a806ecb8b Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 16:38:13 +0200 Subject: [PATCH 21/50] fix --- .../runs/functions/WriteCheckpointV2IntegrationTests.scala | 3 ++- .../server/api/repository/CheckpointRepositoryUnitTests.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index 7b220b1d0..0c21cea04 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -7,6 +7,7 @@ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.model.WriteCheckpointV2Args +import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ import zio.interop.catz.asyncInstance @@ -34,7 +35,7 @@ object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { for { writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointV2DTO)) - } yield assertTrue(result == Right(Row(FunctionStatus(11, "Checkpoint created"),()))) + } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( WriteCheckpointV2.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index a7cea896e..198a8d31c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -45,7 +45,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO1))) .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO2))) - .thenReturn(ZIO.left(DataConflictException(FunctionStatus(31, "conflict in data")))) + .thenReturn(ZIO.left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) From c45b35d3e369103b93473ef1b9523315abba7c16 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Aug 2024 16:48:36 +0200 Subject: [PATCH 22/50] fix --- .../api/controller/BaseController.scala | 13 ++++++++--- .../runs/functions/WriteCheckpointV2.scala | 23 +++++++++---------- .../absa/atum/server/api/http/Endpoints.scala | 4 ++-- .../co/absa/atum/server/api/http/Routes.scala | 5 ++-- .../api/repository/BaseRepository.scala | 5 +++- .../api/repository/CheckpointRepository.scala | 4 ++-- .../PartitioningRepositoryImpl.scala | 4 +++- .../api/service/CheckpointServiceImpl.scala | 3 ++- .../api/service/PartitioningServiceImpl.scala | 4 +++- .../atum/server/model/CheckpointFromDB.scala | 13 ++++++++--- .../atum/server/model/ErrorResponse.scala | 6 ++--- .../atum/server/model/SuccessResponse.scala | 2 +- 12 files changed, 52 insertions(+), 34 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 950a5a00d..4a791e492 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -19,7 +19,12 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.config.SslConfig -import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} +import za.co.absa.atum.server.model.{ + ConflictErrorResponse, + ErrorResponse, + InternalServerErrorResponse, + NotFoundErrorResponse +} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ @@ -56,14 +61,16 @@ trait BaseController { protected def createResourceUri(parts: Seq[String]): IO[ErrorResponse, String] = { for { - hostname <- System.env("HOSTNAME") + hostname <- System + .env("HOSTNAME") // fails the request if the hostname is not found, we need to make sure that the hostname is always available .orElseFail(InternalServerErrorResponse("Failed to get hostname")) .flatMap { case Some(value) => ZIO.succeed(value) case None => ZIO.fail(InternalServerErrorResponse("Failed to get hostname")) } - sslConfig <- ZIO.config[SslConfig](SslConfig.config) + sslConfig <- ZIO + .config[SslConfig](SslConfig.config) .orElseFail(InternalServerErrorResponse("Failed to get SSL config")) } yield { val protocol = if (sslConfig.enabled) "https" else "http" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala index c1990f0c4..4705f5ff3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala @@ -31,18 +31,18 @@ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => - Seq( - fr"${args.partitioningId}", - fr"${args.checkpointV2DTO.id}", - fr"${args.checkpointV2DTO.name}", - fr"${args.checkpointV2DTO.processStartTime}", - fr"${args.checkpointV2DTO.processEndTime}", - fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", - fr"${args.checkpointV2DTO.measuredByAtumAgent}", - fr"${args.checkpointV2DTO.author}" + extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => + Seq( + fr"${args.partitioningId}", + fr"${args.checkpointV2DTO.id}", + fr"${args.checkpointV2DTO.name}", + fr"${args.checkpointV2DTO.processStartTime}", + fr"${args.checkpointV2DTO.processEndTime}", + fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", + fr"${args.checkpointV2DTO.measuredByAtumAgent}", + fr"${args.checkpointV2DTO.author}" + ) ) - ) with StandardStatusHandling object WriteCheckpointV2 { @@ -52,4 +52,3 @@ object WriteCheckpointV2 { } yield new WriteCheckpointV2()(Runs, dbProvider.dbEngine) } } - diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 11eeb0cf7..4b5b3b569 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -38,7 +38,7 @@ trait Endpoints extends BaseEndpoints { } protected val postCheckpointEndpointV2 - : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { + : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { apiV2.post .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints) .in(jsonBody[CheckpointV2DTO]) @@ -76,7 +76,7 @@ trait Endpoints extends BaseEndpoints { } protected val getPartitioningCheckpointEndpointV2 - : PublicEndpoint[(Long, String), ErrorResponse, SingleSuccessResponse[CheckpointV2DTO], Any] = { + : PublicEndpoint[(Long, String), ErrorResponse, SingleSuccessResponse[CheckpointV2DTO], Any] = { apiV2.get .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints / path[String]("checkpointId")) .out(statusCode(StatusCode.Ok)) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 14d5c6d1e..f002e7eb8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -66,9 +66,8 @@ trait Routes extends Endpoints with ServerOptions { SingleSuccessResponse[CheckpointV2DTO] ]( getPartitioningCheckpointEndpointV2, - { - case(partitioningId: Long, checkpointId: String) => - CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointId) + { case (partitioningId: Long, checkpointId: String) => + CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointId) } ), createServerEndpoint(getPartitioningCheckpointsEndpointV2, PartitioningController.getPartitioningCheckpointsV2), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala index 866d32d8a..d7c0b65d9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/BaseRepository.scala @@ -53,7 +53,10 @@ trait BaseRepository { GeneralDatabaseError(s"Operation '$operationName' failed with unexpected error: ${error.getMessage}") } - protected def dbSingleResultCallWithStatus[R](dbFuncCall: Task[FailedOrRow[R]], operationName: String): IO[DatabaseError, R] = { + protected def dbSingleResultCallWithStatus[R]( + dbFuncCall: Task[FailedOrRow[R]], + operationName: String + ): IO[DatabaseError, R] = { logAndReturn(operationName, dbFuncCall) .flatMap { case Left(statusException) => ZIO.fail(statusException) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index 404856602..1608615f2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.model.CheckpointFromDB import zio._ @@ -25,6 +25,6 @@ import zio.macros.accessible @accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] - def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointFromDB] def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] + def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointFromDB] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 523a98f0f..68ff69276 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -65,7 +65,9 @@ class PartitioningRepositoryImpl( }) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, InitialAdditionalDataDTO] = { + override def getPartitioningAdditionalData( + partitioning: PartitioningDTO + ): IO[DatabaseError, InitialAdditionalDataDTO] = { dbMultipleResultCallWithAggregatedStatus( getPartitioningAdditionalDataFn(partitioning), "getPartitioningAdditionalData" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index 7324d3a0f..8607176b0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -45,7 +45,8 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointRepository.getCheckpointV2(partitioningId, checkpointId), "getCheckpoint" ) - checkpointDTO <- ZIO.fromEither(CheckpointFromDB.toCheckpointV2DTO(partitioningId, checkpointFromDB)) + checkpointDTO <- ZIO + .fromEither(CheckpointFromDB.toCheckpointV2DTO(checkpointFromDB)) .mapError(error => GeneralServiceError(error.getMessage)) } yield checkpointDTO diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 3066878c3..eebb7a2ba 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -48,7 +48,9 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) ) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[ServiceError, InitialAdditionalDataDTO] = { + override def getPartitioningAdditionalData( + partitioning: PartitioningDTO + ): IO[ServiceError, InitialAdditionalDataDTO] = { repositoryCall( partitioningRepository.getPartitioningAdditionalData(partitioning), "getPartitioningAdditionalData" diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 943e49de8..09c55e73e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -24,7 +24,14 @@ import za.co.absa.atum.model.dto.{ MeasurementDTO, PartitioningDTO } -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} +import za.co.absa.atum.model.dto.{ + CheckpointDTO, + CheckpointV2DTO, + MeasureDTO, + MeasureResultDTO, + MeasurementDTO, + PartitioningDTO +} import io.circe.{DecodingFailure, Json} import java.time.ZonedDateTime @@ -80,8 +87,8 @@ object CheckpointFromDB { def toCheckpointV2DTO( // partitioningId: Long, - checkpointQueryResult: CheckpointFromDB - ): Either[DecodingFailure, CheckpointV2DTO] = { + checkpointQueryResult: CheckpointFromDB + ): Either[DecodingFailure, CheckpointV2DTO] = { val measureResultOrErr = checkpointQueryResult.measurementValue.get.as[MeasureResultDTO] measureResultOrErr match { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index ac3be1e73..78ff0b183 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -37,16 +37,14 @@ object BadRequestResponse { implicit val encodeBadRequestResponse: Encoder[BadRequestResponse] = deriveEncoder } -final case class ConflictErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class ConflictErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object ConflictErrorResponse { implicit val decoderConflictErrorResponse: Decoder[ConflictErrorResponse] = deriveDecoder implicit val encoderConflictErrorResponse: Encoder[ConflictErrorResponse] = deriveEncoder } -final case class NotFoundErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class NotFoundErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object NotFoundErrorResponse { implicit val decoderNotFoundErrorResponse: Decoder[NotFoundErrorResponse] = deriveDecoder diff --git a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala index 3c3341300..7fb5c6da3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala @@ -40,7 +40,7 @@ object SuccessResponse { } case class PaginatedResponse[T](data: Seq[T], pagination: Pagination, requestId: UUID = UUID.randomUUID()) - extends SuccessResponse + extends SuccessResponse object PaginatedResponse { implicit def encoder[T: Encoder]: Encoder[PaginatedResponse[T]] = deriveEncoder From 63df9ec9e06efa24d4efc9aaea33c245bae2d753 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Fri, 9 Aug 2024 09:23:02 +0200 Subject: [PATCH 23/50] fix --- ...V1.9.3__get_partitioning_checkpoint_v2.sql | 95 +++++++++++-------- .../runs/functions/GetCheckpointV2.scala | 15 ++- 2 files changed, 67 insertions(+), 43 deletions(-) diff --git a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql index 47cb9f5b9..c9d6cf375 100644 --- a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql @@ -18,15 +18,15 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoint_v2( IN i_checkpoint_id UUID, OUT status INTEGER, OUT status_text TEXT, - OUT id_checkpoint UUID, - OUT checkpoint_name TEXT, - OUT author TEXT, - OUT measured_by_atum_agent BOOLEAN, - OUT measure_name TEXT, - OUT measured_columns TEXT[], - OUT measurement_value JSONB, - OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, - OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE + OUT o_id_checkpoint UUID, + OUT o_checkpoint_name TEXT, + OUT o_author TEXT, + OUT o_measured_by_atum_agent BOOLEAN, + OUT o_measure_name TEXT, + OUT o_measured_columns TEXT[], + OUT o_measurement_value JSONB, + OUT o_checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT o_checkpoint_end_time TIMESTAMP WITH TIME ZONE ) RETURNS record AS $$ @@ -43,15 +43,15 @@ $$ -- Returns: -- status - Status code -- status_text - Status message --- id_checkpoint - ID of the checkpoint --- checkpoint_name - Name of the checkpoint --- author - Author of the checkpoint --- measuredByAtumAgent - Flag indicating whether the checkpoint was measured by ATUM agent --- measure_name - Name of the measure --- measure_columns - Columns of the measure --- measurement_value - Value of the measurement --- checkpoint_start_time - Time of the checkpoint --- checkpoint_end_time - End time of the checkpoint computation +-- o_id_checkpoint - ID of the checkpoint +-- o_checkpoint_name - Name of the checkpoint +-- o_author - Author of the checkpoint +-- o_measuredByAtumAgent - Flag indicating whether the checkpoint was measured by ATUM agent +-- o_measure_name - Name of the measure +-- o_measure_columns - Columns of the measure +-- o_measurement_value - Value of the measurement +-- o_checkpoint_start_time - Time of the checkpoint +-- o_checkpoint_end_time - End time of the checkpoint computation -- -- Status codes: -- 11 - OK @@ -65,30 +65,41 @@ BEGIN RETURN; END IF; - RETURN QUERY - SELECT - 11 AS status, - 'Ok' AS status_text, - C.id_checkpoint, - C.checkpoint_name, - C.created_by AS author, - C.measured_by_atum_agent, - md.measure_name, - md.measured_columns, - M.measurement_value, - C.process_start_time AS checkpoint_start_time, - C.process_end_time AS checkpoint_end_time - FROM - runs.checkpoints C - JOIN - runs.measurements M ON C.id_checkpoint = M.fk_checkpoint - JOIN - runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition - WHERE - C.fk_partitioning = i_partitioning_id - AND - C.id_checkpoint = i_checkpoint_id - LIMIT 1; + SELECT + 11 AS status, + 'Ok' AS status_text, + C.id_checkpoint, + C.checkpoint_name, + C.created_by AS author, + C.measured_by_atum_agent, + md.measure_name, + md.measured_columns, + M.measurement_value, + C.process_start_time AS checkpoint_start_time, + C.process_end_time AS checkpoint_end_time + INTO + status, + status_text, + o_id_checkpoint, + o_checkpoint_name, + o_author, + o_measured_by_atum_agent, + o_measure_name, + o_measured_columns, + o_measurement_value, + o_checkpoint_start_time, + o_checkpoint_end_time + FROM + runs.checkpoints C + JOIN + runs.measurements M ON C.id_checkpoint = M.fk_checkpoint + JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + WHERE + C.fk_partitioning = i_partitioning_id + AND + C.id_checkpoint = i_checkpoint_id + LIMIT 1; IF NOT FOUND THEN status := 41; diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala index d68002c9c..b96eb24d0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala @@ -36,7 +36,20 @@ class GetCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) fr"${input.checkpointId}" ) ) - with StandardStatusHandling + with StandardStatusHandling { + + override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq( + "o_id_checkpoint", + "o_checkpoint_name", + "o_author", + "o_measured_by_atum_agent", + "o_measure_name", + "o_measured_columns", + "o_measurement_value", + "o_checkpoint_start_time", + "o_checkpoint_end_time" + ) +} object GetCheckpointV2 { val layer: URLayer[PostgresDatabaseProvider, GetCheckpointV2] = ZLayer { From 900bdc285159c413370b41040159bf9fc67ee43b Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Fri, 9 Aug 2024 12:06:53 +0200 Subject: [PATCH 24/50] fix --- ...V1.9.3__get_partitioning_checkpoint_v2.sql | 79 ++++++++----------- .../absa/atum/model/dto/CheckpointV2DTO.scala | 16 ++-- .../runs/functions/GetCheckpointV2.scala | 27 ++++--- .../api/repository/CheckpointRepository.scala | 2 +- .../repository/CheckpointRepositoryImpl.scala | 64 +++++++++++++-- .../api/service/CheckpointServiceImpl.scala | 17 ++-- .../atum/server/model/CheckpointFromDB.scala | 49 +----------- .../server/model/CheckpointItemFromDB.scala | 34 ++++++++ 8 files changed, 155 insertions(+), 133 deletions(-) create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala diff --git a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql index c9d6cf375..d25efe128 100644 --- a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql @@ -18,17 +18,17 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoint_v2( IN i_checkpoint_id UUID, OUT status INTEGER, OUT status_text TEXT, - OUT o_id_checkpoint UUID, - OUT o_checkpoint_name TEXT, - OUT o_author TEXT, - OUT o_measured_by_atum_agent BOOLEAN, - OUT o_measure_name TEXT, - OUT o_measured_columns TEXT[], - OUT o_measurement_value JSONB, - OUT o_checkpoint_start_time TIMESTAMP WITH TIME ZONE, - OUT o_checkpoint_end_time TIMESTAMP WITH TIME ZONE + OUT id_checkpoint UUID, + OUT checkpoint_name TEXT, + OUT author TEXT, + OUT measured_by_atum_agent BOOLEAN, + OUT measure_name TEXT, + OUT measured_columns TEXT[], + OUT measurement_value JSONB, + OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE ) - RETURNS record AS + RETURNS SETOF record AS $$ ------------------------------------------------------------------------------- -- @@ -65,41 +65,30 @@ BEGIN RETURN; END IF; - SELECT - 11 AS status, - 'Ok' AS status_text, - C.id_checkpoint, - C.checkpoint_name, - C.created_by AS author, - C.measured_by_atum_agent, - md.measure_name, - md.measured_columns, - M.measurement_value, - C.process_start_time AS checkpoint_start_time, - C.process_end_time AS checkpoint_end_time - INTO - status, - status_text, - o_id_checkpoint, - o_checkpoint_name, - o_author, - o_measured_by_atum_agent, - o_measure_name, - o_measured_columns, - o_measurement_value, - o_checkpoint_start_time, - o_checkpoint_end_time - FROM - runs.checkpoints C - JOIN - runs.measurements M ON C.id_checkpoint = M.fk_checkpoint - JOIN - runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition - WHERE - C.fk_partitioning = i_partitioning_id - AND - C.id_checkpoint = i_checkpoint_id - LIMIT 1; + RETURN QUERY + SELECT + 11 AS status, + 'Ok' AS status_text, + C.id_checkpoint, + C.checkpoint_name, + C.created_by AS author, + C.measured_by_atum_agent, + md.measure_name, + md.measured_columns, + M.measurement_value, + C.process_start_time AS checkpoint_start_time, + C.process_end_time AS checkpoint_end_time + FROM + runs.checkpoints C + JOIN + runs.measurements M ON C.id_checkpoint = M.fk_checkpoint + JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + WHERE + C.fk_partitioning = i_partitioning_id + AND + C.id_checkpoint = i_checkpoint_id + ; IF NOT FOUND THEN status := 41; diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala index 0ae4ad3fa..ad80b373e 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -23,14 +23,14 @@ import java.time.ZonedDateTime import java.util.UUID case class CheckpointV2DTO( - id: UUID, - name: String, - author: String, - measuredByAtumAgent: Boolean = false, - processStartTime: ZonedDateTime, - processEndTime: Option[ZonedDateTime], - measurements: Set[MeasurementDTO] - ) + id: UUID, + name: String, + author: String, + measuredByAtumAgent: Boolean = false, + processStartTime: ZonedDateTime, + processEndTime: Option[ZonedDateTime], + measurements: Set[MeasurementDTO] +) object CheckpointV2DTO { implicit val decodeCheckpointDTO: Decoder[CheckpointV2DTO] = deriveDecoder diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala index b96eb24d0..263ac908e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala @@ -19,35 +19,36 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs -import za.co.absa.atum.server.model.{CheckpointFromDB, GetCheckpointV2Args} +import za.co.absa.atum.server.model.{CheckpointItemFromDB, GetCheckpointV2Args} import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine -import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus +import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet +import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator class GetCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[GetCheckpointV2Args, CheckpointFromDB, Task](input => + extends DoobieMultipleResultFunctionWithAggStatus[GetCheckpointV2Args, Option[CheckpointItemFromDB], Task](input => Seq( fr"${input.partitioningId}", fr"${input.checkpointId}" ) ) - with StandardStatusHandling { + with StandardStatusHandling with ByFirstRowStatusAggregator { override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq( - "o_id_checkpoint", - "o_checkpoint_name", - "o_author", - "o_measured_by_atum_agent", - "o_measure_name", - "o_measured_columns", - "o_measurement_value", - "o_checkpoint_start_time", - "o_checkpoint_end_time" + "id_checkpoint", + "checkpoint_name", + "author", + "measured_by_atum_agent", + "measure_name", + "measured_columns", + "measurement_value", + "checkpoint_start_time", + "checkpoint_end_time" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index 1608615f2..fb1bc802a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -26,5 +26,5 @@ import zio.macros.accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] - def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointFromDB] + def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointV2DTO] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index c4aba5180..cc8d6b31c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -16,12 +16,13 @@ package za.co.absa.atum.server.api.repository -import za.co.absa.atum.model.dto.CheckpointDTO +import io.circe.DecodingFailure import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, WriteCheckpoint, WriteCheckpointV2} -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO} import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.atum.server.model.{CheckpointFromDB, GetCheckpointV2Args, WriteCheckpointV2Args} +import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError +import za.co.absa.atum.server.api.exception.ServiceError.GeneralServiceError +import za.co.absa.atum.server.model.{CheckpointItemFromDB, GetCheckpointV2Args, WriteCheckpointV2Args} import zio._ import zio.interop.catz.asyncInstance @@ -43,9 +44,60 @@ class CheckpointRepositoryImpl( ) } - override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointFromDB] = { - dbSingleResultCallWithStatus(getCheckpointV2Fn(GetCheckpointV2Args(partitioningId, checkpointId)), "getCheckpoint") + override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointV2DTO] = { + dbMultipleResultCallWithAggregatedStatus( + getCheckpointV2Fn(GetCheckpointV2Args(partitioningId, checkpointId)), + "getCheckpoint" + ) + .map(_.flatten) + .flatMap { checkpointItems => + ZIO + .fromEither(checkpointItemsToCheckpointV2DTO(checkpointItems)) + .mapError(error => GeneralDatabaseError(error.getMessage)) + } } + + private def checkpointItemsToCheckpointV2DTO( + checkpointItems: Seq[CheckpointItemFromDB] + ): Either[DecodingFailure, CheckpointV2DTO] = { + val measurementsOrErr = checkpointItems.map { checkpointItem => + val measureResultOrErr = checkpointItem.measurementValue.as[MeasureResultDTO] + + measureResultOrErr match { + case Left(err) => Left(err) + case Right(measureResult) => + Right( + MeasurementDTO( + measure = MeasureDTO( + measureName = checkpointItem.measureName, + measuredColumns = checkpointItem.measuredColumns + ), + result = measureResult + ) + ) + } + } + + val errors = measurementsOrErr.collect { case Left(err) => err } + + if (errors.nonEmpty) { + Left(errors.head) + } else { + val measurements = measurementsOrErr.flatMap(_.toOption).toSet + Right( + CheckpointV2DTO( + id = checkpointItems.head.idCheckpoint, + name = checkpointItems.head.checkpointName, + author = checkpointItems.head.author, + measuredByAtumAgent = checkpointItems.head.measuredByAtumAgent, + processStartTime = checkpointItems.head.checkpointStartTime, + processEndTime = checkpointItems.head.checkpointEndTime, + measurements = measurements + ) + ) + } + } + } object CheckpointRepositoryImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index 8607176b0..ba22953d1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -18,9 +18,7 @@ package za.co.absa.atum.server.api.service import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.atum.server.api.exception.ServiceError.GeneralServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository -import za.co.absa.atum.server.model.CheckpointFromDB import zio._ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { @@ -40,17 +38,12 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends } override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointV2DTO] = { - for { - checkpointFromDB <- repositoryCall( - checkpointRepository.getCheckpointV2(partitioningId, checkpointId), - "getCheckpoint" - ) - checkpointDTO <- ZIO - .fromEither(CheckpointFromDB.toCheckpointV2DTO(checkpointFromDB)) - .mapError(error => GeneralServiceError(error.getMessage)) - } yield checkpointDTO - + repositoryCall( + checkpointRepository.getCheckpointV2(partitioningId, checkpointId), + "getCheckpoint" + ) } + } object CheckpointServiceImpl { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 09c55e73e..1da0e7317 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -16,23 +16,8 @@ package za.co.absa.atum.server.model -import za.co.absa.atum.model.dto.{ - CheckpointDTO, - CheckpointV2DTO, - MeasureDTO, - MeasureResultDTO, - MeasurementDTO, - PartitioningDTO -} -import za.co.absa.atum.model.dto.{ - CheckpointDTO, - CheckpointV2DTO, - MeasureDTO, - MeasureResultDTO, - MeasurementDTO, - PartitioningDTO -} import io.circe.{DecodingFailure, Json} +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import java.time.ZonedDateTime import java.util.UUID @@ -85,36 +70,4 @@ object CheckpointFromDB { } } - def toCheckpointV2DTO( -// partitioningId: Long, - checkpointQueryResult: CheckpointFromDB - ): Either[DecodingFailure, CheckpointV2DTO] = { - val measureResultOrErr = checkpointQueryResult.measurementValue.get.as[MeasureResultDTO] - - measureResultOrErr match { - case Left(err) => Left(err) - case Right(measureResult) => - Right( - CheckpointV2DTO( - id = checkpointQueryResult.idCheckpoint.get, - name = checkpointQueryResult.checkpointName.get, - author = checkpointQueryResult.author.get, - measuredByAtumAgent = checkpointQueryResult.measuredByAtumAgent.get, -// partitioningId = partitioningId, - processStartTime = checkpointQueryResult.checkpointStartTime.get, - processEndTime = checkpointQueryResult.checkpointEndTime, - measurements = Set( - MeasurementDTO( - measure = MeasureDTO( - measureName = checkpointQueryResult.measureName.get, - measuredColumns = checkpointQueryResult.measuredColumns.get - ), - result = measureResult - ) - ) - ) - ) - } - } - } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala new file mode 100644 index 000000000..cab0ea064 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.model + +import io.circe.Json + +import java.time.ZonedDateTime +import java.util.UUID + +case class CheckpointItemFromDB( + idCheckpoint: UUID, + checkpointName: String, + author: String, + measuredByAtumAgent: Boolean, + measureName: String, + measuredColumns: Seq[String], + measurementValue: Json, // JSON representation of `MeasurementDTO` + checkpointStartTime: ZonedDateTime, + checkpointEndTime: Option[ZonedDateTime] +) From 3704e4cbb581c752b88f2b9210bcd380b1cefde2 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Fri, 9 Aug 2024 12:08:03 +0200 Subject: [PATCH 25/50] remove toCheckpointV2DTO method --- .../atum/server/model/CheckpointFromDB.scala | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 95fc049f1..7d2d16809 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -70,36 +70,4 @@ object CheckpointFromDB { } } - def toCheckpointV2DTO( -// partitioningId: Long, - checkpointQueryResult: CheckpointFromDB - ): Either[DecodingFailure, CheckpointV2DTO] = { - val measureResultOrErr = checkpointQueryResult.measurementValue.get.as[MeasureResultDTO] - - measureResultOrErr match { - case Left(err) => Left(err) - case Right(measureResult) => - Right( - CheckpointV2DTO( - id = checkpointQueryResult.idCheckpoint.get, - name = checkpointQueryResult.checkpointName.get, - author = checkpointQueryResult.author.get, - measuredByAtumAgent = checkpointQueryResult.measuredByAtumAgent.get, -// partitioningId = partitioningId, - processStartTime = checkpointQueryResult.checkpointStartTime.get, - processEndTime = checkpointQueryResult.checkpointEndTime, - measurements = Set( - MeasurementDTO( - measure = MeasureDTO( - measureName = checkpointQueryResult.measureName.get, - measuredColumns = checkpointQueryResult.measuredColumns.get - ), - result = measureResult - ) - ) - ) - ) - } - } - } From b6bbfa4217b6f7e27fa94ce2caf103317c49a2e0 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Fri, 9 Aug 2024 13:15:53 +0200 Subject: [PATCH 26/50] GetPartitioningCheckpointV2EndpointUnitTests --- .../repository/CheckpointRepositoryImpl.scala | 25 +++---- ...tioningCheckpointV2EndpointUnitTests.scala | 75 +++++++++++++++++++ .../PostCheckpointEndpointV2UnitTests.scala | 2 +- 3 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index cc8d6b31c..9b2f60860 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -21,7 +21,6 @@ import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, Writ import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError -import za.co.absa.atum.server.api.exception.ServiceError.GeneralServiceError import za.co.absa.atum.server.model.{CheckpointItemFromDB, GetCheckpointV2Args, WriteCheckpointV2Args} import zio._ import zio.interop.catz.asyncInstance @@ -61,20 +60,14 @@ class CheckpointRepositoryImpl( checkpointItems: Seq[CheckpointItemFromDB] ): Either[DecodingFailure, CheckpointV2DTO] = { val measurementsOrErr = checkpointItems.map { checkpointItem => - val measureResultOrErr = checkpointItem.measurementValue.as[MeasureResultDTO] - - measureResultOrErr match { - case Left(err) => Left(err) - case Right(measureResult) => - Right( - MeasurementDTO( - measure = MeasureDTO( - measureName = checkpointItem.measureName, - measuredColumns = checkpointItem.measuredColumns - ), - result = measureResult - ) - ) + checkpointItem.measurementValue.as[MeasureResultDTO].map { measureResult => + MeasurementDTO( + measure = MeasureDTO( + measureName = checkpointItem.measureName, + measuredColumns = checkpointItem.measuredColumns + ), + result = measureResult + ) } } @@ -83,7 +76,7 @@ class CheckpointRepositoryImpl( if (errors.nonEmpty) { Left(errors.head) } else { - val measurements = measurementsOrErr.flatMap(_.toOption).toSet + val measurements = measurementsOrErr.collect { case Right(measurement) => measurement }.toSet Right( CheckpointV2DTO( id = checkpointItems.head.idCheckpoint, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala new file mode 100644 index 000000000..117ac9d96 --- /dev/null +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala @@ -0,0 +1,75 @@ +package za.co.absa.atum.server.api.http + +import org.mockito.Mockito.{mock, when} +import sttp.client3.circe.asJson +import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.model.StatusCode +import sttp.tapir.server.stub.TapirStubInterpreter +import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} +import za.co.absa.atum.model.dto.CheckpointV2DTO +import za.co.absa.atum.server.api.TestData +import za.co.absa.atum.server.api.controller.CheckpointController +import za.co.absa.atum.server.model.NotFoundErrorResponse +import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse +import zio.test.Assertion.equalTo +import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} +import zio.{Scope, ZIO, ZLayer} + +object GetPartitioningCheckpointV2EndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { + + private val checkpointControllerMock = mock(classOf[CheckpointController]) + + when(checkpointControllerMock.getPartitioningCheckpointV2(1L, "abc")) + .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointV2DTO1, uuid))) + when(checkpointControllerMock.getPartitioningCheckpointV2(1L, "def")) + .thenReturn(ZIO.fail(NotFoundErrorResponse("not found checkpoint for a given ID"))) + + private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) + + private val getPartitioningCheckpointServerEndpointV2 = getPartitioningCheckpointEndpointV2 + .zServerLogic({ case (partitioningId: Long, checkpointId: String) => + CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointId) + }) + + override def spec: Spec[TestEnvironment with Scope, Any] = { + + val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) + .whenServerEndpoint(getPartitioningCheckpointServerEndpointV2) + .thenRunLogic() + .backend() + + suite("GetPartitioningCheckpointV2EndpointSuite")( + test("Returns an expected CheckpointV2DTO"){ + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints/abc") + .response(asJson[SingleSuccessResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo(Right(SingleSuccessResponse(checkpointV2DTO1, uuid)), StatusCode.Ok) + ) + }, + test("Returns expected 404 when checkpoint for a given ID doesn't exist"){ + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints/def") + .response(asJson[SingleSuccessResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.NotFound)) + } + ) + + }.provide( + checkpointControllerMockLayer + ) +} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala index 760a1081e..95a31ba12 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala @@ -23,7 +23,7 @@ import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} +import za.co.absa.atum.model.dto.CheckpointV2DTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{ConflictErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} From cccb0c1252cd8a0ab79fe0cf3273d3c184756377 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Fri, 9 Aug 2024 14:48:24 +0200 Subject: [PATCH 27/50] tests --- .../api/controller/CheckpointController.scala | 4 ++- .../controller/CheckpointControllerImpl.scala | 4 ++- .../absa/atum/server/api/http/Endpoints.scala | 6 ++-- .../co/absa/atum/server/api/http/Routes.scala | 6 ++-- .../api/repository/CheckpointRepository.scala | 4 ++- .../repository/CheckpointRepositoryImpl.scala | 4 ++- .../api/service/CheckpointService.scala | 4 ++- .../api/service/CheckpointServiceImpl.scala | 4 ++- .../server/model/GetCheckpointV2Args.scala | 4 ++- .../za/co/absa/atum/server/api/TestData.scala | 19 +++++++++++-- .../CheckpointControllerUnitTests.scala | 26 ++++++++++++++++- .../PartitioningControllerUnitTests.scala | 4 +-- .../CreatePartitioningEndpointUnitTests.scala | 4 +-- .../GetFlowCheckpointsEndpointUnitTests.scala | 4 +-- ...tioningCheckpointV2EndpointUnitTests.scala | 16 ++++++----- .../PostCheckpointEndpointV2UnitTests.scala | 4 +-- .../CheckpointRepositoryUnitTests.scala | 28 +++++++++++++++++-- .../service/CheckpointServiceUnitTests.scala | 17 +++++++++++ 18 files changed, 130 insertions(+), 32 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index 49e37bb71..e72ee1ae3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.IO import zio.macros.accessible +import java.util.UUID + @accessible trait CheckpointController { @@ -34,7 +36,7 @@ trait CheckpointController { def getPartitioningCheckpointV2( partitioningId: Long, - checkpointId: String + checkpointId: UUID ): IO[ErrorResponse, SingleSuccessResponse[CheckpointV2DTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index b360ec186..d26d72065 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -23,6 +23,8 @@ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ +import java.util.UUID + class CheckpointControllerImpl(checkpointService: CheckpointService) extends CheckpointController with BaseController { override def createCheckpointV1( @@ -53,7 +55,7 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che override def getPartitioningCheckpointV2( partitioningId: Long, - checkpointId: String + checkpointId: UUID ): IO[ErrorResponse, SingleSuccessResponse[CheckpointV2DTO]] = { mapToSingleSuccessResponse( serviceCall[CheckpointV2DTO, CheckpointV2DTO]( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 4b5b3b569..a58fb9312 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -27,6 +27,8 @@ import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, Singl import sttp.tapir.{PublicEndpoint, endpoint} import za.co.absa.atum.server.api.http.ApiPaths.{V1Paths, V2Paths} +import java.util.UUID + trait Endpoints extends BaseEndpoints { protected val createCheckpointEndpointV1: PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { @@ -76,9 +78,9 @@ trait Endpoints extends BaseEndpoints { } protected val getPartitioningCheckpointEndpointV2 - : PublicEndpoint[(Long, String), ErrorResponse, SingleSuccessResponse[CheckpointV2DTO], Any] = { + : PublicEndpoint[(Long, UUID), ErrorResponse, SingleSuccessResponse[CheckpointV2DTO], Any] = { apiV2.get - .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints / path[String]("checkpointId")) + .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints / path[UUID]("checkpointId")) .out(statusCode(StatusCode.Ok)) .out(jsonBody[SingleSuccessResponse[CheckpointV2DTO]]) .errorOutVariantPrepend(notFoundErrorOneOfVariant) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index f002e7eb8..0c50b9cc3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -36,6 +36,8 @@ import zio._ import zio.interop.catz._ import zio.metrics.connectors.prometheus.PrometheusPublisher +import java.util.UUID + trait Routes extends Endpoints with ServerOptions { private def createAllServerRoutes(httpMonitoringConfig: HttpMonitoringConfig): HttpRoutes[HttpEnv.F] = { @@ -61,12 +63,12 @@ trait Routes extends Endpoints with ServerOptions { PartitioningController.createOrUpdateAdditionalDataV2 ), createServerEndpoint[ - (Long, String), + (Long, UUID), ErrorResponse, SingleSuccessResponse[CheckpointV2DTO] ]( getPartitioningCheckpointEndpointV2, - { case (partitioningId: Long, checkpointId: String) => + { case (partitioningId: Long, checkpointId: UUID) => CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointId) } ), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala index fb1bc802a..875eaad2f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepository.scala @@ -22,9 +22,11 @@ import za.co.absa.atum.server.model.CheckpointFromDB import zio._ import zio.macros.accessible +import java.util.UUID + @accessible trait CheckpointRepository { def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] - def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointV2DTO] + def getCheckpointV2(partitioningId: Long, checkpointId: UUID): IO[DatabaseError, CheckpointV2DTO] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 9b2f60860..503753212 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -25,6 +25,8 @@ import za.co.absa.atum.server.model.{CheckpointItemFromDB, GetCheckpointV2Args, import zio._ import zio.interop.catz.asyncInstance +import java.util.UUID + class CheckpointRepositoryImpl( writeCheckpointFn: WriteCheckpoint, writeCheckpointV2Fn: WriteCheckpointV2, @@ -43,7 +45,7 @@ class CheckpointRepositoryImpl( ) } - override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[DatabaseError, CheckpointV2DTO] = { + override def getCheckpointV2(partitioningId: Long, checkpointId: UUID): IO[DatabaseError, CheckpointV2DTO] = { dbMultipleResultCallWithAggregatedStatus( getCheckpointV2Fn(GetCheckpointV2Args(partitioningId, checkpointId)), "getCheckpoint" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala index 531257bc9..62cf7a231 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointService.scala @@ -21,9 +21,11 @@ import za.co.absa.atum.server.api.exception.ServiceError import zio.IO import zio.macros.accessible +import java.util.UUID + @accessible trait CheckpointService { def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] - def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointV2DTO] + def getCheckpointV2(partitioningId: Long, checkpointId: UUID): IO[ServiceError, CheckpointV2DTO] def saveCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[ServiceError, Unit] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala index ba22953d1..1da3a19f8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/CheckpointServiceImpl.scala @@ -21,6 +21,8 @@ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.repository.CheckpointRepository import zio._ +import java.util.UUID + class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { override def saveCheckpoint(checkpointDTO: CheckpointDTO): IO[ServiceError, Unit] = { @@ -37,7 +39,7 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends ) } - override def getCheckpointV2(partitioningId: Long, checkpointId: String): IO[ServiceError, CheckpointV2DTO] = { + override def getCheckpointV2(partitioningId: Long, checkpointId: UUID): IO[ServiceError, CheckpointV2DTO] = { repositoryCall( checkpointRepository.getCheckpointV2(partitioningId, checkpointId), "getCheckpoint" diff --git a/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala b/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala index 9019fa495..832499c28 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala @@ -16,7 +16,9 @@ package za.co.absa.atum.server.model +import java.util.UUID + case class GetCheckpointV2Args( partitioningId: Long, - checkpointId: String + checkpointId: UUID ) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 5ce0fe048..ec94e1111 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -18,17 +18,18 @@ package za.co.absa.atum.server.api import io.circe.parser import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.model.CheckpointFromDB +import za.co.absa.atum.server.model.{CheckpointFromDB, CheckpointItemFromDB, MeasureFromDB} import java.time.ZonedDateTime import java.util.UUID import MeasureResultDTO.TypedValue +import io.circe.syntax.EncoderOps import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.server.model.MeasureFromDB trait TestData { - protected val uuid: UUID = UUID.randomUUID() + protected val uuid1: UUID = UUID.randomUUID() + protected val uuid2: UUID = UUID.randomUUID() // Partitioning DTO protected val partitioningDTO1: PartitioningDTO = Seq( @@ -272,6 +273,18 @@ trait TestData { checkpointStartTime = Some(checkpointDTO3.processStartTime) ) + protected val checkpointItemFromDB1: CheckpointItemFromDB = CheckpointItemFromDB( + idCheckpoint = checkpointV2DTO1.id, + checkpointName = checkpointV2DTO1.name, + author = checkpointV2DTO1.author, + measuredByAtumAgent = checkpointV2DTO1.measuredByAtumAgent, + measureName = checkpointV2DTO1.measurements.head.measure.measureName, + measuredColumns = checkpointV2DTO1.measurements.head.measure.measuredColumns, + measurementValue = checkpointV2DTO1.measurements.head.result.asJson, + checkpointStartTime = checkpointV2DTO1.processStartTime, + checkpointEndTime = checkpointV2DTO1.processEndTime + ) + protected def createAtumContextDTO(partitioningSubmitDTO: PartitioningSubmitDTO): AtumContextDTO = { val measures: Set[MeasureDTO] = Set(MeasureDTO("count", Seq("*"))) val additionalData: InitialAdditionalDataDTO = Map.empty diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 93f364d49..991971ab0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -24,7 +24,7 @@ import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.server.config.SslConfig -import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.test.Assertion.failsWithA import zio._ @@ -48,6 +48,13 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { when(checkpointServiceMock.saveCheckpointV2(partitioningId, checkpointV2DTO3)) .thenReturn(ZIO.fail(ConflictServiceError("boom!"))) + when(checkpointServiceMock.getCheckpointV2(partitioningId, checkpointV2DTO1.id)) + .thenReturn(ZIO.succeed(checkpointV2DTO1)) + when(checkpointServiceMock.getCheckpointV2(partitioningId, checkpointV2DTO2.id)) + .thenReturn(ZIO.fail(NotFoundServiceError("not found"))) + when(checkpointServiceMock.getCheckpointV2(partitioningId, checkpointV2DTO3.id)) + .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) + private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -97,6 +104,23 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { failsWithA[ConflictErrorResponse] ) } + ), + suite("GetPartitioningCheckpointV2Suite")( + test("Returns expected CheckpointDTO") { + for { + result <- CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointV2DTO1.id) + } yield assertTrue(result.data == checkpointV2DTO1) + }, + test("Returns expected NotFoundErrorResponse"){ + assertZIO(CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointV2DTO2.id).exit)( + failsWithA[NotFoundErrorResponse] + ) + }, + test("Returns expected InternalServerErrorResponse"){ + assertZIO(CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointV2DTO3.id).exit)( + failsWithA[InternalServerErrorResponse] + ) + } ) ).provide( CheckpointControllerImpl.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 30ac2ab0b..5bfa33121 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -73,8 +73,8 @@ object PartitioningControllerUnitTests extends ZIOSpecDefault with TestData { test("Returns expected AdditionalDataSubmitDTO") { for { result <- PartitioningController.createOrUpdateAdditionalDataV2(additionalDataSubmitDTO1) - expected = SingleSuccessResponse(additionalDataSubmitDTO1, uuid) - actual = result.copy(requestId = uuid) + expected = SingleSuccessResponse(additionalDataSubmitDTO1, uuid1) + actual = result.copy(requestId = uuid1) } yield assert(actual)(equalTo(expected)) }, test("Returns expected InternalServerErrorResponse") { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index cb5cc7c96..d56e0206d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -37,7 +37,7 @@ object CreatePartitioningEndpointUnitTests extends ZIOSpecDefault with Endpoints private val createPartitioningEndpointMock = mock(classOf[PartitioningController]) when(createPartitioningEndpointMock.createPartitioningIfNotExistsV2(partitioningSubmitDTO1)) - .thenReturn(ZIO.succeed(SingleSuccessResponse(createAtumContextDTO(partitioningSubmitDTO1), uuid))) + .thenReturn(ZIO.succeed(SingleSuccessResponse(createAtumContextDTO(partitioningSubmitDTO1), uuid1))) when(createPartitioningEndpointMock.createPartitioningIfNotExistsV2(partitioningSubmitDTO2)) .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) when(createPartitioningEndpointMock.createPartitioningIfNotExistsV2(partitioningSubmitDTO3)) @@ -68,7 +68,7 @@ object CreatePartitioningEndpointUnitTests extends ZIOSpecDefault with Endpoints val statusCode = response.map(_.code) assertZIO(body <&> statusCode)( - equalTo(Right(SingleSuccessResponse(createAtumContextDTO(partitioningSubmitDTO1), uuid)), StatusCode.Ok) + equalTo(Right(SingleSuccessResponse(createAtumContextDTO(partitioningSubmitDTO1), uuid1)), StatusCode.Ok) ) }, test("Returns expected BadRequest") { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala index 7e8514e7c..bb11a896e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -37,7 +37,7 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with Endpoints private val flowControllerMock = mock(classOf[FlowController]) when(flowControllerMock.getFlowCheckpointsV2(checkpointQueryDTO1)) - .thenReturn(ZIO.succeed(MultiSuccessResponse(Seq(checkpointDTO1, checkpointDTO2), uuid))) + .thenReturn(ZIO.succeed(MultiSuccessResponse(Seq(checkpointDTO1, checkpointDTO2), uuid1))) when(flowControllerMock.getFlowCheckpointsV2(checkpointQueryDTO2)) .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) when(flowControllerMock.getFlowCheckpointsV2(checkpointQueryDTO3)) @@ -67,7 +67,7 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with Endpoints val body = response.map(_.body) val statusCode = response.map(_.code) - val expectedResult = MultiSuccessResponse(Seq(checkpointDTO1, checkpointDTO2), uuid) + val expectedResult = MultiSuccessResponse(Seq(checkpointDTO1, checkpointDTO2), uuid1) assertZIO(body <&> statusCode)(equalTo(Right(expectedResult), StatusCode.Ok)) }, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala index 117ac9d96..a1c46ccf8 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetPartitioningCheckpointV2EndpointUnitTests.scala @@ -16,19 +16,21 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.util.UUID + object GetPartitioningCheckpointV2EndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { private val checkpointControllerMock = mock(classOf[CheckpointController]) - when(checkpointControllerMock.getPartitioningCheckpointV2(1L, "abc")) - .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointV2DTO1, uuid))) - when(checkpointControllerMock.getPartitioningCheckpointV2(1L, "def")) + when(checkpointControllerMock.getPartitioningCheckpointV2(1L, uuid1)) + .thenReturn(ZIO.succeed(SingleSuccessResponse(checkpointV2DTO1, uuid1))) + when(checkpointControllerMock.getPartitioningCheckpointV2(1L, uuid2)) .thenReturn(ZIO.fail(NotFoundErrorResponse("not found checkpoint for a given ID"))) private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) private val getPartitioningCheckpointServerEndpointV2 = getPartitioningCheckpointEndpointV2 - .zServerLogic({ case (partitioningId: Long, checkpointId: String) => + .zServerLogic({ case (partitioningId: Long, checkpointId: UUID) => CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointId) }) @@ -42,7 +44,7 @@ object GetPartitioningCheckpointV2EndpointUnitTests extends ZIOSpecDefault with suite("GetPartitioningCheckpointV2EndpointSuite")( test("Returns an expected CheckpointV2DTO"){ val request = basicRequest - .get(uri"https://test.com/api/v2/partitionings/1/checkpoints/abc") + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints/$uuid1") .response(asJson[SingleSuccessResponse[CheckpointV2DTO]]) val response = request @@ -52,12 +54,12 @@ object GetPartitioningCheckpointV2EndpointUnitTests extends ZIOSpecDefault with val statusCode = response.map(_.code) assertZIO(body <&> statusCode)( - equalTo(Right(SingleSuccessResponse(checkpointV2DTO1, uuid)), StatusCode.Ok) + equalTo(Right(SingleSuccessResponse(checkpointV2DTO1, uuid1)), StatusCode.Ok) ) }, test("Returns expected 404 when checkpoint for a given ID doesn't exist"){ val request = basicRequest - .get(uri"https://test.com/api/v2/partitionings/1/checkpoints/def") + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints/$uuid2") .response(asJson[SingleSuccessResponse[CheckpointV2DTO]]) val response = request diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala index 95a31ba12..9f5c99310 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/PostCheckpointEndpointV2UnitTests.scala @@ -37,7 +37,7 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w private val checkpointControllerMock = mock(classOf[CheckpointController]) when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO1)) - .thenReturn(ZIO.succeed((SingleSuccessResponse(checkpointV2DTO1, uuid), "some location"))) + .thenReturn(ZIO.succeed((SingleSuccessResponse(checkpointV2DTO1, uuid1), "some location"))) when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO2)) .thenReturn(ZIO.fail(GeneralErrorResponse("error"))) when(checkpointControllerMock.postCheckpointV2(1L, checkpointV2DTO3)) @@ -73,7 +73,7 @@ object PostCheckpointEndpointV2UnitTests extends ZIOSpecDefault with Endpoints w val header = response.map(_.header("Location")) assertZIO(body <&> statusCode <&> header)( - equalTo(Right(SingleSuccessResponse(checkpointV2DTO1, uuid)), StatusCode.Created, Some("some location")) + equalTo(Right(SingleSuccessResponse(checkpointV2DTO1, uuid1)), StatusCode.Created, Some("some location")) ) }, test("Returns expected BadRequest") { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index c7aab23c6..71fcfca8a 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -21,8 +21,8 @@ import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, Writ import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.atum.server.model.WriteCheckpointV2Args -import za.co.absa.db.fadb.exceptions.DataConflictException +import za.co.absa.atum.server.model.{GetCheckpointV2Args, WriteCheckpointV2Args} +import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance @@ -50,6 +50,13 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) + when(getCheckpointMockV2.apply(GetCheckpointV2Args(partitioningId, checkpointV2DTO1.id))) + .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "OK"), Some(checkpointItemFromDB1))))) + when(getCheckpointMockV2.apply(GetCheckpointV2Args(partitioningId, checkpointV2DTO2.id))) + .thenReturn(ZIO.left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + when(getCheckpointMockV2.apply(GetCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) + .thenReturn(ZIO.fail(new Exception("boom!"))) + private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointMockV2) private val getCheckpointV2MockLayer = ZLayer.succeed(getCheckpointMockV2) @@ -90,6 +97,23 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { failsWithA[GeneralDatabaseError] ) } + ), + suite("GetCheckpointV2Suite")( + test("Returns an expected Right with CheckpointV2DTO") { + for { + result <- CheckpointRepository.getCheckpointV2(partitioningId, checkpointV2DTO1.id) + } yield assertTrue(result == checkpointV2DTO1) + }, + test("Fails with an expected NotFoundDatabaseError") { + assertZIO(CheckpointRepository.getCheckpointV2(partitioningId, checkpointV2DTO2.id).exit)( + failsWithA[NotFoundDatabaseError] + ) + }, + test("Returns an expected DatabaseError") { + assertZIO(CheckpointRepository.getCheckpointV2(partitioningId, checkpointV2DTO3.id).exit)( + failsWithA[GeneralDatabaseError] + ) + } ) ).provide( CheckpointRepositoryImpl.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index 79d9e7127..b80a01645 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -42,6 +42,11 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { when(checkpointRepositoryMock.writeCheckpointV2(partitioningId, checkpointV2DTO3)) .thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) + when(checkpointRepositoryMock.getCheckpointV2(partitioningId, checkpointV2DTO1.id)) + .thenReturn(ZIO.succeed(checkpointV2DTO1)) + when(checkpointRepositoryMock.getCheckpointV2(partitioningId, checkpointV2DTO2.id)) + .thenReturn(ZIO.fail(NotFoundDatabaseError("not found"))) + private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -80,6 +85,18 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { failsWithA[GeneralServiceError] ) } + ), + suite("GetCheckpointV2Suite")( + test("Returns an expected CheckpointV2DTO") { + for { + result <- CheckpointService.getCheckpointV2(partitioningId, checkpointV2DTO1.id) + } yield assertTrue(result == checkpointV2DTO1) + }, + test("Fails with an expected NotFoundServiceError") { + assertZIO(CheckpointService.getCheckpointV2(partitioningId, checkpointV2DTO2.id).exit)( + failsWithA[NotFoundServiceError] + ) + } ) ).provide( CheckpointServiceImpl.layer, From 44b9f36a6e882a09461290e628d698746767144b Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Aug 2024 15:27:08 +0200 Subject: [PATCH 28/50] tests --- ...V1.9.3__get_partitioning_checkpoint_v2.sql | 29 ++-- ...itioningCheckpointV2IntegrationTests.scala | 163 ++++++++++++++++++ .../scala/za/co/absa/atum/server/Main.scala | 2 +- .../api/controller/BaseController.scala | 33 +--- .../controller/CheckpointControllerImpl.scala | 2 +- ...cala => GetPartitioningCheckpointV2.scala} | 8 +- .../repository/CheckpointRepositoryImpl.scala | 8 +- .../CheckpointControllerUnitTests.scala | 22 +-- ...itioningCheckpointV2IntegrationTests.scala | 47 +++++ .../CheckpointRepositoryUnitTests.scala | 4 +- 10 files changed, 253 insertions(+), 65 deletions(-) create mode 100644 database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala rename server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/{GetCheckpointV2.scala => GetPartitioningCheckpointV2.scala} (87%) create mode 100644 server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala diff --git a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql index d25efe128..388140925 100644 --- a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql @@ -43,25 +43,28 @@ $$ -- Returns: -- status - Status code -- status_text - Status message --- o_id_checkpoint - ID of the checkpoint --- o_checkpoint_name - Name of the checkpoint --- o_author - Author of the checkpoint --- o_measuredByAtumAgent - Flag indicating whether the checkpoint was measured by ATUM agent --- o_measure_name - Name of the measure --- o_measure_columns - Columns of the measure --- o_measurement_value - Value of the measurement --- o_checkpoint_start_time - Time of the checkpoint --- o_checkpoint_end_time - End time of the checkpoint computation +-- id_checkpoint - ID of the checkpoint +-- checkpoint_name - Name of the checkpoint +-- author - Author of the checkpoint +-- measuredByAtumAgent - Flag indicating whether the checkpoint was measured by ATUM agent +-- measure_name - Name of the measure +-- measure_columns - Columns of the measure +-- measurement_value - Value of the measurement +-- checkpoint_start_time - Time of the checkpoint +-- checkpoint_end_time - End time of the checkpoint computation -- -- Status codes: -- 11 - OK --- 41 - Partitioning or Checkpoint not found +-- 41 - Partitioning not found +-- 42 - Checkpoint not found -- ------------------------------------------------------------------------------- BEGIN - IF NOT EXISTS (SELECT 1 FROM runs.partitionings WHERE id_partitioning = i_partitioning_id) THEN + PERFORM 1 FROM runs.partitionings WHERE id_partitioning = i_partitioning_id; + IF NOT FOUND THEN status := 41; status_text := 'Partitioning not found'; + RETURN NEXT; RETURN; END IF; @@ -91,8 +94,10 @@ BEGIN ; IF NOT FOUND THEN - status := 41; + status := 42; status_text := 'Checkpoint not found'; + RETURN NEXT; + RETURN; END IF; END; $$ diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala new file mode 100644 index 000000000..770d0c4f9 --- /dev/null +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala @@ -0,0 +1,163 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.database.runs + +import za.co.absa.balta.DBTestSuite +import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.setter.CustomDBType + +import java.time.OffsetDateTime +import java.util.UUID + +class GetPartitioningCheckpointV2IntegrationTests extends DBTestSuite{ + + private val fncGetPartitioningCheckpointV2 = "runs.get_partitioning_checkpoint_v2" + + case class MeasuredDetails( + measureName: String, + measureColumns: Seq[String], + measurementValue: JsonBString + ) + + private val partitioning1 = JsonBString( + """ + |{ + | "version": 1, + | "keys": ["keyX", "keyY", "keyZ"], + | "keysToValues": { + | "keyX": "value1", + | "keyZ": "value3", + | "keyY": "value2" + | } + |} + |""".stripMargin + ) + + private val partitioning2 = JsonBString( + """ + |{ + | "version": 1, + | "keys": ["key1", "key3", "key2", "key4"], + | "keysToValues": { + | "key1": "valueX", + | "key2": "valueY", + | "key3": "valueZ", + | "key4": "valueA" + | } + |} + |""".stripMargin + ) + + private val measurement1 = JsonBString("""1""".stripMargin) + + private val measured_columns = CustomDBType("""{"col2"}""", "TEXT[]") + + test("Get partitioning checkpoints returns checkpoints for partitioning with checkpoints") { + + val uuid = UUID.randomUUID + val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") + val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") + + val id_measure_definition: Long = 1 + + table("runs.partitionings").insert( + add("partitioning", partitioning1) + .add("created_by", "Daniel") + ) + + val fkPartitioning1: Long = table("runs.partitionings") + .fieldValue("partitioning", partitioning1, "id_partitioning").get.get + + table("runs.checkpoints").insert( + add("id_checkpoint", uuid) + .add("fk_partitioning", fkPartitioning1) + .add("checkpoint_name", "checkpoint_1") + .add("process_start_time", startTime) + .add("process_end_time", endTime) + .add("measured_by_atum_agent", true) + .add("created_by", "Daniel") + ) + + table("runs.measure_definitions").insert( + add("id_measure_definition", id_measure_definition) + .add("fk_partitioning", fkPartitioning1) + .add("created_by", "Daniel") + .add("measure_name", "measure_1") + .add("measured_columns", measured_columns) + ) + + table("runs.measurements").insert( + add("fk_checkpoint", uuid) + .add("fk_measure_definition", id_measure_definition) + .add("measurement_value", measurement1) + ) + + function(fncGetPartitioningCheckpointV2) + .setParam("i_partitioning_id", fkPartitioning1) + .setParam("i_checkpoint_id", uuid) + .execute { queryResult => + assert(queryResult.hasNext) + val results = queryResult.next() + assert(results.getInt("status").contains(11)) + assert(results.getString("status_text").contains("Ok")) + assert(results.getString("checkpoint_name").contains("checkpoint_1")) + assert(results.getUUID("id_checkpoint").contains(uuid)) + assert(results.getOffsetDateTime("checkpoint_start_time").contains(startTime)) + assert(results.getOffsetDateTime("checkpoint_end_time").contains(endTime)) + assert(results.getJsonB("measurement_value").contains(measurement1)) + assert(results.getString("measure_name").contains("measure_1")) + assert(!queryResult.hasNext) + } + } + + test("Get partitioning checkpoints returns no checkpoints for partitioning without checkpoints") { + + table("runs.partitionings").insert( + add("partitioning", partitioning2) + .add("created_by", "Daniel") + ) + + val fkPartitioning2: Long = table("runs.partitionings") + .fieldValue("partitioning", partitioning2, "id_partitioning").get.get + + function(fncGetPartitioningCheckpointV2) + .setParam("i_partitioning_id", fkPartitioning2) + .setParam("i_checkpoint_id", UUID.randomUUID()) + .execute { queryResult => + assert(queryResult.hasNext) + val results = queryResult.next() + assert(results.getInt("status").contains(42)) + assert(results.getString("status_text").contains("Checkpoint not found")) + } + + } + + test("Get partitioning checkpoints no checkpoints non-existent partitionings") { + + function(fncGetPartitioningCheckpointV2) + .setParam("i_partitioning_id", 0L) + .setParam("i_checkpoint_id", UUID.randomUUID()) + .execute { queryResult => + assert(queryResult.hasNext) + val results = queryResult.next() + assert(results.getInt("status").contains(41)) + assert(results.getString("status_text").contains("Partitioning not found")) + } + + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 1d0c61c7f..32ddbf835 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -57,7 +57,7 @@ object Main extends ZIOAppDefault with Server { GetPartitioningCheckpoints.layer, WriteCheckpoint.layer, WriteCheckpointV2.layer, - GetCheckpointV2.layer, + GetPartitioningCheckpointV2.layer, GetFlowCheckpoints.layer, PostgresDatabaseProvider.layer, TransactorProvider.layer, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 4a791e492..aa32bcf6c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -18,13 +18,8 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ -import za.co.absa.atum.server.config.SslConfig -import za.co.absa.atum.server.model.{ - ConflictErrorResponse, - ErrorResponse, - InternalServerErrorResponse, - NotFoundErrorResponse -} +import za.co.absa.atum.server.api.http.ApiPaths +import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ @@ -59,24 +54,10 @@ trait BaseController { effect.map(MultiSuccessResponse(_)) } - protected def createResourceUri(parts: Seq[String]): IO[ErrorResponse, String] = { - for { - hostname <- System - .env("HOSTNAME") - // fails the request if the hostname is not found, we need to make sure that the hostname is always available - .orElseFail(InternalServerErrorResponse("Failed to get hostname")) - .flatMap { - case Some(value) => ZIO.succeed(value) - case None => ZIO.fail(InternalServerErrorResponse("Failed to get hostname")) - } - sslConfig <- ZIO - .config[SslConfig](SslConfig.config) - .orElseFail(InternalServerErrorResponse("Failed to get SSL config")) - } yield { - val protocol = if (sslConfig.enabled) "https" else "http" - val port = if (sslConfig.enabled) 8443 else 8080 - val path = parts.mkString("/") - s"$protocol://$hostname:$port/$path" - } + // Root-anchored URL path + // https://stackoverflow.com/questions/2005079/absolute-vs-relative-urls/78439286#78439286 + protected def createV2RootAnchoredResourcePath(parts: Seq[String]): IO[ErrorResponse, String] = { + ZIO.succeed(s"/${ApiPaths.Api}/${ApiPaths.V2}/${parts.mkString("/")}") } + } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index d26d72065..fce02274e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -47,7 +47,7 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che _ => checkpointV2DTO ) ) - uri <- createResourceUri( + uri <- createV2RootAnchoredResourcePath( Seq(V2Paths.Partitionings, partitioningId.toString, V2Paths.Checkpoints, checkpointV2DTO.id.toString) ) } yield (response, uri) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala similarity index 87% rename from server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala rename to server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala index 263ac908e..9cec562a2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala @@ -30,7 +30,7 @@ import doobie.postgres.implicits._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator -class GetCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) +class GetPartitioningCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetCheckpointV2Args, Option[CheckpointItemFromDB], Task](input => Seq( fr"${input.partitioningId}", @@ -52,10 +52,10 @@ class GetCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ) } -object GetCheckpointV2 { - val layer: URLayer[PostgresDatabaseProvider, GetCheckpointV2] = ZLayer { +object GetPartitioningCheckpointV2 { + val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpointV2] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] - } yield new GetCheckpointV2()(Runs, dbProvider.dbEngine) + } yield new GetPartitioningCheckpointV2()(Runs, dbProvider.dbEngine) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 503753212..50eb22881 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.repository import io.circe.DecodingFailure -import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{GetPartitioningCheckpointV2, WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError @@ -30,7 +30,7 @@ import java.util.UUID class CheckpointRepositoryImpl( writeCheckpointFn: WriteCheckpoint, writeCheckpointV2Fn: WriteCheckpointV2, - getCheckpointV2Fn: GetCheckpointV2 + getCheckpointV2Fn: GetPartitioningCheckpointV2 ) extends CheckpointRepository with BaseRepository { @@ -96,11 +96,11 @@ class CheckpointRepositoryImpl( } object CheckpointRepositoryImpl { - val layer: URLayer[WriteCheckpoint with WriteCheckpointV2 with GetCheckpointV2, CheckpointRepository] = ZLayer { + val layer: URLayer[WriteCheckpoint with WriteCheckpointV2 with GetPartitioningCheckpointV2, CheckpointRepository] = ZLayer { for { writeCheckpoint <- ZIO.service[WriteCheckpoint] writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] - getCheckpointV2 <- ZIO.service[GetCheckpointV2] + getCheckpointV2 <- ZIO.service[GetPartitioningCheckpointV2] } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2, getCheckpointV2) } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 991971ab0..35b63c6bf 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -17,17 +17,15 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} +import za.co.absa.atum.model.dto.CheckpointV2DTO import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError._ -import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.config.SslConfig -import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse -import zio.test.Assertion.failsWithA +import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} import zio._ +import zio.test.Assertion.failsWithA import zio.test._ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { @@ -79,19 +77,13 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { ), suite("PostCheckpointV2Suite")( test("Returns expected CheckpointDTO") { - val host = "testHost" for { - _ <- TestSystem.putEnv("HOSTNAME", host) - sslConfig <- ZIO.config[SslConfig](SslConfig.config) result <- CheckpointController.postCheckpointV2(partitioningId, checkpointV2DTO1) - protocol = if (sslConfig.enabled) "https" else "http" - port = if (sslConfig.enabled) 8443 else 8080 - path = s"${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}/${checkpointV2DTO1.id}" - expectedUri = s"$protocol://$host:$port/$path" + path = s"/api/v2/partitionings/$partitioningId/checkpoints/${checkpointV2DTO1.id}" } yield assertTrue( result._1.isInstanceOf[SingleSuccessResponse[CheckpointV2DTO]] && result._1.data == checkpointV2DTO1 - && result._2 == expectedUri + && result._2 == path ) }, test("Returns expected ConflictServiceError") { @@ -111,12 +103,12 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { result <- CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointV2DTO1.id) } yield assertTrue(result.data == checkpointV2DTO1) }, - test("Returns expected NotFoundErrorResponse"){ + test("Returns expected NotFoundErrorResponse") { assertZIO(CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointV2DTO2.id).exit)( failsWithA[NotFoundErrorResponse] ) }, - test("Returns expected InternalServerErrorResponse"){ + test("Returns expected InternalServerErrorResponse") { assertZIO(CheckpointController.getPartitioningCheckpointV2(partitioningId, checkpointV2DTO3.id).exit)( failsWithA[InternalServerErrorResponse] ) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala new file mode 100644 index 000000000..654dabb6b --- /dev/null +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.database.runs.functions + +import za.co.absa.atum.server.ConfigProviderTest +import za.co.absa.atum.server.api.TestTransactorProvider +import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.atum.server.model.GetCheckpointV2Args +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus +import zio.interop.catz.asyncInstance +import zio.{Scope, ZIO} +import zio.test._ + +import java.util.UUID + +object GetPartitioningCheckpointV2IntegrationTests extends ConfigProviderTest { + + override def spec: Spec[Unit with TestEnvironment with Scope, Any] = { + suite("GetPartitioningCheckpointV2IntegrationTests")( + test("Returns expected sequence of Checkpoints with existing partitioning") { + for { + getPartitioningCheckpointV2Fn <- ZIO.service[GetPartitioningCheckpointV2] + result <- getPartitioningCheckpointV2Fn(GetCheckpointV2Args(1L, UUID.randomUUID())) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + } + ).provide( + GetPartitioningCheckpointV2.layer, + PostgresDatabaseProvider.layer, + TestTransactorProvider.layerWithRollback + ) + } +} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 71fcfca8a..ba86c5dbf 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{GetCheckpointV2, WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{GetPartitioningCheckpointV2, WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.DatabaseError._ @@ -34,7 +34,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) private val writeCheckpointMockV2: WriteCheckpointV2 = mock(classOf[WriteCheckpointV2]) - private val getCheckpointMockV2: GetCheckpointV2 = mock(classOf[GetCheckpointV2]) + private val getCheckpointMockV2: GetPartitioningCheckpointV2 = mock(classOf[GetPartitioningCheckpointV2]) when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(writeCheckpointMock.apply(checkpointDTO2)) From 68ee4b2833d5bd89fa28a7f0c1f32f880200376e Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Aug 2024 15:27:45 +0200 Subject: [PATCH 29/50] tests --- .../co/absa/atum/server/api/controller/BaseController.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index aa32bcf6c..a2cce3236 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -19,8 +19,8 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ import za.co.absa.atum.server.api.http.ApiPaths -import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} +import za.co.absa.atum.server.model._ +import za.co.absa.atum.server.model.SuccessResponse._ import zio._ trait BaseController { From f012746e2e115187a8dc471bd3f29c96b34d7442 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 09:57:57 +0200 Subject: [PATCH 30/50] comments addressed --- .../postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql | 2 +- .../src/main/scala/za/co/absa/atum/server/api/http/Routes.scala | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql index 388140925..b11f4371b 100644 --- a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql @@ -106,4 +106,4 @@ $$ ALTER FUNCTION runs.get_partitioning_checkpoint_v2(BIGINT, UUID) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoint_v2(BIGINT, UUID) TO atum_owner; \ No newline at end of file +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoint_v2(BIGINT, UUID) TO atum_owner; diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 0c50b9cc3..3b20484a2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -90,6 +90,7 @@ trait Routes extends Endpoints with ServerOptions { createPartitioningEndpointV2, createOrUpdateAdditionalDataEndpointV2, getPartitioningCheckpointsEndpointV2, + getPartitioningCheckpointEndpointV2, getFlowCheckpointsEndpointV2 ) ZHttp4sServerInterpreter[HttpEnv.Env](http4sServerOptions(None)) From c361d18014652baf8c890a951eea52bf252c933f Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 14:17:31 +0200 Subject: [PATCH 31/50] formatting --- .../postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql index b11f4371b..4adcd809b 100644 --- a/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.3__get_partitioning_checkpoint_v2.sql @@ -90,8 +90,7 @@ BEGIN WHERE C.fk_partitioning = i_partitioning_id AND - C.id_checkpoint = i_checkpoint_id - ; + C.id_checkpoint = i_checkpoint_id; IF NOT FOUND THEN status := 42; @@ -102,7 +101,7 @@ BEGIN END; $$ - LANGUAGE plpgsql VOLATILE SECURITY DEFINER; +LANGUAGE plpgsql VOLATILE SECURITY DEFINER; ALTER FUNCTION runs.get_partitioning_checkpoint_v2(BIGINT, UUID) OWNER TO atum_owner; From 451a56aff3687cea5764c501ddf4ed6212d13b93 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 14:18:12 +0200 Subject: [PATCH 32/50] formatting --- ...titioningCheckpointV2IntegrationTests.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala index 770d0c4f9..123dff16d 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointV2IntegrationTests.scala @@ -23,15 +23,15 @@ import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime import java.util.UUID -class GetPartitioningCheckpointV2IntegrationTests extends DBTestSuite{ +class GetPartitioningCheckpointV2IntegrationTests extends DBTestSuite { private val fncGetPartitioningCheckpointV2 = "runs.get_partitioning_checkpoint_v2" case class MeasuredDetails( - measureName: String, - measureColumns: Seq[String], - measurementValue: JsonBString - ) + measureName: String, + measureColumns: Seq[String], + measurementValue: JsonBString + ) private val partitioning1 = JsonBString( """ @@ -80,7 +80,9 @@ class GetPartitioningCheckpointV2IntegrationTests extends DBTestSuite{ ) val fkPartitioning1: Long = table("runs.partitionings") - .fieldValue("partitioning", partitioning1, "id_partitioning").get.get + .fieldValue("partitioning", partitioning1, "id_partitioning") + .get + .get table("runs.checkpoints").insert( add("id_checkpoint", uuid) @@ -132,7 +134,9 @@ class GetPartitioningCheckpointV2IntegrationTests extends DBTestSuite{ ) val fkPartitioning2: Long = table("runs.partitionings") - .fieldValue("partitioning", partitioning2, "id_partitioning").get.get + .fieldValue("partitioning", partitioning2, "id_partitioning") + .get + .get function(fncGetPartitioningCheckpointV2) .setParam("i_partitioning_id", fkPartitioning2) From 4b40110343ff0acb83af2eb45afda0ddc33fd3a9 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 16:38:41 +0200 Subject: [PATCH 33/50] refactoring as per discussion --- .../GetPartitioningCheckpointV2.scala | 9 ++- .../repository/CheckpointRepositoryImpl.scala | 66 +++++-------------- .../server/model/CheckpointItemFromDB.scala | 42 +++++++++++- .../server/model/GetCheckpointV2Args.scala | 24 ------- ...itioningCheckpointV2IntegrationTests.scala | 4 +- .../CheckpointRepositoryUnitTests.scala | 15 +++-- 6 files changed, 78 insertions(+), 82 deletions(-) delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala index 9cec562a2..908e62535 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs -import za.co.absa.atum.server.model.{CheckpointItemFromDB, GetCheckpointV2Args} +import za.co.absa.atum.server.model.CheckpointItemFromDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWithAggStatus @@ -27,11 +27,14 @@ import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ +import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator +import java.util.UUID + class GetPartitioningCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[GetCheckpointV2Args, Option[CheckpointItemFromDB], Task](input => + extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointV2Args, Option[CheckpointItemFromDB], Task](input => Seq( fr"${input.partitioningId}", fr"${input.checkpointId}" @@ -53,6 +56,8 @@ class GetPartitioningCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEng } object GetPartitioningCheckpointV2 { + case class GetPartitioningCheckpointV2Args(partitioningId: Long, checkpointId: UUID) + val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpointV2] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 50eb22881..1b7bf27bd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -16,12 +16,16 @@ package za.co.absa.atum.server.api.repository -import io.circe.DecodingFailure -import za.co.absa.atum.server.api.database.runs.functions.{GetPartitioningCheckpointV2, WriteCheckpoint, WriteCheckpointV2} -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO} +import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} +import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.{ + GetPartitioningCheckpointV2, + WriteCheckpoint, + WriteCheckpointV2 +} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError -import za.co.absa.atum.server.model.{CheckpointItemFromDB, GetCheckpointV2Args, WriteCheckpointV2Args} +import za.co.absa.atum.server.model.{CheckpointItemFromDB, WriteCheckpointV2Args} import zio._ import zio.interop.catz.asyncInstance @@ -47,60 +51,26 @@ class CheckpointRepositoryImpl( override def getCheckpointV2(partitioningId: Long, checkpointId: UUID): IO[DatabaseError, CheckpointV2DTO] = { dbMultipleResultCallWithAggregatedStatus( - getCheckpointV2Fn(GetCheckpointV2Args(partitioningId, checkpointId)), + getCheckpointV2Fn(GetPartitioningCheckpointV2Args(partitioningId, checkpointId)), "getCheckpoint" ) .map(_.flatten) .flatMap { checkpointItems => ZIO - .fromEither(checkpointItemsToCheckpointV2DTO(checkpointItems)) + .fromEither(CheckpointItemFromDB.fromItemsToCheckpointV2DTO(checkpointItems)) .mapError(error => GeneralDatabaseError(error.getMessage)) } } - private def checkpointItemsToCheckpointV2DTO( - checkpointItems: Seq[CheckpointItemFromDB] - ): Either[DecodingFailure, CheckpointV2DTO] = { - val measurementsOrErr = checkpointItems.map { checkpointItem => - checkpointItem.measurementValue.as[MeasureResultDTO].map { measureResult => - MeasurementDTO( - measure = MeasureDTO( - measureName = checkpointItem.measureName, - measuredColumns = checkpointItem.measuredColumns - ), - result = measureResult - ) - } - } - - val errors = measurementsOrErr.collect { case Left(err) => err } - - if (errors.nonEmpty) { - Left(errors.head) - } else { - val measurements = measurementsOrErr.collect { case Right(measurement) => measurement }.toSet - Right( - CheckpointV2DTO( - id = checkpointItems.head.idCheckpoint, - name = checkpointItems.head.checkpointName, - author = checkpointItems.head.author, - measuredByAtumAgent = checkpointItems.head.measuredByAtumAgent, - processStartTime = checkpointItems.head.checkpointStartTime, - processEndTime = checkpointItems.head.checkpointEndTime, - measurements = measurements - ) - ) - } - } - } object CheckpointRepositoryImpl { - val layer: URLayer[WriteCheckpoint with WriteCheckpointV2 with GetPartitioningCheckpointV2, CheckpointRepository] = ZLayer { - for { - writeCheckpoint <- ZIO.service[WriteCheckpoint] - writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] - getCheckpointV2 <- ZIO.service[GetPartitioningCheckpointV2] - } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2, getCheckpointV2) - } + val layer: URLayer[WriteCheckpoint with WriteCheckpointV2 with GetPartitioningCheckpointV2, CheckpointRepository] = + ZLayer { + for { + writeCheckpoint <- ZIO.service[WriteCheckpoint] + writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] + getCheckpointV2 <- ZIO.service[GetPartitioningCheckpointV2] + } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2, getCheckpointV2) + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala index cab0ea064..035e55fc9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointItemFromDB.scala @@ -16,7 +16,8 @@ package za.co.absa.atum.server.model -import io.circe.Json +import io.circe.{DecodingFailure, Json} +import za.co.absa.atum.model.dto.{CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO} import java.time.ZonedDateTime import java.util.UUID @@ -32,3 +33,42 @@ case class CheckpointItemFromDB( checkpointStartTime: ZonedDateTime, checkpointEndTime: Option[ZonedDateTime] ) + +object CheckpointItemFromDB { + + def fromItemsToCheckpointV2DTO( + checkpointItems: Seq[CheckpointItemFromDB] + ): Either[DecodingFailure, CheckpointV2DTO] = { + val measurementsOrErr = checkpointItems.map { checkpointItem => + checkpointItem.measurementValue.as[MeasureResultDTO].map { measureResult => + MeasurementDTO( + measure = MeasureDTO( + measureName = checkpointItem.measureName, + measuredColumns = checkpointItem.measuredColumns + ), + result = measureResult + ) + } + } + + val errors = measurementsOrErr.collect { case Left(err) => err } + + if (errors.nonEmpty) { + Left(errors.head) + } else { + val measurements = measurementsOrErr.collect { case Right(measurement) => measurement }.toSet + Right( + CheckpointV2DTO( + id = checkpointItems.head.idCheckpoint, + name = checkpointItems.head.checkpointName, + author = checkpointItems.head.author, + measuredByAtumAgent = checkpointItems.head.measuredByAtumAgent, + processStartTime = checkpointItems.head.checkpointStartTime, + processEndTime = checkpointItems.head.checkpointEndTime, + measurements = measurements + ) + ) + } + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala b/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala deleted file mode 100644 index 832499c28..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/GetCheckpointV2Args.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import java.util.UUID - -case class GetCheckpointV2Args( - partitioningId: Long, - checkpointId: UUID -) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala index 654dabb6b..4b9b42f6d 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2IntegrationTests.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.atum.server.model.GetCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio.interop.catz.asyncInstance @@ -35,7 +35,7 @@ object GetPartitioningCheckpointV2IntegrationTests extends ConfigProviderTest { test("Returns expected sequence of Checkpoints with existing partitioning") { for { getPartitioningCheckpointV2Fn <- ZIO.service[GetPartitioningCheckpointV2] - result <- getPartitioningCheckpointV2Fn(GetCheckpointV2Args(1L, UUID.randomUUID())) + result <- getPartitioningCheckpointV2Fn(GetPartitioningCheckpointV2Args(1L, UUID.randomUUID())) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index ba86c5dbf..86087b026 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,11 +17,16 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{GetPartitioningCheckpointV2, WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{ + GetPartitioningCheckpointV2, + WriteCheckpoint, + WriteCheckpointV2 +} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData +import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.atum.server.model.{GetCheckpointV2Args, WriteCheckpointV2Args} +import za.co.absa.atum.server.model.WriteCheckpointV2Args import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} import za.co.absa.db.fadb.status.FunctionStatus import zio._ @@ -50,11 +55,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) - when(getCheckpointMockV2.apply(GetCheckpointV2Args(partitioningId, checkpointV2DTO1.id))) + when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO1.id))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "OK"), Some(checkpointItemFromDB1))))) - when(getCheckpointMockV2.apply(GetCheckpointV2Args(partitioningId, checkpointV2DTO2.id))) + when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO2.id))) .thenReturn(ZIO.left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) - when(getCheckpointMockV2.apply(GetCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) + when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) .thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) From 2a62b39f0e23224759e914cc8ea312aeb8cd48ad Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 16:40:56 +0200 Subject: [PATCH 34/50] refactoring as per discussion --- .../runs/functions/WriteCheckpointV2.scala | 6 ++++-- .../repository/CheckpointRepositoryImpl.scala | 2 +- .../server/model/WriteCheckpointV2Args.scala | 21 ------------------- .../WriteCheckpointV2IntegrationTests.scala | 2 +- .../CheckpointRepositoryUnitTests.scala | 2 +- 5 files changed, 7 insertions(+), 26 deletions(-) delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala index c1990f0c4..c67ef2718 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator -import za.co.absa.atum.server.model.WriteCheckpointV2Args import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus @@ -26,9 +25,10 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ import io.circe.syntax._ - import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ +import za.co.absa.atum.model.dto.CheckpointV2DTO +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => @@ -46,6 +46,8 @@ class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) with StandardStatusHandling object WriteCheckpointV2 { + case class WriteCheckpointV2Args(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) + val layer: URLayer[PostgresDatabaseProvider, WriteCheckpointV2] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index dd9646c14..86bd1ea75 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,9 +17,9 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError -import za.co.absa.atum.server.model.WriteCheckpointV2Args import zio._ import zio.interop.catz.asyncInstance diff --git a/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala b/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala deleted file mode 100644 index 3ce505ef6..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/WriteCheckpointV2Args.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import za.co.absa.atum.model.dto.CheckpointV2DTO - -case class WriteCheckpointV2Args(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index 0c21cea04..afbd6f18b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -6,7 +6,7 @@ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.atum.server.model.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 198a8d31c..4a876bed7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -20,8 +20,8 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.atum.server.model.WriteCheckpointV2Args import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.FunctionStatus import zio._ From bbe5efa2d265a1c2ea52c7418fb4e382af91a13d Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 16:44:11 +0200 Subject: [PATCH 35/50] refactoring as per discussion --- .../api/repository/CheckpointRepositoryImpl.scala | 11 +++-------- .../repository/CheckpointRepositoryUnitTests.scala | 2 -- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index be38bde19..f09ae5bf2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,17 +17,12 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.{ - GetPartitioningCheckpointV2, - WriteCheckpoint, - WriteCheckpointV2 -} +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.{GetPartitioningCheckpointV2, WriteCheckpoint, WriteCheckpointV2} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError -import za.co.absa.atum.server.model.{CheckpointItemFromDB, WriteCheckpointV2Args} +import za.co.absa.atum.server.model.CheckpointItemFromDB import zio._ import zio.interop.catz.asyncInstance diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 10b7b4266..785d61b1b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -27,9 +27,7 @@ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.atum.server.model.WriteCheckpointV2Args import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} -import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance From b8363a92dc2046e313d10531917feb73c9eeadcf Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 16:46:45 +0200 Subject: [PATCH 36/50] refactoring as per discussion --- .../absa/atum/model/dto/AtumContextDTO.scala | 1 - .../atum/model/dto/PartitioningSubmitDTO.scala | 2 +- .../za/co/absa/atum/model/dto/package.scala | 1 - .../model/utils/JsonSyntaxExtensions.scala | 18 +++++++++--------- .../GetPartitioningCheckpointV2.scala | 7 +++++-- .../repository/CheckpointRepositoryImpl.scala | 6 +++++- .../atum/server/model/PaginatedResult.scala | 1 - 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala index 7510dd4ca..8bb36a312 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.model.dto - import io.circe.generic.semiauto._ import io.circe._ diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala index ad4fce6e0..e3c5c8aad 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.model.dto import io.circe.generic.semiauto._ import io.circe._ -case class PartitioningSubmitDTO ( +case class PartitioningSubmitDTO( partitioning: PartitioningDTO, parentPartitioning: Option[PartitioningDTO], authorIfNew: String diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index a528e7348..e9e92829c 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.model - import io.circe._ package object dto { diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index 4a95546a9..d69d9dc36 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -22,17 +22,17 @@ import io.circe.{Decoder, Encoder} object JsonSyntaxExtensions { - implicit class JsonSerializationSyntax[T: Encoder](obj: T) { - def asJsonString: String = obj.asJson.noSpaces - } + implicit class JsonSerializationSyntax[T: Encoder](obj: T) { + def asJsonString: String = obj.asJson.noSpaces + } - implicit class JsonDeserializationSyntax(jsonStr: String) { - def as[T: Decoder]: T = { - decode[T](jsonStr) match { - case Right(value) => value - case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") - } + implicit class JsonDeserializationSyntax(jsonStr: String) { + def as[T: Decoder]: T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") } } + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala index 908e62535..56ac73377 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointV2.scala @@ -34,13 +34,16 @@ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAgg import java.util.UUID class GetPartitioningCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointV2Args, Option[CheckpointItemFromDB], Task](input => + extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointV2Args, Option[ + CheckpointItemFromDB + ], Task](input => Seq( fr"${input.partitioningId}", fr"${input.checkpointId}" ) ) - with StandardStatusHandling with ByFirstRowStatusAggregator { + with StandardStatusHandling + with ByFirstRowStatusAggregator { override def fieldsToSelect: Seq[String] = super.fieldsToSelect ++ Seq( "id_checkpoint", diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index f09ae5bf2..afdab753a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -19,7 +19,11 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.{GetPartitioningCheckpointV2, WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{ + GetPartitioningCheckpointV2, + WriteCheckpoint, + WriteCheckpointV2 +} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError import za.co.absa.atum.server.model.CheckpointItemFromDB diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala b/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala index f5f0b9154..e3a2898e4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala @@ -26,4 +26,3 @@ object PaginatedResult { case class ResultNoMore[R](data: Seq[R]) extends PaginatedResult[R] } - From e99d8632c014e5d45d72f68e8ae11d9510df7908 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 16:47:44 +0200 Subject: [PATCH 37/50] refactoring as per discussion --- .../absa/atum/model/dto/AtumContextDTO.scala | 1 - .../absa/atum/model/dto/CheckpointV2DTO.scala | 16 ++++++------- .../model/dto/PartitioningSubmitDTO.scala | 2 +- .../za/co/absa/atum/model/dto/package.scala | 1 - .../model/utils/JsonSyntaxExtensions.scala | 18 +++++++-------- .../api/controller/BaseController.scala | 6 +++-- .../runs/functions/WriteCheckpointV2.scala | 23 +++++++++---------- .../absa/atum/server/api/http/Endpoints.scala | 2 +- .../PartitioningRepositoryImpl.scala | 4 +++- .../api/service/PartitioningServiceImpl.scala | 4 +++- .../atum/server/model/CheckpointFromDB.scala | 9 +++++++- .../atum/server/model/ErrorResponse.scala | 6 ++--- .../atum/server/model/PaginatedResult.scala | 1 - .../atum/server/model/SuccessResponse.scala | 2 +- 14 files changed, 51 insertions(+), 44 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala index 7510dd4ca..8bb36a312 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.model.dto - import io.circe.generic.semiauto._ import io.circe._ diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala index 0ae4ad3fa..ad80b373e 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -23,14 +23,14 @@ import java.time.ZonedDateTime import java.util.UUID case class CheckpointV2DTO( - id: UUID, - name: String, - author: String, - measuredByAtumAgent: Boolean = false, - processStartTime: ZonedDateTime, - processEndTime: Option[ZonedDateTime], - measurements: Set[MeasurementDTO] - ) + id: UUID, + name: String, + author: String, + measuredByAtumAgent: Boolean = false, + processStartTime: ZonedDateTime, + processEndTime: Option[ZonedDateTime], + measurements: Set[MeasurementDTO] +) object CheckpointV2DTO { implicit val decodeCheckpointDTO: Decoder[CheckpointV2DTO] = deriveDecoder diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala index ad4fce6e0..e3c5c8aad 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.model.dto import io.circe.generic.semiauto._ import io.circe._ -case class PartitioningSubmitDTO ( +case class PartitioningSubmitDTO( partitioning: PartitioningDTO, parentPartitioning: Option[PartitioningDTO], authorIfNew: String diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index a528e7348..e9e92829c 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.model - import io.circe._ package object dto { diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index 4a95546a9..d69d9dc36 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -22,17 +22,17 @@ import io.circe.{Decoder, Encoder} object JsonSyntaxExtensions { - implicit class JsonSerializationSyntax[T: Encoder](obj: T) { - def asJsonString: String = obj.asJson.noSpaces - } + implicit class JsonSerializationSyntax[T: Encoder](obj: T) { + def asJsonString: String = obj.asJson.noSpaces + } - implicit class JsonDeserializationSyntax(jsonStr: String) { - def as[T: Decoder]: T = { - decode[T](jsonStr) match { - case Right(value) => value - case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") - } + implicit class JsonDeserializationSyntax(jsonStr: String) { + def as[T: Decoder]: T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") } } + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index ad33cf4a9..14e29939e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -55,14 +55,16 @@ trait BaseController { protected def createResourceUri(parts: Seq[String]): IO[ErrorResponse, String] = { for { - hostname <- System.env("HOSTNAME") + hostname <- System + .env("HOSTNAME") // fails the request if the hostname is not found, we need to make sure that the hostname is always available .orElseFail(InternalServerErrorResponse("Failed to get hostname")) .flatMap { case Some(value) => ZIO.succeed(value) case None => ZIO.fail(InternalServerErrorResponse("Failed to get hostname")) } - sslConfig <- ZIO.config[SslConfig](SslConfig.config) + sslConfig <- ZIO + .config[SslConfig](SslConfig.config) .orElseFail(InternalServerErrorResponse("Failed to get SSL config")) } yield { val protocol = if (sslConfig.enabled) "https" else "http" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala index c67ef2718..e66346cf9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala @@ -31,18 +31,18 @@ import za.co.absa.atum.model.dto.CheckpointV2DTO import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => - Seq( - fr"${args.partitioningId}", - fr"${args.checkpointV2DTO.id}", - fr"${args.checkpointV2DTO.name}", - fr"${args.checkpointV2DTO.processStartTime}", - fr"${args.checkpointV2DTO.processEndTime}", - fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", - fr"${args.checkpointV2DTO.measuredByAtumAgent}", - fr"${args.checkpointV2DTO.author}" + extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => + Seq( + fr"${args.partitioningId}", + fr"${args.checkpointV2DTO.id}", + fr"${args.checkpointV2DTO.name}", + fr"${args.checkpointV2DTO.processStartTime}", + fr"${args.checkpointV2DTO.processEndTime}", + fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", + fr"${args.checkpointV2DTO.measuredByAtumAgent}", + fr"${args.checkpointV2DTO.author}" + ) ) - ) with StandardStatusHandling object WriteCheckpointV2 { @@ -54,4 +54,3 @@ object WriteCheckpointV2 { } yield new WriteCheckpointV2()(Runs, dbProvider.dbEngine) } } - diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 19b25716a..083bf222b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -38,7 +38,7 @@ trait Endpoints extends BaseEndpoints { } protected val postCheckpointEndpointV2 - : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { + : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { apiV2.post .in(V2Paths.Partitionings / path[Long]("partitioningId") / V2Paths.Checkpoints) .in(jsonBody[CheckpointV2DTO]) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala index 523a98f0f..68ff69276 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/PartitioningRepositoryImpl.scala @@ -65,7 +65,9 @@ class PartitioningRepositoryImpl( }) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[DatabaseError, InitialAdditionalDataDTO] = { + override def getPartitioningAdditionalData( + partitioning: PartitioningDTO + ): IO[DatabaseError, InitialAdditionalDataDTO] = { dbMultipleResultCallWithAggregatedStatus( getPartitioningAdditionalDataFn(partitioning), "getPartitioningAdditionalData" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala index 3066878c3..eebb7a2ba 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/service/PartitioningServiceImpl.scala @@ -48,7 +48,9 @@ class PartitioningServiceImpl(partitioningRepository: PartitioningRepository) ) } - override def getPartitioningAdditionalData(partitioning: PartitioningDTO): IO[ServiceError, InitialAdditionalDataDTO] = { + override def getPartitioningAdditionalData( + partitioning: PartitioningDTO + ): IO[ServiceError, InitialAdditionalDataDTO] = { repositoryCall( partitioningRepository.getPartitioningAdditionalData(partitioning), "getPartitioningAdditionalData" diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 7d2d16809..c2e4177b3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -16,7 +16,14 @@ package za.co.absa.atum.server.model -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} +import za.co.absa.atum.model.dto.{ + CheckpointDTO, + CheckpointV2DTO, + MeasureDTO, + MeasureResultDTO, + MeasurementDTO, + PartitioningDTO +} import io.circe.{DecodingFailure, Json} import java.time.ZonedDateTime diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index ac3be1e73..78ff0b183 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -37,16 +37,14 @@ object BadRequestResponse { implicit val encodeBadRequestResponse: Encoder[BadRequestResponse] = deriveEncoder } -final case class ConflictErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class ConflictErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object ConflictErrorResponse { implicit val decoderConflictErrorResponse: Decoder[ConflictErrorResponse] = deriveDecoder implicit val encoderConflictErrorResponse: Encoder[ConflictErrorResponse] = deriveEncoder } -final case class NotFoundErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) - extends ErrorResponse +final case class NotFoundErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object NotFoundErrorResponse { implicit val decoderNotFoundErrorResponse: Decoder[NotFoundErrorResponse] = deriveDecoder diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala b/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala index f5f0b9154..e3a2898e4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PaginatedResult.scala @@ -26,4 +26,3 @@ object PaginatedResult { case class ResultNoMore[R](data: Seq[R]) extends PaginatedResult[R] } - diff --git a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala index 3c3341300..7fb5c6da3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala @@ -40,7 +40,7 @@ object SuccessResponse { } case class PaginatedResponse[T](data: Seq[T], pagination: Pagination, requestId: UUID = UUID.randomUUID()) - extends SuccessResponse + extends SuccessResponse object PaginatedResponse { implicit def encoder[T: Encoder]: Encoder[PaginatedResponse[T]] = deriveEncoder From f65b7c55ca8e353dd296291168443767c81b9a7d Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Aug 2024 16:56:10 +0200 Subject: [PATCH 38/50] fixes --- .../api/controller/BaseController.scala | 25 ++++--------------- .../controller/CheckpointControllerImpl.scala | 2 +- .../CheckpointControllerUnitTests.scala | 9 +------ .../WriteCheckpointV2IntegrationTests.scala | 16 ++++++++++++ 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 14e29939e..1d2316911 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ -import za.co.absa.atum.server.config.SslConfig +import za.co.absa.atum.server.api.http.ApiPaths import za.co.absa.atum.server.model.{ConflictErrorResponse, ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ @@ -53,24 +53,9 @@ trait BaseController { effect.map(MultiSuccessResponse(_)) } - protected def createResourceUri(parts: Seq[String]): IO[ErrorResponse, String] = { - for { - hostname <- System - .env("HOSTNAME") - // fails the request if the hostname is not found, we need to make sure that the hostname is always available - .orElseFail(InternalServerErrorResponse("Failed to get hostname")) - .flatMap { - case Some(value) => ZIO.succeed(value) - case None => ZIO.fail(InternalServerErrorResponse("Failed to get hostname")) - } - sslConfig <- ZIO - .config[SslConfig](SslConfig.config) - .orElseFail(InternalServerErrorResponse("Failed to get SSL config")) - } yield { - val protocol = if (sslConfig.enabled) "https" else "http" - val port = if (sslConfig.enabled) 8443 else 8080 - val path = parts.mkString("/") - s"$protocol://$hostname:$port/$path" - } + // Root-anchored URL path + // https://stackoverflow.com/questions/2005079/absolute-vs-relative-urls/78439286#78439286 + protected def createV2RootAnchoredResourcePath(parts: Seq[String]): IO[ErrorResponse, String] = { + ZIO.succeed(s"/${ApiPaths.Api}/${ApiPaths.V2}/${parts.mkString("/")}") } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 55b8fd58f..5b6e3ceb8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -45,7 +45,7 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che _ => checkpointV2DTO ) ) - uri <- createResourceUri( + uri <- createV2RootAnchoredResourcePath( Seq(V2Paths.Partitionings, partitioningId.toString, V2Paths.Checkpoints, checkpointV2DTO.id.toString) ) } yield (response, uri) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 93f364d49..41e0daaa4 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -72,19 +72,12 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { ), suite("PostCheckpointV2Suite")( test("Returns expected CheckpointDTO") { - val host = "testHost" for { - _ <- TestSystem.putEnv("HOSTNAME", host) - sslConfig <- ZIO.config[SslConfig](SslConfig.config) result <- CheckpointController.postCheckpointV2(partitioningId, checkpointV2DTO1) - protocol = if (sslConfig.enabled) "https" else "http" - port = if (sslConfig.enabled) 8443 else 8080 - path = s"${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}/${checkpointV2DTO1.id}" - expectedUri = s"$protocol://$host:$port/$path" } yield assertTrue( result._1.isInstanceOf[SingleSuccessResponse[CheckpointV2DTO]] && result._1.data == checkpointV2DTO1 - && result._2 == expectedUri + && result._2 == s"/api/v2/partitionings/$partitioningId/checkpoints/${checkpointV2DTO1.id}" ) }, test("Returns expected ConflictServiceError") { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index afbd6f18b..36fe73ee9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType From 4ad55052d0ac793fc37f5f2762b5aee2561dff22 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 14 Aug 2024 15:13:15 +0200 Subject: [PATCH 39/50] docs for sql updated --- database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql index 4feb584d0..82064f658 100644 --- a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql @@ -28,7 +28,7 @@ CREATE OR REPLACE FUNCTION runs.write_checkpoint_v2( $$ ------------------------------------------------------------------------------- -- --- Function: runs.write_checkpoint_v2(10) +-- Function: runs.write_checkpoint_v2(8) -- Creates a checkpoint and adds all the measurements that it consists of -- -- Parameters: From f76e6f890b50e64f6ecb1425f1a8d2631eb177c4 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 15 Aug 2024 15:30:04 +0200 Subject: [PATCH 40/50] extend Exception instead of Throwable --- .../scala/za/co/absa/atum/server/api/exception/AppError.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala b/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala index e33f02cd4..ff4411b7b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/exception/AppError.scala @@ -16,6 +16,6 @@ package za.co.absa.atum.server.api.exception -abstract class AppError extends Throwable { +abstract class AppError extends Exception { def message: String } From 73af31aa5c7c7c005d508eb72405edf88f3b2921 Mon Sep 17 00:00:00 2001 From: salamonpavel Date: Thu, 15 Aug 2024 15:30:35 +0200 Subject: [PATCH 41/50] Update database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql Co-authored-by: David Benedeki <14905969+benedeki@users.noreply.github.com> --- database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql index 82064f658..b515c0ef3 100644 --- a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql @@ -106,4 +106,4 @@ $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; ALTER FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; \ No newline at end of file +GRANT EXECUTE ON FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; From 4a8eca2e5c20e56425958f1d71d8050b5aba365f Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 09:18:13 +0200 Subject: [PATCH 42/50] github comments addressed --- ...t.sql => V1.5.10__write_checkpoint_v1.sql} | 58 ++++------- ...nt_v2.sql => V1.9.2__write_checkpoint.sql} | 8 +- .../WriteCheckpointIntegrationTests.scala | 95 ++++++++++--------- ...> WriteCheckpointV1IntegrationTests.scala} | 93 +++++++++--------- 4 files changed, 117 insertions(+), 137 deletions(-) rename database/src/main/postgres/runs/{V1.5.10__write_checkpoint.sql => V1.5.10__write_checkpoint_v1.sql} (63%) rename database/src/main/postgres/runs/{V1.9.2__write_checkpoint_v2.sql => V1.9.2__write_checkpoint.sql} (91%) rename database/src/test/scala/za/co/absa/atum/database/runs/{WriteCheckpointV2IntegrationTests.scala => WriteCheckpointV1IntegrationTests.scala} (88%) diff --git a/database/src/main/postgres/runs/V1.5.10__write_checkpoint.sql b/database/src/main/postgres/runs/V1.5.10__write_checkpoint_v1.sql similarity index 63% rename from database/src/main/postgres/runs/V1.5.10__write_checkpoint.sql rename to database/src/main/postgres/runs/V1.5.10__write_checkpoint_v1.sql index 833769bc1..1d2e3c149 100644 --- a/database/src/main/postgres/runs/V1.5.10__write_checkpoint.sql +++ b/database/src/main/postgres/runs/V1.5.10__write_checkpoint_v1.sql @@ -14,7 +14,7 @@ */ -CREATE OR REPLACE FUNCTION runs.write_checkpoint( +CREATE OR REPLACE FUNCTION runs.write_checkpoint_v1( IN i_partitioning JSONB, IN i_id_checkpoint UUID, IN i_checkpoint_name TEXT, @@ -29,7 +29,7 @@ CREATE OR REPLACE FUNCTION runs.write_checkpoint( $$ ------------------------------------------------------------------------------- -- --- Function: runs.write_checkpoint(10) +-- Function: runs.write_checkpoint_v1(10) -- Creates a checkpoint and adds all the measurements that it consists of -- -- Parameters: @@ -57,58 +57,34 @@ $$ -- -- Status codes: -- 11 - Checkpoint created --- 14 - Checkpoint already present --- 41 - Partitioning not found +-- 31 - Checkpoint already present +-- 32 - Partitioning not found -- ------------------------------------------------------------------------------- DECLARE _fk_partitioning BIGINT; + result RECORD; BEGIN - PERFORM 1 - FROM runs.checkpoints CP - WHERE CP.id_checkpoint = i_id_checkpoint; - - IF found THEN - status := 14; - status_text := 'Checkpoint already present'; - RETURN; - END IF; - _fk_partitioning = runs._get_id_partitioning(i_partitioning); - IF _fk_partitioning IS NULL THEN - status := 41; - status_text := 'Partitioning not found'; - RETURN; - END IF; - - INSERT INTO runs.checkpoints (id_checkpoint, fk_partitioning, - checkpoint_name, measured_by_atum_agent, - process_start_time, process_end_time, created_by) - VALUES (i_id_checkpoint, _fk_partitioning, - i_checkpoint_name, i_measured_by_atum_agent, - i_process_start_time, i_process_end_time, i_by_user); - - -- maybe could use `jsonb_populate_record` function to be little bit more effective - PERFORM runs._write_measurement( - i_id_checkpoint, + result = runs.write_checkpoint( _fk_partitioning, - UN.measurement->'measure'->>'measureName', - jsonb_array_to_text_array(UN.measurement->'measure'->'measuredColumns'), - UN.measurement->'result', + i_id_checkpoint, + i_checkpoint_name, + i_process_start_time, + i_process_end_time, + i_measurements, + i_measured_by_atum_agent, i_by_user - ) - FROM ( - SELECT unnest(i_measurements) AS measurement - ) UN; + ); - status := 11; - status_text := 'Checkpoint created'; + status := result.status; + status_text := result.status_text; RETURN; END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.write_checkpoint(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.write_checkpoint(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; +ALTER FUNCTION runs.write_checkpoint_v1(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.write_checkpoint_v1(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; diff --git a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql b/database/src/main/postgres/runs/V1.9.2__write_checkpoint.sql similarity index 91% rename from database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql rename to database/src/main/postgres/runs/V1.9.2__write_checkpoint.sql index b515c0ef3..61bbcd31f 100644 --- a/database/src/main/postgres/runs/V1.9.2__write_checkpoint_v2.sql +++ b/database/src/main/postgres/runs/V1.9.2__write_checkpoint.sql @@ -13,7 +13,7 @@ * limitations under the License. */ -CREATE OR REPLACE FUNCTION runs.write_checkpoint_v2( +CREATE OR REPLACE FUNCTION runs.write_checkpoint( IN i_partitioning_id BIGINT, IN i_id_checkpoint UUID, IN i_checkpoint_name TEXT, @@ -28,7 +28,7 @@ CREATE OR REPLACE FUNCTION runs.write_checkpoint_v2( $$ ------------------------------------------------------------------------------- -- --- Function: runs.write_checkpoint_v2(8) +-- Function: runs.write_checkpoint(8) -- Creates a checkpoint and adds all the measurements that it consists of -- -- Parameters: @@ -105,5 +105,5 @@ END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.write_checkpoint_v2(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; +ALTER FUNCTION runs.write_checkpoint(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.write_checkpoint(BIGINT, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala index 3c087558f..c816d25ac 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala @@ -25,6 +25,8 @@ import java.util.UUID class WriteCheckpointIntegrationTests extends DBTestSuite { + private val fnWriteCheckpoint = "runs.write_checkpoint" + private val partitioning = JsonBString( """ |{ @@ -40,6 +42,42 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { |""".stripMargin ) + private val measurements = + """ + |{ + | "{ + | \"measure\": { + | \"measureName\": \"count\", + | \"measuredColumns\": [] + | }, + | \"result\":{ + | \"value\":\"3\", + | \"type\":\"int\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"col1\"] + | }, + | \"result\":{ + | \"value\":\"3.14\", + | \"type\":\"double\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"a\",\"b\"] + | }, + | \"result\":{ + | \"value\":\"2.71\", + | \"type\":\"double\" + | } + | }" + |} + |""".stripMargin + test("Write new checkpoint without data") { val uuid = UUID.randomUUID val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") @@ -62,8 +100,8 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) - function("runs.write_checkpoint") - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Empty path") .setParam("i_process_start_time", startTime) @@ -98,41 +136,7 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { val user = "Franz Kafka" val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") - val measurements = - """ - |{ - | "{ - | \"measure\": { - | \"measureName\": \"count\", - | \"measuredColumns\": [] - | }, - | \"result\":{ - | \"value\":\"3\", - | \"type\":\"int\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"col1\"] - | }, - | \"result\":{ - | \"value\":\"3.14\", - | \"type\":\"double\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"a\",\"b\"] - | }, - | \"result\":{ - | \"value\":\"2.71\", - | \"type\":\"double\" - | } - | }" - |} - |""".stripMargin + table("runs.partitionings").insert( add("partitioning", partitioning) @@ -150,8 +154,8 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) - function("runs.write_checkpoint") - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Happy path") .setParam("i_process_start_time", startTime) @@ -237,8 +241,8 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { .add("created_by", origAuthor) ) - function("runs.write_checkpoint") - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -249,7 +253,7 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() - assert(row.getInt("status").contains(14)) + assert(row.getInt("status").contains(31)) assert(row.getString("status_text").contains("Checkpoint already present")) } @@ -264,8 +268,8 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { test("Partitioning of the checkpoint does not exist") { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() - function("runs.write_checkpoint") - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", 0L) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -276,9 +280,10 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() - assert(row.getInt("status").contains(41)) + assert(row.getInt("status").contains(32)) assert(row.getString("status_text").contains("Partitioning not found")) } assert(table("runs.checkpoints").count() == count) } + } diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV1IntegrationTests.scala similarity index 88% rename from database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala rename to database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV1IntegrationTests.scala index 2ebdb3d29..6d26748b0 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV2IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV1IntegrationTests.scala @@ -23,7 +23,9 @@ import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime import java.util.UUID -class WriteCheckpointV2IntegrationTests extends DBTestSuite { +class WriteCheckpointV1IntegrationTests extends DBTestSuite { + + private val fnWriteCheckpointV1 = "runs.write_checkpoint_v1" private val partitioning = JsonBString( """ @@ -40,42 +42,6 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { |""".stripMargin ) - private val measurements = - """ - |{ - | "{ - | \"measure\": { - | \"measureName\": \"count\", - | \"measuredColumns\": [] - | }, - | \"result\":{ - | \"value\":\"3\", - | \"type\":\"int\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"col1\"] - | }, - | \"result\":{ - | \"value\":\"3.14\", - | \"type\":\"double\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"a\",\"b\"] - | }, - | \"result\":{ - | \"value\":\"2.71\", - | \"type\":\"double\" - | } - | }" - |} - |""".stripMargin - test("Write new checkpoint without data") { val uuid = UUID.randomUUID val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") @@ -98,8 +64,8 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) - function("runs.write_checkpoint_v2") - .setParam("i_partitioning_id", fkPartitioning) + function(fnWriteCheckpointV1) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Empty path") .setParam("i_process_start_time", startTime) @@ -134,7 +100,41 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { val user = "Franz Kafka" val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") - + val measurements = + """ + |{ + | "{ + | \"measure\": { + | \"measureName\": \"count\", + | \"measuredColumns\": [] + | }, + | \"result\":{ + | \"value\":\"3\", + | \"type\":\"int\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"col1\"] + | }, + | \"result\":{ + | \"value\":\"3.14\", + | \"type\":\"double\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"a\",\"b\"] + | }, + | \"result\":{ + | \"value\":\"2.71\", + | \"type\":\"double\" + | } + | }" + |} + |""".stripMargin table("runs.partitionings").insert( add("partitioning", partitioning) @@ -152,8 +152,8 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) - function("runs.write_checkpoint_v2") - .setParam("i_partitioning_id", fkPartitioning) + function(fnWriteCheckpointV1) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Happy path") .setParam("i_process_start_time", startTime) @@ -239,8 +239,8 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { .add("created_by", origAuthor) ) - function("runs.write_checkpoint_v2") - .setParam("i_partitioning_id", fkPartitioning) + function(fnWriteCheckpointV1) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -266,8 +266,8 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { test("Partitioning of the checkpoint does not exist") { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() - function("runs.write_checkpoint_v2") - .setParam("i_partitioning_id", 0L) + function(fnWriteCheckpointV1) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -283,5 +283,4 @@ class WriteCheckpointV2IntegrationTests extends DBTestSuite { } assert(table("runs.checkpoints").count() == count) } - } From 2418cfd302554b441aa03bc81980bc3e7ac3a0e7 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 09:20:08 +0200 Subject: [PATCH 43/50] v1 endpoint changed --- .../main/scala/za/co/absa/atum/server/api/http/Endpoints.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 083bf222b..174babd8f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -35,6 +35,7 @@ trait Endpoints extends BaseEndpoints { .in(jsonBody[CheckpointDTO]) .out(statusCode(StatusCode.Created)) .out(jsonBody[CheckpointDTO]) + .errorOutVariantPrepend(conflictErrorOneOfVariant) } protected val postCheckpointEndpointV2 From ac410843c35eea9d9c306590310c40b4fb277479 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 09:24:24 +0200 Subject: [PATCH 44/50] v1 tests --- .../controller/CheckpointControllerUnitTests.scala | 12 +++++------- .../repository/CheckpointRepositoryUnitTests.scala | 8 +++----- .../api/service/CheckpointServiceUnitTests.scala | 4 ++-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 41e0daaa4..6b4ed6b2e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -17,17 +17,15 @@ package za.co.absa.atum.server.api.controller import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} +import za.co.absa.atum.model.dto.CheckpointV2DTO import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError._ -import za.co.absa.atum.server.api.http.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.config.SslConfig -import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse -import zio.test.Assertion.failsWithA +import za.co.absa.atum.server.model.{ConflictErrorResponse, InternalServerErrorResponse} import zio._ +import zio.test.Assertion.failsWithA import zio.test._ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { @@ -38,7 +36,7 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { when(checkpointServiceMock.saveCheckpoint(checkpointDTO2)) .thenReturn(ZIO.fail(GeneralServiceError("error in data"))) when(checkpointServiceMock.saveCheckpoint(checkpointDTO3)) - .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) + .thenReturn(ZIO.fail(ConflictServiceError("boom!"))) private val partitioningId = 1L @@ -66,7 +64,7 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { }, test("Returns expected ConflictServiceError") { assertZIO(CheckpointController.createCheckpointV1(checkpointDTO3).exit)( - failsWithA[InternalServerErrorResponse] + failsWithA[ConflictErrorResponse] ) } ), diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 4a876bed7..7ad091e72 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -37,7 +37,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) when(writeCheckpointMock.apply(checkpointDTO2)) - .thenReturn(ZIO.fail(GeneralDatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null"))) + .thenReturn(ZIO.fail(DataConflictException(FunctionStatus(31, "conflict")))) when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val partitioningId = 1L @@ -62,10 +62,8 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { } yield assertTrue(result == ()) }, test("Returns expected Left with StatusException") { - for { - result <- CheckpointRepository.writeCheckpoint(checkpointDTO2).exit - } yield assertTrue( - result == Exit.fail(GeneralDatabaseError("Operation 'writeCheckpoint' failed with unexpected error: null")) + assertZIO(CheckpointRepository.writeCheckpoint(checkpointDTO2).exit)( + failsWithA[ConflictDatabaseError] ) }, test("Returns expected DatabaseError") { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala index 79d9e7127..57a706cfd 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/service/CheckpointServiceUnitTests.scala @@ -31,7 +31,7 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO1)).thenReturn(ZIO.unit) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO2)) - .thenReturn(ZIO.fail(GeneralDatabaseError("error in data"))) + .thenReturn(ZIO.fail(ConflictDatabaseError("error in data"))) when(checkpointRepositoryMock.writeCheckpoint(checkpointDTO3)).thenReturn(ZIO.fail(GeneralDatabaseError("boom!"))) private val partitioningId = 1L @@ -57,7 +57,7 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { for { result <- CheckpointService.saveCheckpoint(checkpointDTO2).exit } yield assertTrue( - result == Exit.fail(GeneralServiceError("Failed to perform 'saveCheckpoint': error in data")) + result == Exit.fail(ConflictServiceError("Failed to perform 'saveCheckpoint': error in data")) ) }, test("Returns expected ServiceError") { From aec2ec18bfb450b13e6a0d39f6620a9cca4c1ebc Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 09:43:45 +0200 Subject: [PATCH 45/50] v1 tests --- .../scala/za/co/absa/atum/server/Main.scala | 2 +- .../runs/functions/WriteCheckpoint.scala | 27 +++++++------- ...kpointV2.scala => WriteCheckpointV1.scala} | 35 ++++++++++--------- .../repository/CheckpointRepositoryImpl.scala | 16 ++++----- .../WriteCheckpointIntegrationTests.scala | 18 +++++----- ...> WriteCheckpointV1IntegrationTests.scala} | 22 ++++++------ .../CheckpointRepositoryUnitTests.scala | 22 ++++++------ 7 files changed, 71 insertions(+), 71 deletions(-) rename server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/{WriteCheckpointV2.scala => WriteCheckpointV1.scala} (59%) rename server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/{WriteCheckpointV2IntegrationTests.scala => WriteCheckpointV1IntegrationTests.scala} (73%) diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 5a07279b6..146502c9b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -55,8 +55,8 @@ object Main extends ZIOAppDefault with Server { GetPartitioningAdditionalData.layer, CreateOrUpdateAdditionalData.layer, GetPartitioningCheckpoints.layer, + WriteCheckpointV1.layer, WriteCheckpoint.layer, - WriteCheckpointV2.layer, GetFlowCheckpoints.layer, PostgresDatabaseProvider.layer, TransactorProvider.layer, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index ce6c5c129..33a05ec67 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -17,8 +17,6 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator -import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus @@ -27,28 +25,29 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ import io.circe.syntax._ -import za.co.absa.atum.model.dto.MeasureResultDTO._ -import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get -import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ +import za.co.absa.atum.model.dto.CheckpointV2DTO +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => + extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => Seq( - fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", - fr"${values.id}", - fr"${values.name}", - fr"${values.processStartTime}", - fr"${values.processEndTime}", - fr"${values.measurements.toList.map(_.asJson)}", - fr"${values.measuredByAtumAgent}", - fr"${values.author}" + fr"${args.partitioningId}", + fr"${args.checkpointV2DTO.id}", + fr"${args.checkpointV2DTO.name}", + fr"${args.checkpointV2DTO.processStartTime}", + fr"${args.checkpointV2DTO.processEndTime}", + fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", + fr"${args.checkpointV2DTO.measuredByAtumAgent}", + fr"${args.checkpointV2DTO.author}" ) ) with StandardStatusHandling object WriteCheckpoint { + case class WriteCheckpointV2Args(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) + val layer: URLayer[PostgresDatabaseProvider, WriteCheckpoint] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1.scala similarity index 59% rename from server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala rename to server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1.scala index e66346cf9..00570927c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1.scala @@ -17,6 +17,8 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator +import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus @@ -25,32 +27,31 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ import io.circe.syntax._ +import za.co.absa.atum.model.dto.MeasureResultDTO._ +import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ -import za.co.absa.atum.model.dto.CheckpointV2DTO -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args -class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => +class WriteCheckpointV1(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => Seq( - fr"${args.partitioningId}", - fr"${args.checkpointV2DTO.id}", - fr"${args.checkpointV2DTO.name}", - fr"${args.checkpointV2DTO.processStartTime}", - fr"${args.checkpointV2DTO.processEndTime}", - fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", - fr"${args.checkpointV2DTO.measuredByAtumAgent}", - fr"${args.checkpointV2DTO.author}" + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.id}", + fr"${values.name}", + fr"${values.processStartTime}", + fr"${values.processEndTime}", + fr"${values.measurements.toList.map(_.asJson)}", + fr"${values.measuredByAtumAgent}", + fr"${values.author}" ) ) with StandardStatusHandling -object WriteCheckpointV2 { - case class WriteCheckpointV2Args(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) - - val layer: URLayer[PostgresDatabaseProvider, WriteCheckpointV2] = ZLayer { +object WriteCheckpointV1 { + val layer: URLayer[PostgresDatabaseProvider, WriteCheckpointV1] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] - } yield new WriteCheckpointV2()(Runs, dbProvider.dbEngine) + } yield new WriteCheckpointV1()(Runs, dbProvider.dbEngine) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 86bd1ea75..45078dcd5 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,33 +17,33 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} import za.co.absa.atum.server.api.exception.DatabaseError import zio._ import zio.interop.catz.asyncInstance -class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint, writeCheckpointV2Fn: WriteCheckpointV2) +class CheckpointRepositoryImpl(writeCheckpointV1Fn: WriteCheckpointV1, writeCheckpointFn: WriteCheckpoint) extends CheckpointRepository with BaseRepository { override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { - dbSingleResultCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") + dbSingleResultCallWithStatus(writeCheckpointV1Fn(checkpointDTO), "writeCheckpoint") } override def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus( - writeCheckpointV2Fn(WriteCheckpointV2Args(partitioningId, checkpointV2DTO)), + writeCheckpointFn(WriteCheckpointV2Args(partitioningId, checkpointV2DTO)), "writeCheckpoint" ) } } object CheckpointRepositoryImpl { - val layer: URLayer[WriteCheckpoint with WriteCheckpointV2, CheckpointRepository] = ZLayer { + val layer: URLayer[WriteCheckpointV1 with WriteCheckpoint, CheckpointRepository] = ZLayer { for { + writeCheckpointV1 <- ZIO.service[WriteCheckpointV1] writeCheckpoint <- ZIO.service[WriteCheckpoint] - writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] - } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2) + } yield new CheckpointRepositoryImpl(writeCheckpointV1, writeCheckpoint) } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 68436230c..584851d71 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -17,13 +17,14 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} +import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.db.fadb.exceptions.DataNotFoundException -import za.co.absa.db.fadb.status.FunctionStatus +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args +import za.co.absa.db.fadb.exceptions.DataConflictException +import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ import zio.interop.catz.asyncInstance import zio.test._ @@ -35,13 +36,12 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("WriteCheckpointSuite")( + suite("WriteCheckpointV2Suite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { - val checkpointDTO = CheckpointDTO( + val checkpointV2DTO = CheckpointV2DTO( id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = Set( @@ -49,9 +49,9 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ) ) for { - writeCheckpoint <- ZIO.service[WriteCheckpoint] - result <- writeCheckpoint(checkpointDTO) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + writeCheckpointV2 <- ZIO.service[WriteCheckpoint] + result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointV2DTO)) + } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( WriteCheckpoint.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala similarity index 73% rename from server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala rename to server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala index 36fe73ee9..6ba52ac5b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala @@ -17,14 +17,13 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto._ +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args -import za.co.absa.db.fadb.exceptions.DataConflictException -import za.co.absa.db.fadb.status.{FunctionStatus, Row} +import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance import zio.test._ @@ -32,16 +31,17 @@ import zio.test._ import java.time.ZonedDateTime import java.util.UUID -object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { +object WriteCheckpointV1IntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("WriteCheckpointV2Suite")( + suite("WriteCheckpointSuite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { - val checkpointV2DTO = CheckpointV2DTO( + val checkpointDTO = CheckpointDTO( id = UUID.randomUUID(), name = "name", author = "author", + partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = Set( @@ -49,12 +49,12 @@ object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { ) ) for { - writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] - result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointV2DTO)) - } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) + writeCheckpoint <- ZIO.service[WriteCheckpointV1] + result <- writeCheckpoint(checkpointDTO) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(32, "Partitioning not found")))) } ).provide( - WriteCheckpointV2.layer, + WriteCheckpointV1.layer, PostgresDatabaseProvider.layer, TestTransactorProvider.layerWithRollback ) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 7ad091e72..21dc1f6af 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,10 +17,10 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV2} +import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.FunctionStatus @@ -32,25 +32,25 @@ import za.co.absa.db.fadb.status.Row object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { + private val writeCheckpointV1Mock: WriteCheckpointV1 = mock(classOf[WriteCheckpointV1]) private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) - private val writeCheckpointMockV2: WriteCheckpointV2 = mock(classOf[WriteCheckpointV2]) - when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) - when(writeCheckpointMock.apply(checkpointDTO2)) + when(writeCheckpointV1Mock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) + when(writeCheckpointV1Mock.apply(checkpointDTO2)) .thenReturn(ZIO.fail(DataConflictException(FunctionStatus(31, "conflict")))) - when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) + when(writeCheckpointV1Mock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val partitioningId = 1L - when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO1))) + when(writeCheckpointMock.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO1))) .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) - when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO2))) + when(writeCheckpointMock.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO2))) .thenReturn(ZIO.left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) - when(writeCheckpointMockV2.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) + when(writeCheckpointMock.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) + private val writeCheckpointV1MockLayer = ZLayer.succeed(writeCheckpointV1Mock) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) - private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointMockV2) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -87,7 +87,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ) - ).provide(CheckpointRepositoryImpl.layer, writeCheckpointMockLayer, writeCheckpointV2MockLayer) + ).provide(CheckpointRepositoryImpl.layer, writeCheckpointV1MockLayer, writeCheckpointMockLayer) } From 6574bf13daeb1eb7c782dc714176bf4621a28f73 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 09:47:53 +0200 Subject: [PATCH 46/50] v1 tests --- .../runs/functions/WriteCheckpointV1IntegrationTests.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala index 6ba52ac5b..988074be9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala @@ -22,7 +22,7 @@ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.db.fadb.exceptions.DataNotFoundException +import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance @@ -51,7 +51,7 @@ object WriteCheckpointV1IntegrationTests extends ConfigProviderTest { for { writeCheckpoint <- ZIO.service[WriteCheckpointV1] result <- writeCheckpoint(checkpointDTO) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(32, "Partitioning not found")))) + } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( WriteCheckpointV1.layer, From d8acd1578eaae8ce4d381ca8554b0f33bd6246bc Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 09:53:04 +0200 Subject: [PATCH 47/50] rename --- .../api/database/runs/functions/WriteCheckpoint.scala | 6 +++--- .../server/api/repository/CheckpointRepositoryImpl.scala | 4 ++-- .../runs/functions/WriteCheckpointIntegrationTests.scala | 4 ++-- .../api/repository/CheckpointRepositoryUnitTests.scala | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 33a05ec67..3eb2ff5a2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -28,10 +28,10 @@ import io.circe.syntax._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ import za.co.absa.atum.model.dto.CheckpointV2DTO -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[WriteCheckpointV2Args, Unit, Task](args => + extends DoobieSingleResultFunctionWithStatus[WriteCheckpointArgs, Unit, Task](args => Seq( fr"${args.partitioningId}", fr"${args.checkpointV2DTO.id}", @@ -46,7 +46,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) with StandardStatusHandling object WriteCheckpoint { - case class WriteCheckpointV2Args(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) + case class WriteCheckpointArgs(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) val layer: URLayer[PostgresDatabaseProvider, WriteCheckpoint] = ZLayer { for { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index 45078dcd5..bfdf5aba9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} import za.co.absa.atum.server.api.exception.DatabaseError import zio._ @@ -33,7 +33,7 @@ class CheckpointRepositoryImpl(writeCheckpointV1Fn: WriteCheckpointV1, writeChec override def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus( - writeCheckpointFn(WriteCheckpointV2Args(partitioningId, checkpointV2DTO)), + writeCheckpointFn(WriteCheckpointArgs(partitioningId, checkpointV2DTO)), "writeCheckpoint" ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 584851d71..39662ac48 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -22,7 +22,7 @@ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ @@ -50,7 +50,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ) for { writeCheckpointV2 <- ZIO.service[WriteCheckpoint] - result <- writeCheckpointV2(WriteCheckpointV2Args(1L, checkpointV2DTO)) + result <- writeCheckpointV2(WriteCheckpointArgs(1L, checkpointV2DTO)) } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 21dc1f6af..bac3a078b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -20,7 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.FunctionStatus @@ -42,11 +42,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val partitioningId = 1L - when(writeCheckpointMock.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO1))) + when(writeCheckpointMock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO1))) .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) - when(writeCheckpointMock.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO2))) + when(writeCheckpointMock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO2))) .thenReturn(ZIO.left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) - when(writeCheckpointMock.apply(WriteCheckpointV2Args(partitioningId, checkpointV2DTO3))) + when(writeCheckpointMock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) private val writeCheckpointV1MockLayer = ZLayer.succeed(writeCheckpointV1Mock) From a6ada8d188cdd7d5966b115225e8aa925bb89e5c Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 19 Aug 2024 10:13:33 +0200 Subject: [PATCH 48/50] merge conflicts resolved --- .../repository/CheckpointRepositoryImpl.scala | 10 +++------- .../WriteCheckpointIntegrationTests.scala | 8 ++++---- .../CheckpointRepositoryUnitTests.scala | 18 ++++-------------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index a4e2052ee..3da3c21e9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -18,20 +18,16 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.{ - GetPartitioningCheckpointV2, - WriteCheckpoint, - WriteCheckpointV2 -} import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} +import za.co.absa.atum.server.api.database.runs.functions._ import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError import za.co.absa.atum.server.model.CheckpointItemFromDB import zio._ import zio.interop.catz.asyncInstance +import java.util.UUID + class CheckpointRepositoryImpl( writeCheckpointV1Fn: WriteCheckpointV1, writeCheckpointFn: WriteCheckpoint, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 39662ac48..63d898955 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -17,14 +17,14 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs import za.co.absa.db.fadb.exceptions.DataConflictException -import za.co.absa.db.fadb.status.{FunctionStatus, Row} +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance import zio.test._ @@ -49,8 +49,8 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ) ) for { - writeCheckpointV2 <- ZIO.service[WriteCheckpoint] - result <- writeCheckpointV2(WriteCheckpointArgs(1L, checkpointV2DTO)) + writeCheckpoint <- ZIO.service[WriteCheckpoint] + result <- writeCheckpoint(WriteCheckpointArgs(1L, checkpointV2DTO)) } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index 66f49f198..a7c385b22 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,31 +17,23 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{ - GetPartitioningCheckpointV2, - WriteCheckpoint, - WriteCheckpointV2 -} -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} -import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointV2Args import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions._ +import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} -import za.co.absa.db.fadb.status.FunctionStatus +import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ import zio.interop.catz.asyncInstance import zio.test.Assertion.failsWithA import zio.test._ -import za.co.absa.db.fadb.status.Row object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointV1Mock: WriteCheckpointV1 = mock(classOf[WriteCheckpointV1]) private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) - private val writeCheckpointMockV2: WriteCheckpointV2 = mock(classOf[WriteCheckpointV2]) private val getCheckpointMockV2: GetPartitioningCheckpointV2 = mock(classOf[GetPartitioningCheckpointV2]) when(writeCheckpointV1Mock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) @@ -67,7 +59,6 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { private val writeCheckpointV1MockLayer = ZLayer.succeed(writeCheckpointV1Mock) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) - private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointMockV2) private val getCheckpointV2MockLayer = ZLayer.succeed(getCheckpointMockV2) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -124,11 +115,10 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { ) ).provide( CheckpointRepositoryImpl.layer, + writeCheckpointV1MockLayer, writeCheckpointMockLayer, - writeCheckpointV2MockLayer, getCheckpointV2MockLayer ) - ).provide(CheckpointRepositoryImpl.layer, writeCheckpointV1MockLayer, writeCheckpointMockLayer) } From 867cf408d0109ef033170935498255548c1b453b Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 20 Aug 2024 10:24:32 +0200 Subject: [PATCH 49/50] overloaded sql code --- ...t_v1.sql => V1.5.10__write_checkpoint.sql} | 39 ++++---- .../WriteCheckpointIntegrationTests.scala | 81 ++++++++-------- ...heckpointOverloadedIntegrationTests.scala} | 93 ++++++++++--------- .../scala/za/co/absa/atum/server/Main.scala | 2 +- .../runs/functions/WriteCheckpoint.scala | 27 +++--- ...kpointV1.scala => WriteCheckpointV2.scala} | 38 ++++---- .../repository/CheckpointRepositoryImpl.scala | 16 ++-- .../WriteCheckpointIntegrationTests.scala | 16 ++-- ...> WriteCheckpointV2IntegrationTests.scala} | 20 ++-- .../CheckpointRepositoryUnitTests.scala | 22 ++--- 10 files changed, 180 insertions(+), 174 deletions(-) rename database/src/main/postgres/runs/{V1.5.10__write_checkpoint_v1.sql => V1.5.10__write_checkpoint.sql} (76%) rename database/src/test/scala/za/co/absa/atum/database/runs/{WriteCheckpointV1IntegrationTests.scala => WriteCheckpointOverloadedIntegrationTests.scala} (88%) rename server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/{WriteCheckpointV1.scala => WriteCheckpointV2.scala} (58%) rename server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/{WriteCheckpointV1IntegrationTests.scala => WriteCheckpointV2IntegrationTests.scala} (76%) diff --git a/database/src/main/postgres/runs/V1.5.10__write_checkpoint_v1.sql b/database/src/main/postgres/runs/V1.5.10__write_checkpoint.sql similarity index 76% rename from database/src/main/postgres/runs/V1.5.10__write_checkpoint_v1.sql rename to database/src/main/postgres/runs/V1.5.10__write_checkpoint.sql index 1d2e3c149..d5983d6a2 100644 --- a/database/src/main/postgres/runs/V1.5.10__write_checkpoint_v1.sql +++ b/database/src/main/postgres/runs/V1.5.10__write_checkpoint.sql @@ -14,7 +14,7 @@ */ -CREATE OR REPLACE FUNCTION runs.write_checkpoint_v1( +CREATE OR REPLACE FUNCTION runs.write_checkpoint( IN i_partitioning JSONB, IN i_id_checkpoint UUID, IN i_checkpoint_name TEXT, @@ -29,7 +29,7 @@ CREATE OR REPLACE FUNCTION runs.write_checkpoint_v1( $$ ------------------------------------------------------------------------------- -- --- Function: runs.write_checkpoint_v1(10) +-- Function: runs.write_checkpoint(10) -- Creates a checkpoint and adds all the measurements that it consists of -- -- Parameters: @@ -63,28 +63,33 @@ $$ ------------------------------------------------------------------------------- DECLARE _fk_partitioning BIGINT; - result RECORD; BEGIN _fk_partitioning = runs._get_id_partitioning(i_partitioning); - result = runs.write_checkpoint( - _fk_partitioning, - i_id_checkpoint, - i_checkpoint_name, - i_process_start_time, - i_process_end_time, - i_measurements, - i_measured_by_atum_agent, - i_by_user - ); + IF _fk_partitioning IS NULL THEN + status := 32; + status_text := 'Partitioning not found'; + RETURN; + END IF; + + SELECT WC.status, WC.status_text + FROM runs.write_checkpoint( + _fk_partitioning, + i_id_checkpoint, + i_checkpoint_name, + i_process_start_time, + i_process_end_time, + i_measurements, + i_measured_by_atum_agent, + i_by_user + ) WC + INTO status, status_text; - status := result.status; - status_text := result.status_text; RETURN; END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.write_checkpoint_v1(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.write_checkpoint_v1(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; +ALTER FUNCTION runs.write_checkpoint(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.write_checkpoint(JSONB, UUID, TEXT, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE, JSONB[], BOOLEAN, TEXT) TO atum_user; diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala index c816d25ac..51631bfaa 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointIntegrationTests.scala @@ -42,42 +42,6 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { |""".stripMargin ) - private val measurements = - """ - |{ - | "{ - | \"measure\": { - | \"measureName\": \"count\", - | \"measuredColumns\": [] - | }, - | \"result\":{ - | \"value\":\"3\", - | \"type\":\"int\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"col1\"] - | }, - | \"result\":{ - | \"value\":\"3.14\", - | \"type\":\"double\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"a\",\"b\"] - | }, - | \"result\":{ - | \"value\":\"2.71\", - | \"type\":\"double\" - | } - | }" - |} - |""".stripMargin - test("Write new checkpoint without data") { val uuid = UUID.randomUUID val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") @@ -101,7 +65,7 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) function(fnWriteCheckpoint) - .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Empty path") .setParam("i_process_start_time", startTime) @@ -136,7 +100,41 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { val user = "Franz Kafka" val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") - + val measurements = + """ + |{ + | "{ + | \"measure\": { + | \"measureName\": \"count\", + | \"measuredColumns\": [] + | }, + | \"result\":{ + | \"value\":\"3\", + | \"type\":\"int\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"col1\"] + | }, + | \"result\":{ + | \"value\":\"3.14\", + | \"type\":\"double\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"a\",\"b\"] + | }, + | \"result\":{ + | \"value\":\"2.71\", + | \"type\":\"double\" + | } + | }" + |} + |""".stripMargin table("runs.partitionings").insert( add("partitioning", partitioning) @@ -155,7 +153,7 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) function(fnWriteCheckpoint) - .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Happy path") .setParam("i_process_start_time", startTime) @@ -242,7 +240,7 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { ) function(fnWriteCheckpoint) - .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -269,7 +267,7 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() function(fnWriteCheckpoint) - .setParam("i_partitioning_id", 0L) + .setParam("i_partitioning", partitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -285,5 +283,4 @@ class WriteCheckpointIntegrationTests extends DBTestSuite { } assert(table("runs.checkpoints").count() == count) } - } diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV1IntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointOverloadedIntegrationTests.scala similarity index 88% rename from database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV1IntegrationTests.scala rename to database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointOverloadedIntegrationTests.scala index 6d26748b0..15f339ce3 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointV1IntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/WriteCheckpointOverloadedIntegrationTests.scala @@ -23,9 +23,9 @@ import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime import java.util.UUID -class WriteCheckpointV1IntegrationTests extends DBTestSuite { +class WriteCheckpointOverloadedIntegrationTests extends DBTestSuite { - private val fnWriteCheckpointV1 = "runs.write_checkpoint_v1" + private val fnWriteCheckpoint = "runs.write_checkpoint" private val partitioning = JsonBString( """ @@ -42,6 +42,42 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { |""".stripMargin ) + private val measurements = + """ + |{ + | "{ + | \"measure\": { + | \"measureName\": \"count\", + | \"measuredColumns\": [] + | }, + | \"result\":{ + | \"value\":\"3\", + | \"type\":\"int\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"col1\"] + | }, + | \"result\":{ + | \"value\":\"3.14\", + | \"type\":\"double\" + | } + | }", + | "{ + | \"measure\": { + | \"measureName\": \"avg\", + | \"measuredColumns\": [\"a\",\"b\"] + | }, + | \"result\":{ + | \"value\":\"2.71\", + | \"type\":\"double\" + | } + | }" + |} + |""".stripMargin + test("Write new checkpoint without data") { val uuid = UUID.randomUUID val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") @@ -64,8 +100,8 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) - function(fnWriteCheckpointV1) - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Empty path") .setParam("i_process_start_time", startTime) @@ -100,41 +136,7 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { val user = "Franz Kafka" val startTime = OffsetDateTime.parse("1992-08-03T10:00:00Z") val endTime = OffsetDateTime.parse("2022-11-05T08:00:00Z") - val measurements = - """ - |{ - | "{ - | \"measure\": { - | \"measureName\": \"count\", - | \"measuredColumns\": [] - | }, - | \"result\":{ - | \"value\":\"3\", - | \"type\":\"int\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"col1\"] - | }, - | \"result\":{ - | \"value\":\"3.14\", - | \"type\":\"double\" - | } - | }", - | "{ - | \"measure\": { - | \"measureName\": \"avg\", - | \"measuredColumns\": [\"a\",\"b\"] - | }, - | \"result\":{ - | \"value\":\"2.71\", - | \"type\":\"double\" - | } - | }" - |} - |""".stripMargin + table("runs.partitionings").insert( add("partitioning", partitioning) @@ -152,8 +154,8 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { assert(table("runs.checkpoints").count(add("fk_partitioning", fkPartitioning)) == 0) - function(fnWriteCheckpointV1) - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Happy path") .setParam("i_process_start_time", startTime) @@ -239,8 +241,8 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { .add("created_by", origAuthor) ) - function(fnWriteCheckpointV1) - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", fkPartitioning) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -266,8 +268,8 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { test("Partitioning of the checkpoint does not exist") { val uuid = UUID.randomUUID val count = table("runs.checkpoints").count() - function(fnWriteCheckpointV1) - .setParam("i_partitioning", partitioning) + function(fnWriteCheckpoint) + .setParam("i_partitioning_id", 0L) .setParam("i_id_checkpoint", uuid) .setParam("i_checkpoint_name", "Won't go in") .setParam("i_process_start_time", now()) @@ -283,4 +285,5 @@ class WriteCheckpointV1IntegrationTests extends DBTestSuite { } assert(table("runs.checkpoints").count() == count) } + } diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 146502c9b..5a07279b6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -55,8 +55,8 @@ object Main extends ZIOAppDefault with Server { GetPartitioningAdditionalData.layer, CreateOrUpdateAdditionalData.layer, GetPartitioningCheckpoints.layer, - WriteCheckpointV1.layer, WriteCheckpoint.layer, + WriteCheckpointV2.layer, GetFlowCheckpoints.layer, PostgresDatabaseProvider.layer, TransactorProvider.layer, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 3eb2ff5a2..ce6c5c129 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -17,6 +17,8 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator +import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus @@ -25,29 +27,28 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ import io.circe.syntax._ +import za.co.absa.atum.model.dto.MeasureResultDTO._ +import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ -import za.co.absa.atum.model.dto.CheckpointV2DTO -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[WriteCheckpointArgs, Unit, Task](args => + extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => Seq( - fr"${args.partitioningId}", - fr"${args.checkpointV2DTO.id}", - fr"${args.checkpointV2DTO.name}", - fr"${args.checkpointV2DTO.processStartTime}", - fr"${args.checkpointV2DTO.processEndTime}", - fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", - fr"${args.checkpointV2DTO.measuredByAtumAgent}", - fr"${args.checkpointV2DTO.author}" + fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", + fr"${values.id}", + fr"${values.name}", + fr"${values.processStartTime}", + fr"${values.processEndTime}", + fr"${values.measurements.toList.map(_.asJson)}", + fr"${values.measuredByAtumAgent}", + fr"${values.author}" ) ) with StandardStatusHandling object WriteCheckpoint { - case class WriteCheckpointArgs(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) - val layer: URLayer[PostgresDatabaseProvider, WriteCheckpoint] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala similarity index 58% rename from server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1.scala rename to server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala index 00570927c..ff797dd70 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2.scala @@ -17,8 +17,6 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.implicits.toSqlInterpolator -import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.db.fadb.DBSchema import za.co.absa.db.fadb.doobie.DoobieEngine import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatus @@ -27,31 +25,33 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ import io.circe.syntax._ -import za.co.absa.atum.model.dto.MeasureResultDTO._ -import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get -import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut import doobie.postgres.implicits._ +import za.co.absa.atum.model.dto.CheckpointV2DTO +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs -class WriteCheckpointV1(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task](values => +class WriteCheckpointV2(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) + extends DoobieSingleResultFunctionWithStatus[WriteCheckpointArgs, Unit, Task](args => Seq( - fr"${PartitioningForDB.fromSeqPartitionDTO(values.partitioning).asJson}", - fr"${values.id}", - fr"${values.name}", - fr"${values.processStartTime}", - fr"${values.processEndTime}", - fr"${values.measurements.toList.map(_.asJson)}", - fr"${values.measuredByAtumAgent}", - fr"${values.author}" - ) + fr"${args.partitioningId}", + fr"${args.checkpointV2DTO.id}", + fr"${args.checkpointV2DTO.name}", + fr"${args.checkpointV2DTO.processStartTime}", + fr"${args.checkpointV2DTO.processEndTime}", + fr"${args.checkpointV2DTO.measurements.toList.map(_.asJson)}", + fr"${args.checkpointV2DTO.measuredByAtumAgent}", + fr"${args.checkpointV2DTO.author}" + ), + Some("write_checkpoint") ) with StandardStatusHandling -object WriteCheckpointV1 { - val layer: URLayer[PostgresDatabaseProvider, WriteCheckpointV1] = ZLayer { +object WriteCheckpointV2 { + case class WriteCheckpointArgs(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO) + + val layer: URLayer[PostgresDatabaseProvider, WriteCheckpointV2] = ZLayer { for { dbProvider <- ZIO.service[PostgresDatabaseProvider] - } yield new WriteCheckpointV1()(Runs, dbProvider.dbEngine) + } yield new WriteCheckpointV2()(Runs, dbProvider.dbEngine) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index bfdf5aba9..0fd132a4d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -17,33 +17,33 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpointV2, WriteCheckpoint} import za.co.absa.atum.server.api.exception.DatabaseError import zio._ import zio.interop.catz.asyncInstance -class CheckpointRepositoryImpl(writeCheckpointV1Fn: WriteCheckpointV1, writeCheckpointFn: WriteCheckpoint) +class CheckpointRepositoryImpl(writeCheckpointFn: WriteCheckpoint, writeCheckpointV2Fn: WriteCheckpointV2) extends CheckpointRepository with BaseRepository { override def writeCheckpoint(checkpointDTO: CheckpointDTO): IO[DatabaseError, Unit] = { - dbSingleResultCallWithStatus(writeCheckpointV1Fn(checkpointDTO), "writeCheckpoint") + dbSingleResultCallWithStatus(writeCheckpointFn(checkpointDTO), "writeCheckpoint") } override def writeCheckpointV2(partitioningId: Long, checkpointV2DTO: CheckpointV2DTO): IO[DatabaseError, Unit] = { dbSingleResultCallWithStatus( - writeCheckpointFn(WriteCheckpointArgs(partitioningId, checkpointV2DTO)), + writeCheckpointV2Fn(WriteCheckpointArgs(partitioningId, checkpointV2DTO)), "writeCheckpoint" ) } } object CheckpointRepositoryImpl { - val layer: URLayer[WriteCheckpointV1 with WriteCheckpoint, CheckpointRepository] = ZLayer { + val layer: URLayer[WriteCheckpoint with WriteCheckpointV2, CheckpointRepository] = ZLayer { for { - writeCheckpointV1 <- ZIO.service[WriteCheckpointV1] writeCheckpoint <- ZIO.service[WriteCheckpoint] - } yield new CheckpointRepositoryImpl(writeCheckpointV1, writeCheckpoint) + writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] + } yield new CheckpointRepositoryImpl(writeCheckpoint, writeCheckpointV2) } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 39662ac48..b5a93234f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -17,14 +17,13 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto._ +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs -import za.co.absa.db.fadb.exceptions.DataConflictException -import za.co.absa.db.fadb.status.{FunctionStatus, Row} +import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} +import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance import zio.test._ @@ -36,12 +35,13 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("WriteCheckpointV2Suite")( + suite("WriteCheckpointSuite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { - val checkpointV2DTO = CheckpointV2DTO( + val checkpointDTO = CheckpointDTO( id = UUID.randomUUID(), name = "name", author = "author", + partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = Set( @@ -49,8 +49,8 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ) ) for { - writeCheckpointV2 <- ZIO.service[WriteCheckpoint] - result <- writeCheckpointV2(WriteCheckpointArgs(1L, checkpointV2DTO)) + writeCheckpoint <- ZIO.service[WriteCheckpoint] + result <- writeCheckpoint(checkpointDTO) } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala similarity index 76% rename from server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala rename to server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala index 988074be9..87a2ec89a 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV1IntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointV2IntegrationTests.scala @@ -17,13 +17,14 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} +import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider -import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} -import za.co.absa.db.fadb.status.FunctionStatus +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs +import za.co.absa.db.fadb.exceptions.DataConflictException +import za.co.absa.db.fadb.status.{FunctionStatus, Row} import zio._ import zio.interop.catz.asyncInstance import zio.test._ @@ -31,17 +32,16 @@ import zio.test._ import java.time.ZonedDateTime import java.util.UUID -object WriteCheckpointV1IntegrationTests extends ConfigProviderTest { +object WriteCheckpointV2IntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("WriteCheckpointSuite")( + suite("WriteCheckpointV2Suite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { - val checkpointDTO = CheckpointDTO( + val checkpointV2DTO = CheckpointV2DTO( id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = Set( @@ -49,12 +49,12 @@ object WriteCheckpointV1IntegrationTests extends ConfigProviderTest { ) ) for { - writeCheckpoint <- ZIO.service[WriteCheckpointV1] - result <- writeCheckpoint(checkpointDTO) + writeCheckpointV2 <- ZIO.service[WriteCheckpointV2] + result <- writeCheckpointV2(WriteCheckpointArgs(1L, checkpointV2DTO)) } yield assertTrue(result == Left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) } ).provide( - WriteCheckpointV1.layer, + WriteCheckpointV2.layer, PostgresDatabaseProvider.layer, TestTransactorProvider.layerWithRollback ) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index bac3a078b..407f5607b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,10 +17,10 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} +import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpointV2, WriteCheckpoint} import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs import za.co.absa.atum.server.api.exception.DatabaseError._ import za.co.absa.db.fadb.exceptions.DataConflictException import za.co.absa.db.fadb.status.FunctionStatus @@ -32,25 +32,25 @@ import za.co.absa.db.fadb.status.Row object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { - private val writeCheckpointV1Mock: WriteCheckpointV1 = mock(classOf[WriteCheckpointV1]) private val writeCheckpointMock: WriteCheckpoint = mock(classOf[WriteCheckpoint]) + private val writeCheckpointV2Mock: WriteCheckpointV2 = mock(classOf[WriteCheckpointV2]) - when(writeCheckpointV1Mock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) - when(writeCheckpointV1Mock.apply(checkpointDTO2)) + when(writeCheckpointMock.apply(checkpointDTO1)).thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) + when(writeCheckpointMock.apply(checkpointDTO2)) .thenReturn(ZIO.fail(DataConflictException(FunctionStatus(31, "conflict")))) - when(writeCheckpointV1Mock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) + when(writeCheckpointMock.apply(checkpointDTO3)).thenReturn(ZIO.fail(new Exception("boom!"))) private val partitioningId = 1L - when(writeCheckpointMock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO1))) + when(writeCheckpointV2Mock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO1))) .thenReturn(ZIO.right(Row(FunctionStatus(0, "success"), ()))) - when(writeCheckpointMock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO2))) + when(writeCheckpointV2Mock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO2))) .thenReturn(ZIO.left(DataConflictException(FunctionStatus(32, "Partitioning not found")))) - when(writeCheckpointMock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO3))) + when(writeCheckpointV2Mock.apply(WriteCheckpointArgs(partitioningId, checkpointV2DTO3))) .thenReturn(ZIO.fail(new Exception("boom!"))) - private val writeCheckpointV1MockLayer = ZLayer.succeed(writeCheckpointV1Mock) private val writeCheckpointMockLayer = ZLayer.succeed(writeCheckpointMock) + private val writeCheckpointV2MockLayer = ZLayer.succeed(writeCheckpointV2Mock) override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -87,7 +87,7 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { ) } ) - ).provide(CheckpointRepositoryImpl.layer, writeCheckpointV1MockLayer, writeCheckpointMockLayer) + ).provide(CheckpointRepositoryImpl.layer, writeCheckpointMockLayer, writeCheckpointV2MockLayer) } From 5c21538cb7c2c19bfc77714e77511fa5d8be63fb Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 20 Aug 2024 13:19:49 +0200 Subject: [PATCH 50/50] conflicts resolved --- .../server/api/repository/CheckpointRepositoryImpl.scala | 2 +- .../api/repository/CheckpointRepositoryUnitTests.scala | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala index b884de4ba..4972d7e0b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.repository import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.server.api.database.runs.functions._ import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.exception.DatabaseError.GeneralDatabaseError import za.co.absa.atum.server.model.CheckpointItemFromDB diff --git a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala index f27c57c96..687bd628b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/repository/CheckpointRepositoryUnitTests.scala @@ -17,12 +17,13 @@ package za.co.absa.atum.server.api.repository import org.mockito.Mockito.{mock, when} -import za.co.absa.atum.server.api.database.runs.functions.{WriteCheckpoint, WriteCheckpointV1} +import za.co.absa.atum.server.api.database.runs.functions._ import za.co.absa.atum.server.api.exception.DatabaseError import za.co.absa.atum.server.api.TestData -import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpoint.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpointV2.GetPartitioningCheckpointV2Args +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs import za.co.absa.atum.server.api.exception.DatabaseError._ -import za.co.absa.db.fadb.exceptions.DataConflictException +import za.co.absa.db.fadb.exceptions.{DataConflictException, DataNotFoundException} import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.interop.catz.asyncInstance