From f5e679574fd9f7b2199b217c501012a8aeeb1bc0 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 14 Jun 2024 08:35:19 +0200 Subject: [PATCH 01/34] Addid circe dependency and using it --- project/Dependencies.scala | 13 +++ .../CreateOrUpdateAdditionalData.scala | 8 +- .../CreatePartitioningIfNotExists.scala | 14 +-- .../runs/functions/WriteCheckpoint.scala | 2 +- .../absa/atum/server/api/http/Endpoints.scala | 2 +- .../server/model/CirceJsonImplicits.scala | 73 ++++++++++++++ .../atum/server/model/ErrorResponse.scala | 50 ++++++++-- .../atum/server/model/PartitioningForDB.scala | 34 +++++-- .../atum/server/model/PlayJsonImplicits.scala | 96 ------------------- ...ateCheckpointEndpointIntegrationTest.scala | 2 +- ...PartitioningEndpointIntegrationTests.scala | 2 +- 11 files changed, 172 insertions(+), 124 deletions(-) create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4bb377157..9dadfcbf0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -74,6 +74,7 @@ object Dependencies { val http4sBlazeBackend = "0.23.15" val http4sPrometheus = "0.23.6" val playJson = "3.0.1" + val circeJson = "0.14.7" val sttpPlayJson = "3.9.3" val awssdk = "2.23.15" @@ -115,6 +116,8 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided + lazy val sttp = "com.softwaremill.sttp.client3" %% "core" % Versions.sttp + Seq( jacksonModuleScala, json4sExt, @@ -161,6 +164,12 @@ object Dependencies { lazy val playJson = playOrg %% "play-json" % Versions.playJson lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test + lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson + lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson + lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson + lazy val circeGenericExtras = "io.circe" %% "circe-generic-extras" % Versions.circeJson + lazy val circeSttp = "com.softwaremill.sttp.client3" %% "circe" % Versions.sttp + // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb @@ -191,7 +200,11 @@ object Dependencies { tapirPlayJson, tapirPrometheus, tapirStubServer, + circeGenericExtras, playJson, + circeCore, + circeGeneric, + circeParser, sttpPlayJson, awsSecretsManagerSdk, zioTest, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 563d8ace3..0b0df1b8b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -16,10 +16,12 @@ package za.co.absa.atum.server.api.database.runs.functions + import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read -import play.api.libs.json.Json +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.atum.model.dto.AdditionalDataSubmitDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs @@ -34,12 +36,12 @@ import zio.interop.catz._ import doobie.postgres.implicits._ class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task] with StandardStatusHandling { override def sql(values: AdditionalDataSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = Json.toJson(partitioning).toString + val partitioningJsonString = partitioning.asJson.noSpaces // implicits from Doobie can't handle Map[String, Option[String]] -> HStore, so we converted None to null basically val additionalDataNormalized = values.additionalData.map{ case (k, v) => (k, v.orNull)} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 8c6135476..82aa9d056 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -16,10 +16,12 @@ package za.co.absa.atum.server.api.database.runs.functions + import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read -import play.api.libs.json.Json +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.atum.model.dto.PartitioningSubmitDTO import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.fadb.DBSchema @@ -32,16 +34,16 @@ import zio._ import zio.interop.catz._ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] with StandardStatusHandling { override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = Json.toJson(partitioning).toString + val partitioningJsonString = partitioning.asJson.noSpaces val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - Json.toJson(parentPartitioningForDB).toString + parentPartitioningForDB.asJson.noSpaces } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( @@ -49,8 +51,8 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningJsonString }, - ${values.authorIfNew}, - ${ + ${values.authorIfNew}, + ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString parentPartitioningJsonString } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 35b615dad..25e45c9c4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -30,7 +30,7 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import play.api.libs.json.Json -import za.co.absa.atum.server.model.PlayJsonImplicits.writesMeasurementDTO +import za.co.absa.atum.server.model.CirceJsonImplicit._ import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 7babe13dc..25d38baa0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -24,7 +24,7 @@ import sttp.tapir.ztapir._ import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO, AdditionalDataSubmitDTO} import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse -import za.co.absa.atum.server.model.PlayJsonImplicits._ +import za.co.absa.atum.server.model.CirceJsonImplicits._ trait Endpoints extends BaseEndpoints { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala new file mode 100644 index 000000000..450e2e56c --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.model + +import io.circe._, io.circe.generic.semiauto._ +import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import za.co.absa.atum.model.dto._ + +object CirceJsonImplicits { + + implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] + implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] + + implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { + case "String" => Right(ResultValueType.String) + case "Long" => Right(ResultValueType.Long) + case "BigDecimal" => Right(ResultValueType.BigDecimal) + case "Double" => Right(ResultValueType.Double) + case _ => Left("Invalid ResultValueType") + } + + implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap[ResultValueType] { + case ResultValueType.String => "String" + case ResultValueType.Long => "Long" + case ResultValueType.BigDecimal => "BigDecimal" + case ResultValueType.Double => "Double" + } + + implicit val decodeTypedValue: Decoder[MeasureResultDTO.TypedValue] = deriveDecoder + implicit val encodeTypedValue: Encoder[MeasureResultDTO.TypedValue] = deriveEncoder + + implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = deriveDecoder + implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = deriveEncoder + + implicit val decodeMeasureDTO: Decoder[MeasureDTO] = deriveDecoder + implicit val encodeMeasureDTO: Encoder[MeasureDTO] = deriveEncoder + + implicit val decodeMeasurementDTO: Decoder[MeasurementDTO] = deriveDecoder + implicit val encodeMeasurementDTO: Encoder[MeasurementDTO] = deriveEncoder + + implicit val decodePartitionDTO: Decoder[PartitionDTO] = deriveDecoder + implicit val encodePartitionDTO: Encoder[PartitionDTO] = deriveEncoder + + implicit val decodeCheckpointDTO: Decoder[CheckpointDTO] = deriveDecoder + implicit val encodeCheckpointDTO: Encoder[CheckpointDTO] = deriveEncoder + + implicit val decodePartitioningSubmitDTO: Decoder[PartitioningSubmitDTO] = deriveDecoder + implicit val encodePartitioningSubmitDTO: Encoder[PartitioningSubmitDTO] = deriveEncoder + + implicit val decodeStringMap: Decoder[Map[String, Option[String]]] = Decoder.decodeMap[String, Option[String]] + implicit val encodeStringMap: Encoder[Map[String, Option[String]]] = Encoder.encodeMap[String, Option[String]] + + implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder + implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder + + implicit val decodeAtumContextDTO: Decoder[AtumContextDTO] = deriveDecoder + implicit val encodeAtumContextDTO: Encoder[AtumContextDTO] = deriveEncoder + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index 2fbc0c697..c1af90ad1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -16,34 +16,66 @@ package za.co.absa.atum.server.model -import play.api.libs.json.{Json, Reads, Writes} +//import play.api.libs.json.{Json, Reads, Writes} +// +//sealed trait ErrorResponse { +// def message: String +//} +// +//object ErrorResponse { +// implicit val reads: Reads[ErrorResponse] = Json.reads[ErrorResponse] +// implicit val writes: Writes[ErrorResponse] = Json.writes[ErrorResponse] +//} +// +//final case class BadRequestResponse(message: String) extends ErrorResponse +// +//object BadRequestResponse { +// implicit val reads: Reads[BadRequestResponse] = Json.reads[BadRequestResponse] +// implicit val writes: Writes[BadRequestResponse] = Json.writes[BadRequestResponse] +//} +// +//final case class GeneralErrorResponse(message: String) extends ErrorResponse +// +//object GeneralErrorResponse { +// implicit val reads: Reads[GeneralErrorResponse] = Json.reads[GeneralErrorResponse] +// implicit val writes: Writes[GeneralErrorResponse] = Json.writes[GeneralErrorResponse] +//} +// +//final case class InternalServerErrorResponse(message: String) extends ErrorResponse +// +//object InternalServerErrorResponse { +// implicit val reads: Reads[InternalServerErrorResponse] = Json.reads[InternalServerErrorResponse] +// implicit val writes: Writes[InternalServerErrorResponse] = Json.writes[InternalServerErrorResponse] +//} + +import io.circe._, io.circe.generic.semiauto._ sealed trait ErrorResponse { def message: String } object ErrorResponse { - implicit val reads: Reads[ErrorResponse] = Json.reads[ErrorResponse] - implicit val writes: Writes[ErrorResponse] = Json.writes[ErrorResponse] + implicit val decodeErrorResponse: Decoder[ErrorResponse] = deriveDecoder + implicit val encodeErrorResponse: Encoder[ErrorResponse] = deriveEncoder } final case class BadRequestResponse(message: String) extends ErrorResponse object BadRequestResponse { - implicit val reads: Reads[BadRequestResponse] = Json.reads[BadRequestResponse] - implicit val writes: Writes[BadRequestResponse] = Json.writes[BadRequestResponse] + implicit val decodeBadRequestResponse: Decoder[BadRequestResponse] = deriveDecoder + implicit val encodeBadRequestResponse: Encoder[BadRequestResponse] = deriveEncoder } final case class GeneralErrorResponse(message: String) extends ErrorResponse object GeneralErrorResponse { - implicit val reads: Reads[GeneralErrorResponse] = Json.reads[GeneralErrorResponse] - implicit val writes: Writes[GeneralErrorResponse] = Json.writes[GeneralErrorResponse] + implicit val decodeGeneralErrorResponse: Decoder[GeneralErrorResponse] = deriveDecoder + implicit val encodeGeneralErrorResponse: Encoder[GeneralErrorResponse] = deriveEncoder } final case class InternalServerErrorResponse(message: String) extends ErrorResponse object InternalServerErrorResponse { - implicit val reads: Reads[InternalServerErrorResponse] = Json.reads[InternalServerErrorResponse] - implicit val writes: Writes[InternalServerErrorResponse] = Json.writes[InternalServerErrorResponse] + implicit val decodeInternalServerErrorResponse: Decoder[InternalServerErrorResponse] = deriveDecoder + implicit val encodeInternalServerErrorResponse: Encoder[InternalServerErrorResponse] = deriveEncoder } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index 55a349923..bb5a14006 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,14 +16,36 @@ package za.co.absa.atum.server.model -import play.api.libs.json.{Json, Writes} +//import play.api.libs.json.{Json, Writes} +//import za.co.absa.atum.model.dto.PartitioningDTO +// +//private[server] case class PartitioningForDB private ( +// version: Int = 1, +// keys: Seq[String], +// keysToValues: Map[String, String] +//) +// +//object PartitioningForDB { +// +// def fromSeqPartitionDTO(partitioning: PartitioningDTO): PartitioningForDB = { +// val allKeys = partitioning.map(_.key) +// val mapOfKeysAndValues = partitioning.map(p => p.key -> p.value).toMap[String, String] +// +// PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) +// } +// +// implicit val writes: Writes[PartitioningForDB] = Json.writes +// +//} + +import io.circe._, io.circe.generic.semiauto._ import za.co.absa.atum.model.dto.PartitioningDTO private[server] case class PartitioningForDB private ( - version: Int = 1, - keys: Seq[String], - keysToValues: Map[String, String] -) + version: Int = 1, + keys: Seq[String], + keysToValues: Map[String, String] + ) object PartitioningForDB { @@ -34,6 +56,6 @@ object PartitioningForDB { PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) } - implicit val writes: Writes[PartitioningForDB] = Json.writes + implicit val encodePartitioningForDB: Encoder[PartitioningForDB] = deriveEncoder } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala deleted file mode 100644 index 03a3776df..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import play.api.libs.functional.syntax.toFunctionalBuilderOps -import play.api.libs.json._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} -import za.co.absa.atum.model.dto._ - -object PlayJsonImplicits { - - implicit val optionStringReads: Reads[Option[String]] = new Reads[Option[String]] { - def reads(json: JsValue): JsResult[Option[String]] = json match { - case JsNull => JsSuccess(None) - case JsString(s) => JsSuccess(Some(s)) - case _ => JsError("Expected JsString or JsNull") - } - } - - implicit val optionStringWrites: Writes[Option[String]] = new Writes[Option[String]] { - def writes(opt: Option[String]): JsValue = opt match { - case Some(s) => JsString(s) - case None => JsNull - } - } - - implicit val resultValueTypeReads: Reads[ResultValueType] = new Reads[ResultValueType] { - override def reads(json: JsValue): JsResult[ResultValueType] = json match { - case JsString("String") => JsSuccess(ResultValueType.String) - case JsString("Long") => JsSuccess(ResultValueType.Long) - case JsString("BigDecimal") => JsSuccess(ResultValueType.BigDecimal) - case JsString("Double") => JsSuccess(ResultValueType.Double) - case _ => JsError("Invalid ResultValueType") - } - } - - implicit val resultValueTypeWrites: Writes[ResultValueType] = new Writes[ResultValueType] { - def writes(resultValueType: ResultValueType): JsValue = resultValueType match { - case ResultValueType.String => Json.toJson("String") - case ResultValueType.Long => Json.toJson("Long") - case ResultValueType.BigDecimal => Json.toJson("BigDecimal") - case ResultValueType.Double => Json.toJson("Double") - } - } - - implicit val readsTypedValue: Reads[MeasureResultDTO.TypedValue] = Json.reads[MeasureResultDTO.TypedValue] - implicit val writesTypedValue: Writes[MeasureResultDTO.TypedValue] = Json.writes[MeasureResultDTO.TypedValue] - - implicit val readsMeasureResultDTO: Reads[MeasureResultDTO] = { - ((__ \ "mainValue").read[MeasureResultDTO.TypedValue] and - (__ \ "supportValues").readNullable[Map[String, TypedValue]].map(_.getOrElse(Map.empty)) - )(MeasureResultDTO.apply _) - } - - implicit val writesMeasureResultDTO: Writes[MeasureResultDTO] = Json.writes[MeasureResultDTO] - - implicit val readsMeasureDTO: Reads[MeasureDTO] = Json.reads[MeasureDTO] - implicit val writesMeasureDTO: Writes[MeasureDTO] = Json.writes[MeasureDTO] - - implicit val readsMeasurementDTO: Reads[MeasurementDTO] = Json.reads[MeasurementDTO] - implicit val writesMeasurementDTO: Writes[MeasurementDTO] = Json.writes[MeasurementDTO] - - implicit val readsPartitionDTO: Reads[PartitionDTO] = Json.reads[PartitionDTO] - implicit val writesPartitionDTO: Writes[PartitionDTO] = Json.writes[PartitionDTO] - - implicit val readsCheckpointDTO: Reads[CheckpointDTO] = Json.reads[CheckpointDTO] - implicit val writesCheckpointDTO: Writes[CheckpointDTO] = Json.writes[CheckpointDTO] - - implicit val readsPartitioningSubmitDTO: Reads[PartitioningSubmitDTO] = Json.reads[PartitioningSubmitDTO] - implicit val writesPartitioningSubmitDTO: Writes[PartitioningSubmitDTO] = Json.writes[PartitioningSubmitDTO] - - implicit val readsStringMap: Reads[Map[String, Option[String]]] = Reads.mapReads[Option[String]] - implicit val writesStringMap: OWrites[MapWrites.Map[String, Option[String]]] = - Writes.genericMapWrites[Option[String], MapWrites.Map] - - implicit val readsAdditionalDataSubmitDTO: Reads[AdditionalDataSubmitDTO] = Json.reads[AdditionalDataSubmitDTO] - implicit val writesAdditionalDataSubmitDTO: Writes[AdditionalDataSubmitDTO] = Json.writes[AdditionalDataSubmitDTO] - - implicit val readsAtumContextDTO: Reads[AtumContextDTO] = Json.reads[AtumContextDTO] - implicit val writesAtumContextDTO: Writes[AtumContextDTO] = Json.writes[AtumContextDTO] - -} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala index c53a2db82..5cf59d286 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala @@ -29,7 +29,7 @@ import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} import zio.test._ import zio._ -import za.co.absa.atum.server.model.PlayJsonImplicits.{readsCheckpointDTO, writesCheckpointDTO} +import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeCheckpointDTO, encodeCheckpointDTO} import zio.test.Assertion.equalTo object CreateCheckpointEndpointIntegrationTests extends ZIOSpecDefault with Endpoints with TestData { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala index b7a3de799..923c3284c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala @@ -30,7 +30,7 @@ import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorRe import zio.test.Assertion.equalTo import zio._ import zio.test._ -import za.co.absa.atum.server.model.PlayJsonImplicits.{readsAtumContextDTO, writesPartitioningSubmitDTO} +import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} object CreatePartitioningEndpointIntegrationTests extends ZIOSpecDefault with Endpoints with TestData { From ac908df2adf085e6b3ecffb296a5a4799367ff27 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 17 Jun 2024 13:18:07 +0200 Subject: [PATCH 02/34] modifying dependencies --- project/Dependencies.scala | 5 +++ .../absa/atum/server/api/http/Endpoints.scala | 7 ++-- ...PartitioningEndpointIntegrationTests.scala | 34 ++++++++++++++----- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6f952da64..d53aad28c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -76,6 +76,7 @@ object Dependencies { val http4sPrometheus = "0.23.6" val playJson = "3.0.1" val sttpPlayJson = "3.9.3" + val sttpCirceJson = "4.0.0-M16" val awssdk = "2.23.15" @@ -140,6 +141,7 @@ object Dependencies { val logbackOrg = "ch.qos.logback" val awsSdkOrg = "software.amazon.awssdk" val sttpClient3Org = "com.softwaremill.sttp.client3" + val sttpClient4Org = "com.softwaremill.sttp.client4" // zio lazy val zioCore = zioOrg %% "zio" % Versions.zio @@ -166,6 +168,8 @@ object Dependencies { // json lazy val playJson = playOrg %% "play-json" % Versions.playJson lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test + lazy val sttpCirceJson = sttpClient4Org %% "circe" % Versions.sttpCirceJson % Test + // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb @@ -201,6 +205,7 @@ object Dependencies { tapirStubServer, playJson, sttpPlayJson, + sttpCirceJson, awsSecretsManagerSdk, zioTest, zioTestSbt, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 4c63adbf8..b98f0e374 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -21,14 +21,13 @@ import sttp.tapir.{PublicEndpoint, endpoint} import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.play.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{PublicEndpoint, endpoint} import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse -//import za.co.absa.atum.server.model.PlayJsonImplicits._ -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.ErrorResponse +import za.co.absa.atum.server.model.PlayJsonImplicits._ +import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.CirceJsonImplicits._ +import sttp.tapir.{PublicEndpoint, endpoint} trait Endpoints extends BaseEndpoints { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala index 4a9c1824a..f1e15a501 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala @@ -17,22 +17,33 @@ package za.co.absa.atum.server.api.http import org.mockito.Mockito.{mock, when} -import sttp.client3._ -import sttp.client3.playJson._ -import sttp.client3.testing.SttpBackendStub +//import sttp.client3._ +//import sttp.client3.playJson._ +//import sttp.client3.testing.SttpBackendStub +import sttp.client4._ +import sttp.client4.circe._ +import io.circe.generic.auto._ +import sttp.client4.testing._ +//import scala.concurrent.Future +//import scala.concurrent.ExecutionContext.Implicits.global +//import sttp.client4._ +//import sttp.client4.testing._ +//import sttp.client4.circe._ +//import io.circe.generic.auto._ + import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} //import za.co.absa.atum.server.model.PlayJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo import zio.test._ -import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} +//import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} object CreatePartitioningEndpointIntegrationTests extends ZIOSpecDefault with Endpoints with TestData { @@ -51,10 +62,15 @@ object CreatePartitioningEndpointIntegrationTests extends ZIOSpecDefault with En createPartitioningEndpointV2.zServerLogic(PartitioningController.createPartitioningIfNotExistsV2) def spec: Spec[TestEnvironment with Scope, Any] = { - val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[PartitioningController])) - .whenServerEndpoint(createPartitioningServerEndpoint) - .thenRunLogic() - .backend() + val backendStub = SttpBackendStub(new RIOMonadError[PartitioningController]) + .whenRequestMatches(_ => true) + .thenRespondWrapped( + createPartitioningServerEndpoint.logic(_) + .foldM( + e => ZIO.succeed(Response(e, StatusCode.BadRequest)), + s => ZIO.succeed(Response.ok(s)) + ) + ) val request = basicRequest .post(uri"https://test.com/api/v2/createPartitioning") From 852d4fd6b5ac8403022779321a530ca198e0328b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 18 Jun 2024 10:57:42 +0200 Subject: [PATCH 03/34] using circe syntax in flow classes --- .../api/database/flows/functions/GetFlowCheckpoints.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 0b30ee8d3..b8102987e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -19,7 +19,6 @@ package za.co.absa.atum.server.api.database.flows.functions import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read -import play.api.libs.json.Json import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.flows.Flows @@ -51,7 +50,7 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] override def sql(values: CheckpointQueryDTO)(implicit read: Read[CheckpointFromDB]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = Json.toJson(partitioning).toString + val partitioningNormalized = partitioning.asJson.noSpaces sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( From 37b54348bdbd83dee95268608102911341b65d72 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 18 Jun 2024 11:24:51 +0200 Subject: [PATCH 04/34] using circe in api http module --- .../za/co/absa/atum/server/api/http/BaseEndpoints.scala | 4 ++-- .../scala/za/co/absa/atum/server/api/http/Endpoints.scala | 6 +++--- .../za/co/absa/atum/server/api/http/ServerOptions.scala | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index 4f067fbc5..422f044ef 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -18,12 +18,12 @@ package za.co.absa.atum.server.api.http import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass -import sttp.tapir.json.play.jsonBody +import sttp.tapir.json.circe.jsonBody +import za.co.absa.atum.server.model.{BadRequestResponse, ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import sttp.tapir.typelevel.MatchType import sttp.tapir.ztapir._ import sttp.tapir.{EndpointOutput, PublicEndpoint} import za.co.absa.atum.server.Constants.Endpoints.{Api, V1, V2} -import za.co.absa.atum.server.model.ErrorResponse._ import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index e7381f0bd..d7cd1dc1f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -16,15 +16,15 @@ package za.co.absa.atum.server.api.http + +import io.circe.generic.auto.{exportDecoder, exportEncoder} import sttp.model.StatusCode -import sttp.tapir.{PublicEndpoint, endpoint} import sttp.tapir.generic.auto.schemaForCaseClass -import sttp.tapir.json.play.jsonBody import sttp.tapir.ztapir._ +import sttp.tapir.json.circe.jsonBody import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse -import za.co.absa.atum.server.model.PlayJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.CirceJsonImplicits._ import sttp.tapir.{PublicEndpoint, endpoint} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala index f6bbe79cd..c42f6343d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala @@ -18,7 +18,6 @@ package za.co.absa.atum.server.api.http import sttp.monad.MonadError import sttp.tapir.DecodeResult -import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.play.jsonBody import sttp.tapir.server.http4s.Http4sServerOptions import sttp.tapir.server.interceptor.DecodeFailureContext @@ -27,7 +26,7 @@ import sttp.tapir.server.interceptor.decodefailure.DefaultDecodeFailureHandler.r import sttp.tapir.server.interceptor.metrics.MetricsRequestInterceptor import sttp.tapir.server.model.ValuedEndpointOutput import sttp.tapir.ztapir.{headers, statusCode} -import za.co.absa.atum.server.model.ErrorResponse.BadRequestResponse +import za.co.absa.atum.server.model.BadRequestResponse import zio.interop.catz._ trait ServerOptions { From eb77ba5326e994a4a9821ffbd5dcab80f8d6b84f Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 18 Jun 2024 11:59:25 +0200 Subject: [PATCH 05/34] fix implicit import in ServerOptions --- .../scala/za/co/absa/atum/server/api/http/ServerOptions.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala index c42f6343d..1eefa98c1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala @@ -18,7 +18,8 @@ package za.co.absa.atum.server.api.http import sttp.monad.MonadError import sttp.tapir.DecodeResult -import sttp.tapir.json.play.jsonBody +import sttp.tapir.generic.auto.schemaForCaseClass +import sttp.tapir.json.circe.jsonBody import sttp.tapir.server.http4s.Http4sServerOptions import sttp.tapir.server.interceptor.DecodeFailureContext import sttp.tapir.server.interceptor.decodefailure.DecodeFailureHandler From f6d2ece553c6a8f777bf8f9e3bfda27166531c5d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jun 2024 13:23:04 +0200 Subject: [PATCH 06/34] merge master, fix dependencies and conclude server implicit implementation --- project/Dependencies.scala | 28 +++++++++++---- .../api/controller/BaseController.scala | 2 +- .../api/controller/CheckpointController.scala | 2 +- .../controller/CheckpointControllerImpl.scala | 2 +- .../api/controller/FlowController.scala | 2 +- .../api/controller/FlowControllerImpl.scala | 2 +- .../controller/PartitioningController.scala | 2 +- .../PartitioningControllerImpl.scala | 2 +- .../CreateOrUpdateAdditionalData.scala | 1 - .../CreatePartitioningIfNotExists.scala | 1 - .../GetPartitioningAdditionalData.scala | 1 - .../GetPartitioningCheckpoints.scala | 1 - .../functions/GetPartitioningMeasures.scala | 1 - .../absa/atum/server/api/http/Endpoints.scala | 23 +++++------- .../server/model/CirceJsonImplicits.scala | 5 +-- .../atum/server/model/ErrorResponse.scala | 31 ---------------- .../atum/server/model/PartitioningForDB.scala | 35 +++++-------------- .../CheckpointControllerUnitTests.scala | 2 +- .../controller/FlowControllerUnitTests.scala | 2 +- .../PartitioningControllerUnitTests.scala | 4 +-- .../CreateCheckpointEndpointUnitTests.scala | 21 +++++------ .../CreatePartitioningEndpointUnitTests.scala | 19 ++++------ .../GetFlowCheckpointsEndpointUnitTests.scala | 8 ++--- 23 files changed, 72 insertions(+), 125 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 50e530e18..29769d1c0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,6 @@ object Dependencies { val balta = "0.1.0" val jacksonModuleScala = "2.14.2" - val circeVersion = "0.14.5" val specs2 = "4.10.0" val typesafeConfig = "1.4.2" @@ -76,7 +75,8 @@ object Dependencies { val http4sPrometheus = "0.23.6" val playJson = "3.0.1" val sttpPlayJson = "3.9.3" - val sttpCirceJson = "4.0.0-M16" + val circeJson = "0.14.7" + val sttpCirceJson = "3.9.7" val awssdk = "2.23.15" @@ -117,8 +117,10 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided - lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeVersion - lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeVersion + // Circe dependencies + lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson + lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson + lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson Seq( jacksonModuleScala, @@ -128,6 +130,7 @@ object Dependencies { json4sNative, circeCore, circeParser, + circeGeneric, ) } @@ -141,7 +144,6 @@ object Dependencies { val logbackOrg = "ch.qos.logback" val awsSdkOrg = "software.amazon.awssdk" val sttpClient3Org = "com.softwaremill.sttp.client3" - val sttpClient4Org = "com.softwaremill.sttp.client4" // zio lazy val zioCore = zioOrg %% "zio" % Versions.zio @@ -165,11 +167,21 @@ object Dependencies { lazy val tapirPrometheus = tapirOrg %% "tapir-prometheus-metrics" % Versions.tapir lazy val tapirStubServer = tapirOrg %% "tapir-sttp-stub-server" % Versions.tapir % Test + lazy val tapirCirce = tapirOrg %% "tapir-json-circe" % Versions.tapir + lazy val tapirOpenApiDocs = tapirOrg %% "tapir-openapi-docs" % Versions.tapir + lazy val tapirOpenApiCirceYaml = tapirOrg %% "tapir-openapi-circe-yaml" % Versions.tapir + lazy val tapirHttp4sServer = tapirOrg %% "tapir-http4s-server" % Versions.tapir + lazy val tapirCore = tapirOrg %% "tapir-core" % Versions.tapir + lazy val tapirSwaggerUi = tapirOrg %% "tapir-swagger-ui-http4s" % Versions.tapir + // json lazy val playJson = playOrg %% "play-json" % Versions.playJson lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test - lazy val sttpCirceJson = sttpClient4Org %% "circe" % Versions.sttpCirceJson % Test + // STTP core and Circe integration + lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpCirceJson % Test + lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpCirceJson + lazy val clientBackend = sttpClient3Org %% "async-http-client-backend-zio" % Versions.sttpCirceJson // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb @@ -201,11 +213,13 @@ object Dependencies { tapirHttp4sZio, tapirSwagger, tapirPlayJson, + tapirCirce, tapirPrometheus, tapirStubServer, playJson, sttpPlayJson, - sttpCirceJson, + sttpCirce, + sttpCore, awsSecretsManagerSdk, zioTest, zioTestSbt, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 1877cc0b4..9bfe5c83e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.atum.server.model.ErrorResponse.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.fadb.exceptions.StatusException import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index 402b67884..6b547c3cb 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 69a140d55..8815a89c2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala index 20783e9f4..d122a93c2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointQueryDTO} -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.MultiSuccessResponse import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala index 23d12d8d2..8382b212c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointQueryDTO} import za.co.absa.atum.server.api.service.FlowService -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.MultiSuccessResponse import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala index c5e7dc737..2dfc53016 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.model.dto.{ CheckpointQueryDTO, PartitioningSubmitDTO } -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala index 964d57634..d37cc94a1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.PartitioningService -import za.co.absa.atum.server.model.ErrorResponse.{ErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 8447ebff5..f7b8da0d3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -31,7 +31,6 @@ import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 5adb17aa5..3da9fd091 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -31,7 +31,6 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 9992b75b5..961076ac8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -29,7 +29,6 @@ import za.co.absa.fadb.doobie.DoobieEngine import zio.interop.catz.asyncInstance import zio.{Task, URLayer, ZIO, ZLayer} import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 4f043b43c..52a6832f8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -29,7 +29,6 @@ import za.co.absa.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index f2a6d7c97..5d541417b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -29,7 +29,6 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index d7cd1dc1f..f2d03fffd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -29,10 +29,10 @@ import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, Singl import za.co.absa.atum.server.model.CirceJsonImplicits._ import sttp.tapir.{PublicEndpoint, endpoint} + trait Endpoints extends BaseEndpoints { - protected val createCheckpointEndpointV1 - : PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { + protected val createCheckpointEndpointV1: PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { apiV1.post .in(pathToAPIv1CompatibleFormat(CreateCheckpoint)) .in(jsonBody[CheckpointDTO]) @@ -40,8 +40,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[CheckpointDTO]) } - protected val createCheckpointEndpointV2 - : PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { + protected val createCheckpointEndpointV2: PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { apiV2.post .in(CreateCheckpoint) .in(jsonBody[CheckpointDTO]) @@ -49,8 +48,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) } - protected val createPartitioningEndpointV1 - : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { + protected val createPartitioningEndpointV1: PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { apiV1.post .in(pathToAPIv1CompatibleFormat(CreatePartitioning)) .in(jsonBody[PartitioningSubmitDTO]) @@ -58,8 +56,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[AtumContextDTO]) } - protected val createPartitioningEndpointV2 - : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, SingleSuccessResponse[AtumContextDTO], Any] = { + protected val createPartitioningEndpointV2: PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, SingleSuccessResponse[AtumContextDTO], Any] = { apiV2.post .in(CreatePartitioning) .in(jsonBody[PartitioningSubmitDTO]) @@ -67,8 +64,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AtumContextDTO]]) } - protected val createOrUpdateAdditionalDataEndpointV2 - : PublicEndpoint[AdditionalDataSubmitDTO, ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO], Any] = { + protected val createOrUpdateAdditionalDataEndpointV2: PublicEndpoint[AdditionalDataSubmitDTO, ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO], Any] = { apiV2.post .in(CreateOrUpdateAdditionalData) .in(jsonBody[AdditionalDataSubmitDTO]) @@ -76,8 +72,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AdditionalDataSubmitDTO]]) } - protected val getPartitioningCheckpointsEndpointV2 - : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { + protected val getPartitioningCheckpointsEndpointV2: PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.get .in(GetPartitioningCheckpoints) .in(jsonBody[CheckpointQueryDTO]) @@ -85,8 +80,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[MultiSuccessResponse[CheckpointDTO]]) } - protected val getFlowCheckpointsEndpointV2 - : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { + protected val getFlowCheckpointsEndpointV2: PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.post .in(GetFlowCheckpoints) .in(jsonBody[CheckpointQueryDTO]) @@ -100,5 +94,4 @@ trait Endpoints extends BaseEndpoints { protected val healthEndpoint: PublicEndpoint[Unit, Unit, Unit, Any] = endpoint.get.in(Health) - } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala index 450e2e56c..75df93885 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala @@ -16,8 +16,9 @@ package za.co.absa.atum.server.model -import io.circe._, io.circe.generic.semiauto._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import io.circe._ +import io.circe.generic.semiauto._ +import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType import za.co.absa.atum.model.dto._ object CirceJsonImplicits { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index 821fdcc56..bf17272b4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -16,37 +16,6 @@ package za.co.absa.atum.server.model -//import play.api.libs.json.{Json, Reads, Writes} -// -//sealed trait ErrorResponse { -// def message: String -//} -// -//object ErrorResponse { -// implicit val reads: Reads[ErrorResponse] = Json.reads[ErrorResponse] -// implicit val writes: Writes[ErrorResponse] = Json.writes[ErrorResponse] -//} -// -//final case class BadRequestResponse(message: String) extends ErrorResponse -// -//object BadRequestResponse { -// implicit val reads: Reads[BadRequestResponse] = Json.reads[BadRequestResponse] -// implicit val writes: Writes[BadRequestResponse] = Json.writes[BadRequestResponse] -//} -// -//final case class GeneralErrorResponse(message: String) extends ErrorResponse -// -//object GeneralErrorResponse { -// implicit val reads: Reads[GeneralErrorResponse] = Json.reads[GeneralErrorResponse] -// implicit val writes: Writes[GeneralErrorResponse] = Json.writes[GeneralErrorResponse] -//} -// -//final case class InternalServerErrorResponse(message: String) extends ErrorResponse -// -//object InternalServerErrorResponse { -// implicit val reads: Reads[InternalServerErrorResponse] = Json.reads[InternalServerErrorResponse] -// implicit val writes: Writes[InternalServerErrorResponse] = Json.writes[InternalServerErrorResponse] -//} import io.circe._ import io.circe.generic.semiauto._ diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index bb5a14006..ad987d659 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,30 +16,11 @@ package za.co.absa.atum.server.model -//import play.api.libs.json.{Json, Writes} -//import za.co.absa.atum.model.dto.PartitioningDTO -// -//private[server] case class PartitioningForDB private ( -// version: Int = 1, -// keys: Seq[String], -// keysToValues: Map[String, String] -//) -// -//object PartitioningForDB { -// -// def fromSeqPartitionDTO(partitioning: PartitioningDTO): PartitioningForDB = { -// val allKeys = partitioning.map(_.key) -// val mapOfKeysAndValues = partitioning.map(p => p.key -> p.value).toMap[String, String] -// -// PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) -// } -// -// implicit val writes: Writes[PartitioningForDB] = Json.writes -// -//} - -import io.circe._, io.circe.generic.semiauto._ + +import io.circe.generic.semiauto._ +import io.circe.{Decoder, Encoder} import za.co.absa.atum.model.dto.PartitioningDTO +import scala.collection.immutable.Seq private[server] case class PartitioningForDB private ( version: Int = 1, @@ -50,12 +31,12 @@ private[server] case class PartitioningForDB private ( object PartitioningForDB { def fromSeqPartitionDTO(partitioning: PartitioningDTO): PartitioningForDB = { - val allKeys = partitioning.map(_.key) - val mapOfKeysAndValues = partitioning.map(p => p.key -> p.value).toMap[String, String] + val allKeys: Seq[String] = partitioning.map(_.key) + val mapOfKeysAndValues: Map[String, String] = partitioning.map(p => p.key -> p.value).toMap PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) } - implicit val encodePartitioningForDB: Encoder[PartitioningForDB] = deriveEncoder - + implicit val encoder: Encoder[PartitioningForDB] = deriveEncoder + implicit val decoder: Decoder[PartitioningForDB] = deriveDecoder } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 80cdf125c..faa02bc01 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -20,7 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.fadb.exceptions.ErrorInDataException import za.co.absa.fadb.status.FunctionStatus import zio.test.Assertion.failsWithA diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala index 98bd3a0f1..be3f586b7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala @@ -20,7 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.FlowService -import za.co.absa.atum.server.model.ErrorResponse.InternalServerErrorResponse +import za.co.absa.atum.server.model.InternalServerErrorResponse import zio._ import zio.test.Assertion.failsWithA import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 340949d8e..7de77eb40 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -21,8 +21,8 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.PartitioningService -import za.co.absa.atum.server.model.ErrorResponse.InternalServerErrorResponse -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} +import za.co.absa.atum.server.model.InternalServerErrorResponse +import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.{equalTo, failsWithA} import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index a3a2eb340..7fb322666 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -16,20 +16,22 @@ package za.co.absa.atum.server.api.http +import io.circe.generic.auto._ +import io.circe.syntax.EncoderOps import org.mockito.Mockito.{mock, when} -import sttp.client3._ -import sttp.client3.playJson._ import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.CirceJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ -import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeCheckpointDTO, encodeCheckpointDTO} import zio.test.Assertion.equalTo import zio.test._ @@ -46,8 +48,8 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) - private val createCheckpointServerEndpoint = - createCheckpointEndpointV2.zServerLogic(CheckpointController.createCheckpointV2) + private val createCheckpointServerEndpoint = createCheckpointEndpointV2 + .zServerLogic(CheckpointController.createCheckpointV2) def spec: Spec[TestEnvironment with Scope, Any] = { val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) @@ -62,7 +64,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w suite("CreateCheckpointEndpointSuite")( test("Returns expected CheckpointDTO") { val response = request - .body(checkpointDTO1) + .body(checkpointDTO1.asJson.noSpaces) .send(backendStub) val body = response.map(_.body) @@ -72,7 +74,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected BadRequest") { val response = request - .body(checkpointDTO2) + .body(checkpointDTO2.asJson.noSpaces) .send(backendStub) val statusCode = response.map(_.code) @@ -81,7 +83,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected InternalServerError") { val response = request - .body(checkpointDTO3) + .body(checkpointDTO3.asJson.noSpaces) .send(backendStub) val statusCode = response.map(_.code) @@ -92,5 +94,4 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }.provide( checkpointControllerMockLayer ) - } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index e0a9e16a0..5f7022bd7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -16,31 +16,24 @@ package za.co.absa.atum.server.api.http -import org.mockito.Mockito.{mock, when} -//import sttp.client3._ -//import sttp.client3.playJson._ -//import sttp.client3.testing.SttpBackendStub -import sttp.client4._ -import sttp.client4.circe._ import io.circe.generic.auto._ -import sttp.client4.testing._ -import sttp.client4._ +import org.mockito.Mockito.{mock, when} +import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController -import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} -//import za.co.absa.atum.server.model.PlayJsonImplicits._ -//import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.CirceJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo import zio.test._ -import java.util.UUID - object CreatePartitioningEndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { private val createPartitioningEndpointMock = mock(classOf[PartitioningController]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala index 699ae3a34..7b0ae8a26 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -16,18 +16,18 @@ package za.co.absa.atum.server.api.http +import io.circe.generic.auto._ import org.mockito.Mockito.{mock, when} -import sttp.client3._ -import sttp.client3.playJson._ import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.FlowController -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.server.model.PlayJsonImplicits._ +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.MultiSuccessResponse import zio._ import zio.test.Assertion.equalTo From 3c707e6e438b20be94b8dde8e8327628ea06d455 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 24 Jun 2024 14:14:40 +0200 Subject: [PATCH 07/34] removing play Json dependencies, and PlayJsonImplicits class --- .../atum/model/dto/MeasureResultDTO.scala | 1 + project/Dependencies.scala | 8 -- .../runs/functions/WriteCheckpoint.scala | 9 +- .../atum/server/model/CheckpointFromDB.scala | 1 + .../atum/server/model/PlayJsonImplicits.scala | 102 ------------------ .../WriteCheckpointIntegrationTests.scala | 4 +- 6 files changed, 12 insertions(+), 113 deletions(-) delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 61b26cac6..7bf65ca44 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -17,6 +17,7 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} +import io.circe.generic.auto._ case class MeasureResultDTO( mainValue: MeasureResultDTO.TypedValue, diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 29769d1c0..0f9a58aef 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -73,8 +73,6 @@ object Dependencies { val tapir = "1.9.6" val http4sBlazeBackend = "0.23.15" val http4sPrometheus = "0.23.6" - val playJson = "3.0.1" - val sttpPlayJson = "3.9.3" val circeJson = "0.14.7" val sttpCirceJson = "3.9.7" @@ -174,10 +172,6 @@ object Dependencies { lazy val tapirCore = tapirOrg %% "tapir-core" % Versions.tapir lazy val tapirSwaggerUi = tapirOrg %% "tapir-swagger-ui-http4s" % Versions.tapir - // json - lazy val playJson = playOrg %% "play-json" % Versions.playJson - lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test - // STTP core and Circe integration lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpCirceJson % Test lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpCirceJson @@ -216,8 +210,6 @@ object Dependencies { tapirCirce, tapirPrometheus, tapirStubServer, - playJson, - sttpPlayJson, sttpCirce, sttpCore, awsSecretsManagerSdk, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index c7008f9e2..13372cbed 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -32,7 +32,12 @@ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ +import za.co.absa.atum.model.dto.MeasureResultDTO._ +import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ +import doobie.postgres.circe.jsonb.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task] @@ -48,7 +53,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) values.measurements.map(x => x.asJson.noSpaces) } - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( + val sqlDebug = sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningNormalized @@ -64,6 +69,8 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ${values.measuredByAtumAgent}, ${values.author} ) ${Fragment.const(alias)};""" + println(sqlDebug) + sqlDebug } } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 705e6c319..f2eb606f0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -18,6 +18,7 @@ package za.co.absa.atum.server.model import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import io.circe.{DecodingFailure, Json} +import za.co.absa.atum.server.model.CirceJsonImplicits._ import java.time.ZonedDateTime import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala deleted file mode 100644 index a4e32fe0f..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import play.api.libs.functional.syntax.toFunctionalBuilderOps -import play.api.libs.json._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} -import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} - -object PlayJsonImplicits { - - implicit val optionStringReads: Reads[Option[String]] = new Reads[Option[String]] { - def reads(json: JsValue): JsResult[Option[String]] = json match { - case JsNull => JsSuccess(None) - case JsString(s) => JsSuccess(Some(s)) - case _ => JsError("Expected JsString or JsNull") - } - } - - implicit val optionStringWrites: Writes[Option[String]] = new Writes[Option[String]] { - def writes(opt: Option[String]): JsValue = opt match { - case Some(s) => JsString(s) - case None => JsNull - } - } - - implicit val resultValueTypeReads: Reads[ResultValueType] = new Reads[ResultValueType] { - override def reads(json: JsValue): JsResult[ResultValueType] = json match { - case JsString("String") => JsSuccess(ResultValueType.String) - case JsString("Long") => JsSuccess(ResultValueType.Long) - case JsString("BigDecimal") => JsSuccess(ResultValueType.BigDecimal) - case JsString("Double") => JsSuccess(ResultValueType.Double) - case _ => JsError("Invalid ResultValueType") - } - } - - implicit val resultValueTypeWrites: Writes[ResultValueType] = new Writes[ResultValueType] { - def writes(resultValueType: ResultValueType): JsValue = resultValueType match { - case ResultValueType.String => Json.toJson("String") - case ResultValueType.Long => Json.toJson("Long") - case ResultValueType.BigDecimal => Json.toJson("BigDecimal") - case ResultValueType.Double => Json.toJson("Double") - } - } - - implicit val readsMeasureDTO: Reads[MeasureDTO] = Json.reads[MeasureDTO] - implicit val writesMeasureDTO: Writes[MeasureDTO] = Json.writes[MeasureDTO] - - implicit val readsTypedValue: Reads[MeasureResultDTO.TypedValue] = Json.reads[MeasureResultDTO.TypedValue] - implicit val writesTypedValue: Writes[MeasureResultDTO.TypedValue] = Json.writes[MeasureResultDTO.TypedValue] - - implicit val readsMeasureResultDTO: Reads[MeasureResultDTO] = { - ((__ \ "mainValue").read[MeasureResultDTO.TypedValue] and - (__ \ "supportValues").readNullable[Map[String, TypedValue]].map(_.getOrElse(Map.empty)) - )(MeasureResultDTO.apply _) - } - - implicit val writesMeasureResultDTO: Writes[MeasureResultDTO] = Json.writes[MeasureResultDTO] - - implicit val readsMeasurementDTO: Reads[MeasurementDTO] = Json.reads[MeasurementDTO] - implicit val writesMeasurementDTO: Writes[MeasurementDTO] = Json.writes[MeasurementDTO] - - implicit val readsPartitionDTO: Reads[PartitionDTO] = Json.reads[PartitionDTO] - implicit val writesPartitionDTO: Writes[PartitionDTO] = Json.writes[PartitionDTO] - - implicit val readsCheckpointDTO: Reads[CheckpointDTO] = Json.reads[CheckpointDTO] - implicit val writesCheckpointDTO: Writes[CheckpointDTO] = Json.writes[CheckpointDTO] - - implicit val readsPartitioningSubmitDTO: Reads[PartitioningSubmitDTO] = Json.reads[PartitioningSubmitDTO] - implicit val writesPartitioningSubmitDTO: Writes[PartitioningSubmitDTO] = Json.writes[PartitioningSubmitDTO] - - implicit val readsStringMap: Reads[Map[String, Option[String]]] = Reads.mapReads[Option[String]] - implicit val writesStringMap: OWrites[MapWrites.Map[String, Option[String]]] = - Writes.genericMapWrites[Option[String], MapWrites.Map] - - implicit val readsAdditionalDataSubmitDTO: Reads[AdditionalDataSubmitDTO] = Json.reads[AdditionalDataSubmitDTO] - implicit val writesAdditionalDataSubmitDTO: Writes[AdditionalDataSubmitDTO] = Json.writes[AdditionalDataSubmitDTO] - - implicit val readsAtumContextDTO: Reads[AtumContextDTO] = Json.reads[AtumContextDTO] - implicit val writesAtumContextDTO: Writes[AtumContextDTO] = Json.writes[AtumContextDTO] - - implicit def formatSingleSuccessResponse[T: Format]: Format[SingleSuccessResponse[T]] = Json.format[SingleSuccessResponse[T]] - implicit def formatMultiSuccessResponse[T: Format]: Format[MultiSuccessResponse[T]] = Json.format[MultiSuccessResponse[T]] - - implicit val readsCheckpointQueryDTO: Reads[CheckpointQueryDTO] = Json.reads[CheckpointQueryDTO] - implicit val writesCheckpointQueryDTO: Writes[CheckpointQueryDTO] = Json.writes[CheckpointQueryDTO] -} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index f76e044e8..054e86af9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -16,8 +16,8 @@ package za.co.absa.atum.server.api.database.runs.functions +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} -import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider @@ -41,7 +41,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { author = "author", partitioning = Seq(PartitionDTO("key1", "val1"), PartitionDTO("key2", "val2")), processStartTime = ZonedDateTime.now(), - processEndTime = None, + processEndTime = Option(ZonedDateTime.now()), measurements = Set(MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.Long)))) ) From 935cde04af3457662c9bccd973e20689d428018d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 24 Jun 2024 14:18:23 +0200 Subject: [PATCH 08/34] removing play Json dependencies, and PlayJsonImplicits class --- .../main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 7bf65ca44..61b26cac6 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} -import io.circe.generic.auto._ case class MeasureResultDTO( mainValue: MeasureResultDTO.TypedValue, From 3c2ca0041c59add7100638cd0805fa6a8641c5a8 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 25 Jun 2024 16:19:45 +0200 Subject: [PATCH 09/34] fixing bugs and using circe instead of Json4s dependency --- .../agent/dispatcher/HttpDispatcher.scala | 1 + .../atum/model/utils/SerializationUtils.scala | 82 +++++++++++-------- .../utils/SerializationUtilsUnitTests.scala | 5 +- .../server/api/database/DoobieImplicits.scala | 9 +- .../CreatePartitioningIfNotExists.scala | 4 +- .../runs/functions/WriteCheckpoint.scala | 11 ++- .../WriteCheckpointIntegrationTests.scala | 5 +- 7 files changed, 65 insertions(+), 52 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index cc7ea6623..c773229e3 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -23,6 +23,7 @@ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} import za.co.absa.atum.model.utils.SerializationUtils +import io.circe.generic.auto._ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala index fcd95568d..67d4ea85c 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala @@ -16,33 +16,36 @@ package za.co.absa.atum.model.utils -import org.json4s.JsonAST.JString -import org.json4s.jackson.Serialization -import org.json4s.jackson.Serialization.{write, writePretty} -import org.json4s.{CustomSerializer, Formats, JNull, NoTypeHints, ext} -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType._ +import io.circe.{Decoder, Encoder} +import io.circe.syntax._ +import io.circe.parser._ +import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import java.util.UUID object SerializationUtils { - implicit private val formatsJson: Formats = - Serialization.formats(NoTypeHints).withBigDecimal + - ext.UUIDSerializer + - ZonedDateTimeSerializer + - ResultValueTypeSerializer - // TODO "yyyy-MM-dd'T'hh:mm:ss.SSS'Z'" OR TODO "yyyy-MM-dd HH:mm:ss.SSSSSSX" val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME + implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) + implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => + Right(ZonedDateTime.parse(str, timestampFormat)) + } + + implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) + implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => + Right(UUID.fromString(str)) + } + /** * The method returns arbitrary object as a Json string. * * @return A string representing the object in Json format */ - def asJson[T <: AnyRef](obj: T): String = { - write[T](obj) + def asJson[T: Encoder](obj: T): String = { + obj.asJson.noSpaces } /** @@ -50,8 +53,8 @@ object SerializationUtils { * * @return A string representing the object in Json format */ - def asJsonPretty[T <: AnyRef](obj: T): String = { - writePretty[T](obj) + def asJsonPretty[T: Encoder](obj: T): String = { + obj.asJson.spaces2 } /** @@ -59,27 +62,34 @@ object SerializationUtils { * * @return An object deserialized from the Json string */ - def fromJson[T <: AnyRef](jsonStr: String)(implicit m: Manifest[T]): T = { - Serialization.read[T](jsonStr) + def fromJson[T: Decoder](jsonStr: String): T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + } } - private case object ResultValueTypeSerializer extends CustomSerializer[ResultValueType](format => ( - { - case JString(resultValType) => resultValType match { - case "String" => String - case "Long" => Long - case "BigDecimal" => BigDecimal - case "Double" => Double - } - case JNull => null - }, - { - case resultValType: ResultValueType => resultValType match { - case String => JString("String") - case Long => JString("Long") - case BigDecimal => JString("BigDecimal") - case Double => JString("Double") - } - })) + sealed trait ResultValueType + object ResultValueType { + case object String extends ResultValueType + case object Long extends ResultValueType + case object BigDecimal extends ResultValueType + case object Double extends ResultValueType + + implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { + case ResultValueType.String => "String" + case ResultValueType.Long => "Long" + case ResultValueType.BigDecimal => "BigDecimal" + case ResultValueType.Double => "Double" + } + + implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { + case "String" => Right(ResultValueType.String) + case "Long" => Right(ResultValueType.Long) + case "BigDecimal" => Right(ResultValueType.BigDecimal) + case "Double" => Right(ResultValueType.Double) + case other => Left(s"Cannot decode $other as ResultValueType") + } + } } diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 91ebc9ab9..f43fa083f 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -16,6 +16,7 @@ package za.co.absa.atum.model.utils +import io.circe.generic.auto._ import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} import za.co.absa.atum.model.dto._ @@ -41,7 +42,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAdditionalDataJson = """ |{"partitioning":[{"key":"key","value":"val"}], - |"additionalData":{"key1":"val1","key2":"val2"}, + |"additionalData":{"key1":"val1","key2":"val2","key3":null}, |"author":"testAuthor"} |""".linearize val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) @@ -343,7 +344,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"authorIfNew":"authorTest"}""" + val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index a1131e65b..5a002d2df 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -20,6 +20,7 @@ import cats.Show import cats.data.NonEmptyList import doobie.{Get, Put} import doobie.postgres.implicits._ +import io.circe.Json import org.postgresql.jdbc.PgArray import org.postgresql.util.PGobject @@ -42,7 +43,7 @@ object DoobieImplicits { object Json { - implicit val jsonArrayPutUsingString: Put[List[String]] = { + implicit val jsonArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("json[]") @@ -50,7 +51,7 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - o.setValue(a.mkString("{", ",", "}")) + o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) o } } @@ -100,7 +101,7 @@ object DoobieImplicits { object Jsonb { - implicit val jsonbArrayPutUsingString: Put[List[String]] = { + implicit val jsonbArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("jsonb[]") @@ -108,7 +109,7 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("jsonb[]") - o.setValue(a.mkString("{", ",", "}")) + o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) o } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 3da9fd091..708add14e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -50,8 +50,8 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningJsonString }, - ${values.authorIfNew}, - ${ + ${values.authorIfNew}, + ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString parentPartitioningJsonString } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 13372cbed..58f2f6b3a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.Fragment -import doobie.implicits._ +import doobie.implicits.toSqlInterpolator import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.model.PartitioningForDB @@ -31,13 +31,12 @@ import zio._ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ - import za.co.absa.atum.model.dto.MeasureResultDTO._ import za.co.absa.atum.server.model.CirceJsonImplicits._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits._ +import io.circe.Json class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task] @@ -50,7 +49,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) // List[String] containing json data has to be properly escaped // It would be safer to use Json data type and derive Put instance val measurementsNormalized = { - values.measurements.map(x => x.asJson.noSpaces) + values.measurements.toList.map(_.asJson) } val sqlDebug = sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( @@ -63,8 +62,8 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ${values.processStartTime}, ${values.processEndTime}, ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPutUsingString - measurementsNormalized.toList + import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPut + measurementsNormalized }, ${values.measuredByAtumAgent}, ${values.author} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 054e86af9..b9878b104 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -35,11 +35,12 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { suite("WriteCheckpointSuite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { + val checkpointDTO = CheckpointDTO( id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key1", "val1"), PartitionDTO("key2", "val2")), + partitioning = Seq(PartitionDTO("key2", "value2")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = @@ -53,7 +54,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ).provide( WriteCheckpoint.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithoutRollback ) } From 0f2b220c56956cb15ab5757dedd0b478f07143f7 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jun 2024 15:21:22 +0200 Subject: [PATCH 10/34] revised arrayPutUsing string implicit conversion --- .../server/api/database/DoobieImplicits.scala | 18 ++++++++++++++++-- .../runs/functions/WriteCheckpoint.scala | 4 +--- .../WriteCheckpointIntegrationTests.scala | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 5a002d2df..3a1969ed2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -51,7 +51,14 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) + val arrayElements = a.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s"\"$escapedJsonString\"" + } + // Join all elements into a single string wrapped in curly braces + o.setValue(arrayElements.mkString("{", ",", "}")) o } } @@ -109,7 +116,14 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("jsonb[]") - o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) + val arrayElements = a.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s"\"$escapedJsonString\"" + } + // Join all elements into a single string wrapped in curly braces + o.setValue(arrayElements.mkString("{", ",", "}")) o } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 58f2f6b3a..a6587f296 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -52,7 +52,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) values.measurements.toList.map(_.asJson) } - val sqlDebug = sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( + sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningNormalized @@ -68,8 +68,6 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ${values.measuredByAtumAgent}, ${values.author} ) ${Fragment.const(alias)};""" - println(sqlDebug) - sqlDebug } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index b9878b104..6f4c4c2dc 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -40,7 +40,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key2", "value2")), + partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = From ad5999901e32f4fdfd8273843d05e7f83c373ce8 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 27 Jun 2024 10:35:56 +0200 Subject: [PATCH 11/34] removed json4s and replaced its usage --- .../model/utils/ZonedDateTimeSerializer.scala | 24 +++++++++-------- project/Dependencies.scala | 26 ------------------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala b/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala index 6fe76dd61..97f300321 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala @@ -16,17 +16,19 @@ package za.co.absa.atum.model.utils -import org.json4s.JsonAST.JString -import org.json4s.{CustomSerializer, JNull} - +import io.circe.{Decoder, Encoder} import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter + +object ZonedDateTimeSerializer { + implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap( + _.format(DateTimeFormatter.ISO_ZONED_DATE_TIME)) -case object ZonedDateTimeSerializer extends CustomSerializer[ZonedDateTime](_ => ( - { - case JString(s) => ZonedDateTime.parse(s, SerializationUtils.timestampFormat) - case JNull => null - }, - { - case d: ZonedDateTime => JString(SerializationUtils.timestampFormat.format(d)) + implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => + try { + Right(ZonedDateTime.parse(str, DateTimeFormatter.ISO_ZONED_DATE_TIME)) + } catch { + case e: Throwable => Left(e.getMessage) + } } -)) +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0f9a58aef..41e855530 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -46,22 +46,6 @@ object Dependencies { val fadb = "0.3.0" - val json4s_spark2 = "3.5.3" - val json4s_spark3 = "3.7.0-M11" - def json4s(scalaVersion: Version): String = { - // TODO done this impractical way until https://github.com/AbsaOSS/commons/issues/134 - val maj2 = Component("2") - val min11 = Component("11") - val min12 = Component("12") - val min13 = Component("13") - scalaVersion.components match { - case Seq(`maj2`, `min11`, _) => json4s_spark2 - case Seq(`maj2`, `min12`, _) => json4s_spark3 - case Seq(`maj2`, `min13`, _) => json4s_spark3 - case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12, and 2.13 are currently supported.") - } - } - val logback = "1.2.3" val zio = "2.0.19" @@ -106,15 +90,9 @@ object Dependencies { } private def jsonSerdeDependencies(scalaVersion: Version): Seq[ModuleID] = { - val json4sVersion = Versions.json4s(scalaVersion) lazy val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jacksonModuleScala - lazy val json4sExt = "org.json4s" %% "json4s-ext" % json4sVersion - lazy val json4sCore = "org.json4s" %% "json4s-core" % json4sVersion - lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion - lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided - // Circe dependencies lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson @@ -122,10 +100,6 @@ object Dependencies { Seq( jacksonModuleScala, - json4sExt, - json4sCore, - json4sJackson, - json4sNative, circeCore, circeParser, circeGeneric, From cc9e949af5f292526fa5227db25a21e0c388a097 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 09:51:41 +0200 Subject: [PATCH 12/34] rolling back WriteCheckpointIntegrationTests.scala --- .../runs/functions/WriteCheckpointIntegrationTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 6f4c4c2dc..2c655623f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -54,7 +54,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ).provide( WriteCheckpoint.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithoutRollback + TestTransactorProvider.layerWithRollback ) } From 3f5af8189c956ff9124f6e498adc06bfe79fa1b9 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 15:03:03 +0200 Subject: [PATCH 13/34] defining implicits in the companiong object of the given DTOs --- .../model/dto/AdditionalDataSubmitDTO.scala | 9 + .../absa/atum/model/dto/AtumContextDTO.scala | 9 + .../absa/atum/model/dto/CheckpointDTO.scala | 12 ++ .../atum/model/dto/CheckpointQueryDTO.scala | 8 + .../co/absa/atum/model/dto/MeasureDTO.scala | 8 + .../atum/model/dto/MeasureResultDTO.scala | 68 +++---- .../absa/atum/model/dto/MeasurementDTO.scala | 8 + .../co/absa/atum/model/dto/PartitionDTO.scala | 8 + .../model/dto/PartitioningSubmitDTO.scala | 8 + .../za/co/absa/atum/model/dto/package.scala | 13 ++ .../absa/atum/model/utils/JsonImplicits.scala | 142 +++++++++++++++ .../atum/model/utils/SerializationUtils.scala | 167 ++++++++++-------- .../runs/functions/WriteCheckpoint.scala | 2 +- .../absa/atum/server/api/http/Endpoints.scala | 2 +- .../atum/server/model/CheckpointFromDB.scala | 2 +- .../server/model/CirceJsonImplicits.scala | 74 -------- .../CreateCheckpointEndpointUnitTests.scala | 2 +- .../CreatePartitioningEndpointUnitTests.scala | 2 +- 18 files changed, 364 insertions(+), 180 deletions(-) create mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala index 5beb3aaa1..9fe3bf8e8 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala @@ -16,8 +16,17 @@ package za.co.absa.atum.model.dto + +import io.circe.generic.semiauto._ +import io.circe._ + case class AdditionalDataSubmitDTO ( partitioning: PartitioningDTO, additionalData: AdditionalDataDTO, author: String ) + +object AdditionalDataSubmitDTO { + implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder + implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala index 360986636..486869d43 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AtumContextDTO.scala @@ -16,8 +16,17 @@ package za.co.absa.atum.model.dto + +import io.circe.generic.semiauto._ +import io.circe._ + case class AtumContextDTO( partitioning: PartitioningDTO, measures: Set[MeasureDTO] = Set.empty, additionalData: AdditionalDataDTO = Map.empty ) + +object AtumContextDTO { + implicit val decodeAtumContextDTO: Decoder[AtumContextDTO] = deriveDecoder + implicit val encodeAtumContextDTO: Encoder[AtumContextDTO] = deriveEncoder +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala index b833e128f..674636954 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala @@ -16,9 +16,16 @@ package za.co.absa.atum.model.dto +import io.circe.generic.semiauto._ +import io.circe._ + import java.time.ZonedDateTime import java.util.UUID +import za.co.absa.atum.model.dto._ +//import za.co.absa.atum.model.dto.MeasurementDTO._ +//import za.co.absa.atum.model.dto.PartitionDTO._ + case class CheckpointDTO( id: UUID, name: String, @@ -29,3 +36,8 @@ case class CheckpointDTO( processEndTime: Option[ZonedDateTime], measurements: Set[MeasurementDTO] ) + +object CheckpointDTO { + implicit val decodeCheckpointDTO: Decoder[CheckpointDTO] = deriveDecoder + implicit val encodeCheckpointDTO: Encoder[CheckpointDTO] = deriveEncoder +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointQueryDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointQueryDTO.scala index d47ec5536..6c2d2f95a 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointQueryDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointQueryDTO.scala @@ -16,8 +16,16 @@ package za.co.absa.atum.model.dto +import io.circe.generic.semiauto._ +import io.circe._ + case class CheckpointQueryDTO( partitioning: PartitioningDTO, limit: Option[Int], checkpointName: Option[String] ) + +object CheckpointQueryDTO { + implicit val decodeCheckpointQueryDTO: Decoder[CheckpointQueryDTO] = deriveDecoder + implicit val encodeCheckpointQueryDTO: Encoder[CheckpointQueryDTO] = deriveEncoder +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala index 89ce8d018..00220c37f 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala @@ -16,7 +16,15 @@ package za.co.absa.atum.model.dto +import io.circe._ +import io.circe.generic.semiauto._ + case class MeasureDTO( measureName: String, measuredColumns: Seq[String] ) + +object MeasureDTO { + implicit val decodeMeasureDTO: Decoder[MeasureDTO] = deriveDecoder[MeasureDTO] + implicit val encoderMeasureDTO: Encoder[MeasureDTO] = deriveEncoder[MeasureDTO] +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 61b26cac6..64d625715 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -16,49 +16,53 @@ package za.co.absa.atum.model.dto -import io.circe.{Decoder, Encoder} +import io.circe._ -case class MeasureResultDTO( - mainValue: MeasureResultDTO.TypedValue, - supportValues: Map[String, MeasureResultDTO.TypedValue] = Map.empty -) +sealed trait ResultValueType -object MeasureResultDTO { - case class TypedValue( - value: String, - valueType: ResultValueType - ) +object ResultValueType { + case object String extends ResultValueType - sealed trait ResultValueType + case object Long extends ResultValueType + case object BigDecimal extends ResultValueType + case object Double extends ResultValueType - object ResultValueType { - case object String extends ResultValueType - case object Long extends ResultValueType - case object BigDecimal extends ResultValueType - case object Double extends ResultValueType + implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { + case ResultValueType.String => "String" + case ResultValueType.Long => "Long" + case ResultValueType.BigDecimal => "BigDecimal" + case ResultValueType.Double => "Double" } - - implicit val encodeResultValueType: Encoder[MeasureResultDTO.ResultValueType] = Encoder.encodeString.contramap { - case MeasureResultDTO.ResultValueType.String => "String" - case MeasureResultDTO.ResultValueType.Long => "Long" - case MeasureResultDTO.ResultValueType.BigDecimal => "BigDecimal" - case MeasureResultDTO.ResultValueType.Double => "Double" + implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { + case "String" => Right(ResultValueType.String) + case "Long" => Right(ResultValueType.Long) + case "BigDecimal" => Right(ResultValueType.BigDecimal) + case "Double" => Right(ResultValueType.Double) + case other => Left(s"Cannot decode $other as ResultValueType") } - implicit val decodeResultValueType: Decoder[MeasureResultDTO.ResultValueType] = Decoder.decodeString.emap { - case "String" => Right(MeasureResultDTO.ResultValueType.String) - case "Long" => Right(MeasureResultDTO.ResultValueType.Long) - case "BigDecimal" => Right(MeasureResultDTO.ResultValueType.BigDecimal) - case "Double" => Right(MeasureResultDTO.ResultValueType.Double) - case other => Left(s"Cannot decode $other as ResultValueType") - } +} +case class MeasureResultDTO( + mainValue: TypedValue, + supportValues: Map[String, TypedValue] = Map.empty +) + +case class TypedValue( + value: String, + valueType: ResultValueType + ) + +object TypedValue { - implicit val encodeTypedValue: Encoder[MeasureResultDTO.TypedValue] = + implicit val encodeTypedValue: Encoder[TypedValue] = Encoder.forProduct2("value", "valueType")(tv => (tv.value, tv.valueType)) - implicit val decodeTypedValue: Decoder[MeasureResultDTO.TypedValue] = - Decoder.forProduct2("value", "valueType")(MeasureResultDTO.TypedValue.apply) + implicit val decodeTypedValue: Decoder[TypedValue] = + Decoder.forProduct2("value", "valueType")(TypedValue.apply) + +} +object MeasureResultDTO { implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = Decoder.forProduct2("mainValue", "supportValues")(MeasureResultDTO.apply) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala index d216c483c..d087c53f1 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasurementDTO.scala @@ -16,7 +16,15 @@ package za.co.absa.atum.model.dto +import io.circe.generic.semiauto._ +import io.circe._ + case class MeasurementDTO( measure: MeasureDTO, result: MeasureResultDTO ) + +object MeasurementDTO { + implicit val decodeMeasurementDTO: Decoder[MeasurementDTO] = deriveDecoder[MeasurementDTO] + implicit val encodeMeasurementDTO: Encoder[MeasurementDTO] = deriveEncoder[MeasurementDTO] +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala index 4b19b20df..053c1e1c2 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/PartitionDTO.scala @@ -16,7 +16,15 @@ package za.co.absa.atum.model.dto +import io.circe.generic.semiauto._ +import io.circe._ + case class PartitionDTO( key: String, value: String ) + +object PartitionDTO { + implicit val decodePartitionDTO: Decoder[PartitionDTO] = deriveDecoder + implicit val encodePartitionDTO: Encoder[PartitionDTO] = deriveEncoder +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala index 6599273b5..ad4fce6e0 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/PartitioningSubmitDTO.scala @@ -16,8 +16,16 @@ package za.co.absa.atum.model.dto +import io.circe.generic.semiauto._ +import io.circe._ + case class PartitioningSubmitDTO ( partitioning: PartitioningDTO, parentPartitioning: Option[PartitioningDTO], authorIfNew: String ) + +object PartitioningSubmitDTO { + implicit val decodePartitioningSubmitDTO: Decoder[PartitioningSubmitDTO] = deriveDecoder + implicit val encodePartitioningSubmitDTO: Encoder[PartitioningSubmitDTO] = deriveEncoder +} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index 03a475e8e..810ec42e7 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -16,7 +16,20 @@ package za.co.absa.atum.model + +import io.circe.generic.semiauto._ +import io.circe._ + package object dto { type PartitioningDTO = Seq[PartitionDTO] type AdditionalDataDTO = Map[String, Option[String]] + + // Implicit encoders and decoders for PartitioningDTO + implicit val decodePartitioningDTO: Decoder[PartitioningDTO] = Decoder.decodeSeq[PartitionDTO] + implicit val encodePartitioningDTO: Encoder[PartitioningDTO] = Encoder.encodeSeq[PartitionDTO] + + // Implicit encoders and decoders for AdditionalDataDTO + implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = Decoder.decodeMap[String, Option[String]] + implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = Encoder.encodeMap[String, Option[String]] + } diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala new file mode 100644 index 000000000..efb4446cd --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala @@ -0,0 +1,142 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.utils + + +//import io.circe.{Decoder, Encoder} +//import io.circe.generic.semiauto._ +//import io.circe.syntax._ +//import za.co.absa.atum.model.dto._ +// +//import java.time.ZonedDateTime +//import java.time.format.DateTimeFormatter +//import java.util.UUID + +//object JsonImplicits { +// +// // Timestamp format +// val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME +// +// // Implicit encoders and decoders for ZonedDateTime +// implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) +// implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => +// Right(ZonedDateTime.parse(str, timestampFormat)) +// } +// +// // Implicit encoders and decoders for UUID +// implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) +// implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => +// Right(UUID.fromString(str)) +// } +// +// // Implicit encoders and decoders for Option[String] +// implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] +// implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] +// +// // Implicit encoders and decoders for ResultValueType +// sealed trait ResultValueType +// object ResultValueType { +// case object String extends ResultValueType +// case object Long extends ResultValueType +// case object BigDecimal extends ResultValueType +// case object Double extends ResultValueType +// +// implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { +// case ResultValueType.String => "String" +// case ResultValueType.Long => "Long" +// case ResultValueType.BigDecimal => "BigDecimal" +// case ResultValueType.Double => "Double" +// } +// +// implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { +// case "String" => Right(ResultValueType.String) +// case "Long" => Right(ResultValueType.Long) +// case "BigDecimal" => Right(ResultValueType.BigDecimal) +// case "Double" => Right(ResultValueType.Double) +// case other => Left(s"Cannot decode $other as ResultValueType") +// } +// } +// +// // Implicit encoders and decoders for various DTOs +// implicit val decodeTypedValue: Decoder[MeasureResultDTO.TypedValue] = deriveDecoder +// implicit val encodeTypedValue: Encoder[MeasureResultDTO.TypedValue] = deriveEncoder +// +// implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = deriveDecoder +// implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = deriveEncoder +// +// implicit val decodeMeasureDTO: Decoder[MeasureDTO] = deriveDecoder +// implicit val encodeMeasureDTO: Encoder[MeasureDTO] = deriveEncoder +// +// implicit val decodeMeasurementDTO: Decoder[MeasurementDTO] = deriveDecoder +// implicit val encodeMeasurementDTO: Encoder[MeasurementDTO] = deriveEncoder +// +// implicit val decodePartitionDTO: Decoder[PartitionDTO] = deriveDecoder +// implicit val encodePartitionDTO: Encoder[PartitionDTO] = deriveEncoder +// +// implicit val decodeCheckpointDTO: Decoder[CheckpointDTO] = deriveDecoder +// implicit val encodeCheckpointDTO: Encoder[CheckpointDTO] = deriveEncoder +// +// implicit val decodePartitioningSubmitDTO: Decoder[PartitioningSubmitDTO] = deriveDecoder +// implicit val encodePartitioningSubmitDTO: Encoder[PartitioningSubmitDTO] = deriveEncoder +// +// implicit val decodeStringMap: Decoder[Map[String, Option[String]]] = Decoder.decodeMap[String, Option[String]] +// implicit val encodeStringMap: Encoder[Map[String, Option[String]]] = Encoder.encodeMap[String, Option[String]] +// +// implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder +// implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder +// +// implicit val decodeAtumContextDTO: Decoder[AtumContextDTO] = deriveDecoder +// implicit val encodeAtumContextDTO: Encoder[AtumContextDTO] = deriveEncoder +// +// // JSON serialization and deserialization utilities +// def asJson[T: Encoder](obj: T): String = { +// obj.asJson.noSpaces +// } +// +// def asJsonPretty[T: Encoder](obj: T): String = { +// obj.asJson.spaces2 +// } +// +// def fromJson[T: Decoder](jsonStr: String): T = { +// io.circe.parser.decode[T](jsonStr) match { +// case Right(value) => value +// case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") +// } +// } +//} +// + +import io.circe.{Encoder, Decoder} +import io.circe.syntax._ +import io.circe.parser.decode + +object JsonUtils { + def asJson[T: Encoder](obj: T): String = { + obj.asJson.noSpaces + } + + def asJsonPretty[T: Encoder](obj: T): String = { + obj.asJson.spaces2 + } + + def fromJson[T: Decoder](jsonStr: String): T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + } + } +} diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala index 67d4ea85c..6c67e69f6 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala @@ -17,79 +17,108 @@ package za.co.absa.atum.model.utils -import io.circe.{Decoder, Encoder} -import io.circe.syntax._ -import io.circe.parser._ +//import io.circe.{Decoder, Encoder} +//import io.circe.syntax._ +//import io.circe.parser._ +//import java.time.ZonedDateTime +//import java.time.format.DateTimeFormatter +//import java.util.UUID +// +//object SerializationUtils { +// +// +// val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME +// +// implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) +// implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => +// Right(ZonedDateTime.parse(str, timestampFormat)) +// } +// +// implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) +// implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => +// Right(UUID.fromString(str)) +// } +// +// /** +// * The method returns arbitrary object as a Json string. +// * +// * @return A string representing the object in Json format +// */ +// def asJson[T: Encoder](obj: T): String = { +// obj.asJson.noSpaces +// } +// +// /** +// * The method returns arbitrary object as a pretty Json string. +// * +// * @return A string representing the object in Json format +// */ +// def asJsonPretty[T: Encoder](obj: T): String = { +// obj.asJson.spaces2 +// } +// +// /** +// * The method returns arbitrary object parsed from Json string. +// * +// * @return An object deserialized from the Json string +// */ +// def fromJson[T: Decoder](jsonStr: String): T = { +// decode[T](jsonStr) match { +// case Right(value) => value +// case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") +// } +// } +// +// sealed trait ResultValueType +// object ResultValueType { +// case object String extends ResultValueType +// case object Long extends ResultValueType +// case object BigDecimal extends ResultValueType +// case object Double extends ResultValueType +// +// implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { +// case ResultValueType.String => "String" +// case ResultValueType.Long => "Long" +// case ResultValueType.BigDecimal => "BigDecimal" +// case ResultValueType.Double => "Double" +// } +// +// implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { +// case "String" => Right(ResultValueType.String) +// case "Long" => Right(ResultValueType.Long) +// case "BigDecimal" => Right(ResultValueType.BigDecimal) +// case "Double" => Right(ResultValueType.Double) +// case other => Left(s"Cannot decode $other as ResultValueType") +// } +// } +// +//} + +import io.circe._ +//import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType + import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.util.UUID object SerializationUtils { - - - val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME - - implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) - implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => - Right(ZonedDateTime.parse(str, timestampFormat)) - } - - implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) - implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => - Right(UUID.fromString(str)) - } - - /** - * The method returns arbitrary object as a Json string. - * - * @return A string representing the object in Json format - */ - def asJson[T: Encoder](obj: T): String = { - obj.asJson.noSpaces - } - - /** - * The method returns arbitrary object as a pretty Json string. - * - * @return A string representing the object in Json format - */ - def asJsonPretty[T: Encoder](obj: T): String = { - obj.asJson.spaces2 - } - - /** - * The method returns arbitrary object parsed from Json string. - * - * @return An object deserialized from the Json string - */ - def fromJson[T: Decoder](jsonStr: String): T = { - decode[T](jsonStr) match { - case Right(value) => value - case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") - } - } - - sealed trait ResultValueType - object ResultValueType { - case object String extends ResultValueType - case object Long extends ResultValueType - case object BigDecimal extends ResultValueType - case object Double extends ResultValueType - - implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { - case ResultValueType.String => "String" - case ResultValueType.Long => "Long" - case ResultValueType.BigDecimal => "BigDecimal" - case ResultValueType.Double => "Double" - } - - implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { - case "String" => Right(ResultValueType.String) - case "Long" => Right(ResultValueType.Long) - case "BigDecimal" => Right(ResultValueType.BigDecimal) - case "Double" => Right(ResultValueType.Double) - case other => Left(s"Cannot decode $other as ResultValueType") - } - } +// // Timestamp format +// val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME +// +// // Implicit encoders and decoders for ZonedDateTime +// implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) +// implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => +// Right(ZonedDateTime.parse(str, timestampFormat)) +// } +// +// // Implicit encoders and decoders for UUID +// implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) +// implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => +// Right(UUID.fromString(str)) +// } +// +// // Implicit encoders and decoders for Option[String] +// implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] +// implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index a6587f296..7d0bd599c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -32,7 +32,7 @@ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ import za.co.absa.atum.model.dto.MeasureResultDTO._ -import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index f2d03fffd..e13d1dacd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} -import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.model.utils.JsonImplicits._ import sttp.tapir.{PublicEndpoint, endpoint} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index f2eb606f0..6c10db8ec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.model import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import io.circe.{DecodingFailure, Json} -import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.model.utils.JsonImplicits._ import java.time.ZonedDateTime import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala deleted file mode 100644 index 75df93885..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import io.circe._ -import io.circe.generic.semiauto._ -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType -import za.co.absa.atum.model.dto._ - -object CirceJsonImplicits { - - implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] - implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] - - implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { - case "String" => Right(ResultValueType.String) - case "Long" => Right(ResultValueType.Long) - case "BigDecimal" => Right(ResultValueType.BigDecimal) - case "Double" => Right(ResultValueType.Double) - case _ => Left("Invalid ResultValueType") - } - - implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap[ResultValueType] { - case ResultValueType.String => "String" - case ResultValueType.Long => "Long" - case ResultValueType.BigDecimal => "BigDecimal" - case ResultValueType.Double => "Double" - } - - implicit val decodeTypedValue: Decoder[MeasureResultDTO.TypedValue] = deriveDecoder - implicit val encodeTypedValue: Encoder[MeasureResultDTO.TypedValue] = deriveEncoder - - implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = deriveDecoder - implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = deriveEncoder - - implicit val decodeMeasureDTO: Decoder[MeasureDTO] = deriveDecoder - implicit val encodeMeasureDTO: Encoder[MeasureDTO] = deriveEncoder - - implicit val decodeMeasurementDTO: Decoder[MeasurementDTO] = deriveDecoder - implicit val encodeMeasurementDTO: Encoder[MeasurementDTO] = deriveEncoder - - implicit val decodePartitionDTO: Decoder[PartitionDTO] = deriveDecoder - implicit val encodePartitionDTO: Encoder[PartitionDTO] = deriveEncoder - - implicit val decodeCheckpointDTO: Decoder[CheckpointDTO] = deriveDecoder - implicit val encodeCheckpointDTO: Encoder[CheckpointDTO] = deriveEncoder - - implicit val decodePartitioningSubmitDTO: Decoder[PartitioningSubmitDTO] = deriveDecoder - implicit val encodePartitioningSubmitDTO: Encoder[PartitioningSubmitDTO] = deriveEncoder - - implicit val decodeStringMap: Decoder[Map[String, Option[String]]] = Decoder.decodeMap[String, Option[String]] - implicit val encodeStringMap: Encoder[Map[String, Option[String]]] = Encoder.encodeMap[String, Option[String]] - - implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder - implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder - - implicit val decodeAtumContextDTO: Decoder[AtumContextDTO] = deriveDecoder - implicit val encodeAtumContextDTO: Encoder[AtumContextDTO] = deriveEncoder - -} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 7fb322666..70cab0f8b 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -29,7 +29,7 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index 5f7022bd7..8e2c002ed 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -28,7 +28,7 @@ import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo From 78b6149b7fee473c3859aef829204120a7842f41 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 1 Jul 2024 15:33:22 +0200 Subject: [PATCH 14/34] fix compilation --- .../agent/dispatcher/HttpDispatcher.scala | 24 +- .../za/co/absa/atum/agent/model/Measure.scala | 2 +- .../absa/atum/agent/model/MeasureResult.scala | 2 +- .../atum/agent/model/MeasurementBuilder.scala | 4 +- .../atum/agent/AtumContextUnitTests.scala | 3 +- .../agent/model/AtumMeasureUnitTests.scala | 2 +- .../atum/agent/model/MeasureUnitTests.scala | 2 +- .../model/MeasurementBuilderUnitTests.scala | 3 +- .../atum/model/dto/MeasureResultDTO.scala | 15 +- .../utils/SerializationUtilsUnitTests.scala | 721 +++++++++--------- .../server/api/database/DoobieImplicits.scala | 25 +- .../runs/functions/WriteCheckpoint.scala | 1 - .../absa/atum/server/api/http/Endpoints.scala | 2 +- .../atum/server/model/CheckpointFromDB.scala | 1 - .../atum/server/model/SuccessResponse.scala | 13 + .../za/co/absa/atum/server/api/TestData.scala | 8 +- .../WriteCheckpointIntegrationTests.scala | 3 +- .../CreateCheckpointEndpointUnitTests.scala | 3 +- .../CreatePartitioningEndpointUnitTests.scala | 3 +- .../GetFlowCheckpointsEndpointUnitTests.scala | 2 +- 20 files changed, 437 insertions(+), 402 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index c773229e3..cfbe54aff 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -23,7 +23,9 @@ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} import za.co.absa.atum.model.utils.SerializationUtils -import io.circe.generic.auto._ +//import io.circe.generic.auto._ +import io.circe.syntax.EncoderOps +import io.circe.jawn.decode class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ @@ -48,19 +50,26 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def createPartitioning(partitioning: PartitioningSubmitDTO): AtumContextDTO = { val request = commonAtumRequest .post(createPartitioningEndpoint) - .body(SerializationUtils.asJson(partitioning)) +// .body(SerializationUtils.asJson(partitioning)) + .body(partitioning.asJson.noSpaces) val response = backend.send(request) - SerializationUtils.fromJson[AtumContextDTO]( - handleResponseBody(response) - ) +// SerializationUtils.fromJson[AtumContextDTO]( +// handleResponseBody(response) +// ) + + decode[AtumContextDTO](handleResponseBody(response)) match { + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + case Right(value) => value + } } override protected[agent] def saveCheckpoint(checkpoint: CheckpointDTO): Unit = { val request = commonAtumRequest .post(createCheckpointEndpoint) - .body(SerializationUtils.asJson(checkpoint)) +// .body(SerializationUtils.asJson(checkpoint)) + .body(checkpoint.asJson.noSpaces) val response = backend.send(request) @@ -70,7 +79,8 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def saveAdditionalData(additionalDataSubmitDTO: AdditionalDataSubmitDTO): Unit = { val request = commonAtumRequest .post(createAdditionalDataEndpoint) - .body(SerializationUtils.asJson(additionalDataSubmitDTO)) +// .body(SerializationUtils.asJson(additionalDataSubmitDTO)) + .body(additionalDataSubmitDTO.asJson.noSpaces) val response = backend.send(request) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index dd4d599fb..ad24cd21f 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.types.{DataType, DecimalType, LongType, StringType} import org.apache.spark.sql.{Column, DataFrame} import za.co.absa.atum.agent.core.MeasurementProcessor import za.co.absa.atum.agent.core.MeasurementProcessor.MeasurementFunction -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.dto.ResultValueType /** * Type of different measures to be applied to the columns. diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index f95214401..7a0f8f193 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.agent.model import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.dto.ResultValueType /** * This trait defines a contract for a measure result. diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala index 009e6df14..046042666 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.agent.model import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException -import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO} +import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO, TypedValue} /** * This object provides a functionality to convert a measurement to its DTO representation. @@ -41,7 +41,7 @@ private [agent] object MeasurementBuilder { val measureDTO = MeasureDTO(measure.measureName, measure.measuredColumns) val measureResultDTO = MeasureResultDTO( - MeasureResultDTO.TypedValue(measureResult.resultValue.toString, measureResult.resultValueType) + TypedValue(measureResult.resultValue.toString, measureResult.resultValueType) ) MeasurementDTO(measureDTO, measureResultDTO) } diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala index 67ec7b9a9..012acb261 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala @@ -25,8 +25,7 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{RecordCount, SumOfValuesOfColumn} import za.co.absa.atum.agent.model.{Measure, MeasureResult, MeasurementBuilder, UnknownMeasure} -import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.dto.{CheckpointDTO, ResultValueType} class AtumContextUnitTests extends AnyFlatSpec with Matchers { diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala index 17bd3be2a..70b1ed47e 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala @@ -23,7 +23,7 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.agent.AtumContext.{AtumPartitions, DatasetWrapper} import za.co.absa.atum.agent.model.AtumMeasure._ -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.dto.ResultValueType import za.co.absa.spark.commons.test.SparkTestBase class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala index aef1a7fe6..0f7ad68cc 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala @@ -21,9 +21,9 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{AbsSumOfValuesOfColumn, RecordCount, SumOfHashesOfColumn, SumOfValuesOfColumn} -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType import za.co.absa.spark.commons.test.SparkTestBase import za.co.absa.atum.agent.AtumContext._ +import za.co.absa.atum.model.dto.ResultValueType class MeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala index f96fefbf6..6007be390 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala @@ -18,9 +18,8 @@ package za.co.absa.atum.agent.model import org.scalatest.flatspec.AnyFlatSpec import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException -import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO} +import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO, ResultValueType, TypedValue} import za.co.absa.atum.agent.model.AtumMeasure._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} class MeasurementBuilderUnitTests extends AnyFlatSpec { diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 64d625715..026e628dd 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -48,6 +48,16 @@ case class MeasureResultDTO( supportValues: Map[String, TypedValue] = Map.empty ) +object MeasureResultDTO { + + implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = + Encoder.forProduct2("mainValue", "supportValues")(m => (m.mainValue, m.supportValues)) + + implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = + Decoder.forProduct2("mainValue", "supportValues")(MeasureResultDTO.apply) + +} + case class TypedValue( value: String, valueType: ResultValueType @@ -62,9 +72,4 @@ object TypedValue { Decoder.forProduct2("value", "valueType")(TypedValue.apply) } -object MeasureResultDTO { - - implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = - Decoder.forProduct2("mainValue", "supportValues")(MeasureResultDTO.apply) -} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index f43fa083f..eddffb0ff 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -16,9 +16,10 @@ package za.co.absa.atum.model.utils -import io.circe.generic.auto._ +//import io.circe.generic.auto._ +import io.circe.jawn.decode +import io.circe.syntax.EncoderOps import org.scalatest.flatspec.AnyFlatSpecLike -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.utils.SerializationUtilsTest.StringLinearization @@ -45,7 +46,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"additionalData":{"key1":"val1","key2":"val2","key3":null}, |"author":"testAuthor"} |""".linearize - val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) + val actualAdditionalDataJson = additionalDataDTO.asJson.noSpaces assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -66,364 +67,368 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ), "testAuthor" ) - val actualAdditionalDataDTO = SerializationUtils.fromJson[AdditionalDataSubmitDTO](additionalDataDTOJson) - - assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) - } - - "asJson" should "serialize empty AdditionalDataDTO into json string" in { - val additionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - - val expectedAdditionalDataJson = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" - val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) - - assert(actualAdditionalDataJson == expectedAdditionalDataJson) - } - - "fromJson" should "deserialize empty AdditionalDataDTO from json string" in { - val additionalDataDTOJsonString = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" - - val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - - val actualAdditionalDataDTO = SerializationUtils.fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) - - assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) - } - - // AtumContextDTO - "asJson" should "serialize AtumContextDTO into json string" in { - val seqPartitionDTO = Seq(PartitionDTO("key", "val")) - val seqMeasureDTO = Set(MeasureDTO("count", Seq("col"))) - - val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - - val expectedAdditionalDataJson = - """ - |{ - |"partitioning":[{"key":"key","value":"val"}], - |"measures":[{"measureName":"count","measuredColumns":["col"]}], - |"additionalData":{} - |}""".linearize - val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO) - - assert(actualAdditionalDataJson == expectedAdditionalDataJson) - } - - "fromJson" should "deserialize AtumContextDTO from json string" in { - val atumContextDTOJson = - """ - |{ - |"partitioning":[{"key":"key","value":"val"}], - |"measures":[{"measureName":"count","measuredColumns":["col"]}], - |"additionalData":{} - |}""".linearize - - val seqPartitionDTO = Seq(PartitionDTO("key", "val")) - val seqMeasureDTO = Set(MeasureDTO("count", Seq("col"))) - - val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - - val actualAtumContextDTO = SerializationUtils.fromJson[AtumContextDTO](atumContextDTOJson) - - assert(actualAtumContextDTO == expectedAtumContextDTO) - } - - "asJson" should "serialize AtumContextDTO without measures into json string" in { - val seqPartitionDTO = Seq(PartitionDTO("key", "val")) - - val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO) - - val expectedAdditionalDataJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" - val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO) - - assert(actualAdditionalDataJson == expectedAdditionalDataJson) - } - - "fromJson" should "deserialize AtumContextDTO without measures from json string" in { - val atumContextDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" - - val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) - - val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) - val actualAtumContextDTO = SerializationUtils.fromJson[AtumContextDTO](atumContextDTOJson) - - assert(actualAtumContextDTO == expectedAtumContextDTO) - } - - // CheckpointDTO - "asJson" should "serialize CheckpointDTO into json string" in { - val uuid = UUID.randomUUID() - val seqPartitionDTO = Seq(PartitionDTO("key", "val")) - val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneId.of("CET")) - - val setMeasurementDTO = Set( - MeasurementDTO( - measure = MeasureDTO("count", Seq("col")), result = MeasureResultDTO( - mainValue = TypedValue("1", ResultValueType.Long) - ) - ) - ) - - val checkpointDTO = CheckpointDTO( - id = uuid, - name = "checkpoint", - author = "author", - measuredByAtumAgent = true, - partitioning = seqPartitionDTO, - processStartTime = timeWithZone, - processEndTime = Some(timeWithZone), - measurements = setMeasurementDTO - ) - - val expectedCheckpointDTOJson = - s""" - |{ - |"id":"$uuid", - |"name":"checkpoint", - |"author":"author", - |"measuredByAtumAgent":true, - |"partitioning":[{"key":"key","value":"val"}], - |"processStartTime":"2023-10-24T10:20:59.005+02:00[CET]", - |"processEndTime":"2023-10-24T10:20:59.005+02:00[CET]", - |"measurements":[{"measure":{"measureName":"count","measuredColumns":["col"]}, - |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] - |} - |""".linearize - val actualCheckpointDTOJson = SerializationUtils.asJson(checkpointDTO) - - assert(actualCheckpointDTOJson == expectedCheckpointDTOJson) - } - - "fromJson" should "deserialize CheckpointDTO from json string" in { - val uuid = UUID.randomUUID() - val seqPartitionDTO = Seq(PartitionDTO("key", "val")) - val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneOffset.ofHours(2)) - - val checkpointDTOJson = - s""" - |{ - |"id":"$uuid", - |"name":"checkpoint", - |"author":"author", - |"measuredByAtumAgent":true, - |"partitioning":[{"key":"key","value":"val"}], - |"processStartTime":"2023-10-24T10:20:59.005+02:00", - |"processEndTime":"2023-10-24T10:20:59.005+02:00", - |"measurements":[{"measure":{"measureName":"count","measuredColumns":["col"]}, - |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] - |} - |""".linearize - - - val setMeasurementDTO = Set( - MeasurementDTO( - measure = MeasureDTO("count", Seq("col")), result = MeasureResultDTO( - mainValue = TypedValue("1", ResultValueType.Long) - ) - ) - ) - - val expectedCheckpointDTO = CheckpointDTO( - id = uuid, - name = "checkpoint", - author = "author", - measuredByAtumAgent = true, - partitioning = seqPartitionDTO, - processStartTime = timeWithZone, - processEndTime = Some(timeWithZone), - measurements = setMeasurementDTO - ) - - val actualCheckpointDTO = SerializationUtils.fromJson[CheckpointDTO](checkpointDTOJson) - - assert(actualCheckpointDTO == expectedCheckpointDTO) - } - - // MeasureDTO - "asJson" should "serialize MeasureDTO into json string" in { - val measureDTO = MeasureDTO("count", Seq("col")) - - val expectedMeasureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" - val actualMeasureDTOJson = SerializationUtils.asJson(measureDTO) - - assert(actualMeasureDTOJson == expectedMeasureDTOJson) - } - - "fromJson" should "deserialize MeasureDTO from json string" in { - val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" - - val expectedMeasureDTO = MeasureDTO("count", Seq("col")) - val actualMeasureDTO = SerializationUtils.fromJson[MeasureDTO](measureDTOJson) - - assert(actualMeasureDTO == expectedMeasureDTO) - } - - // MeasurementDTO - "asJson" should "serialize MeasurementDTO into json string" in { - val measureDTO = MeasureDTO("count", Seq("col")) - val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) - - val measurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - - val expectedMeasurementDTOJson = - """ - |{ - |"measure":{"measureName":"count","measuredColumns":["col"]}, - |"result":{"mainValue":{"value":"1","valueType":"Long"}, - |"supportValues":{}} - |} - |""".linearize - val actualMeasurementDTOJson = SerializationUtils.asJson(measurementDTO) - - assert(actualMeasurementDTOJson == expectedMeasurementDTOJson) - } - - "fromJson" should "deserialize MeasurementDTO from json string" in { - val measurementDTOJson = - """ - |{ - |"measure":{"measureName":"count","measuredColumns":["col"]}, - |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}} - |} - |""".stripMargin - - val measureDTO = MeasureDTO("count", Seq("col")) - val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) - - val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - val actualMeasurementDTO = SerializationUtils.fromJson[MeasurementDTO](measurementDTOJson) - - assert(actualMeasurementDTO == expectedMeasurementDTO) - } - - // MeasureResultDTO - "asJson" should "serialize MeasureResultDTO into json string" in { - val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) - - val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" - val actualMeasureResultDTOJson = SerializationUtils.asJson(measureResultDTO) - - assert(actualMeasureResultDTOJson == expectedMeasureResultDTOJson) - } - - "fromJson" should "deserialize MeasureResultDTO from json string" in { - val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" - - val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) - val actualMeasureResultDTO = SerializationUtils.fromJson[MeasureResultDTO](measureResultDTOJson) - - assert(actualMeasureResultDTO == expectedMeasureResultDTO) - } - - // PartitionDTO - "asJson" should "serialize PartitionDTO into json string" in { - val partitionDTO = PartitionDTO("key", "val") - - val expectedPartitionDTOJson = """{"key":"key","value":"val"}""" - val actualPartitionDTOJson = SerializationUtils.asJson(partitionDTO) - - assert(actualPartitionDTOJson == expectedPartitionDTOJson) - } - - "fromJson" should "deserialize PartitionDTO from json string" in { - val partitionDTOJson = """{"key":"key","value":"val"}""" - - val expectedPartitionDTO = PartitionDTO("key", "val") - val actualPartitionDTO = SerializationUtils.fromJson[PartitionDTO](partitionDTOJson) - - assert(actualPartitionDTO == expectedPartitionDTO) - } - - // PartitioningDTO - "asJson" should "serialize PartitioningDTO into json string" in { - val partitionDTO = PartitionDTO("key", "val") - - val partitioningDTO = PartitioningSubmitDTO( - partitioning = Seq(partitionDTO), - parentPartitioning = None, - authorIfNew = "authorTest" - ) - - val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" - val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) - - assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) - } - - "fromJson" should "deserialize PartitioningDTO from json string" in { - val partitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"authorIfNew":"authorTest"}""" - - val expectedPartitionDTO = PartitionDTO("key", "val") - val expectedPartitioningDTO = PartitioningSubmitDTO( - partitioning = Seq(expectedPartitionDTO), - parentPartitioning = None, - authorIfNew = "authorTest" - ) - - val actualPartitioningDTO = SerializationUtils.fromJson[PartitioningSubmitDTO](partitioningDTOJson) - - assert(actualPartitioningDTO == expectedPartitioningDTO) - } - - "asJson" should "serialize PartitioningDTO with parent partitioning into json string" in { - val partitionDTO = PartitionDTO("key", "val") - val parentPartitionDTO = PartitionDTO("parentKey", "parentVal") - - val partitioningDTO = PartitioningSubmitDTO( - partitioning = Seq(partitionDTO), - parentPartitioning = Some(Seq(parentPartitionDTO)), - authorIfNew = "authorTest" - ) - - val expectedPartitioningDTOJson = - """ - |{ - |"partitioning":[{"key":"key","value":"val"}], - |"parentPartitioning":[{"key":"parentKey","value":"parentVal"}], - |"authorIfNew":"authorTest" - |} - |""".linearize - val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) - - assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) - } - - - "asJson" should "serialize Seq[PartitionDTO] into json string" in { - val partitionDTO = Seq( - PartitionDTO("key1", "val1"), - PartitionDTO("key2", "val2"), - PartitionDTO("key3", "val3") - ) - - val expectedPartitionDTOJson = """[{"key":"key1","value":"val1"},{"key":"key2","value":"val2"},{"key":"key3","value":"val3"}]""" - val actualPartitionDTOJson = SerializationUtils.asJson(partitionDTO) +// val actualAdditionalDataDTO = + decode[AdditionalDataSubmitDTO](additionalDataDTOJson) match { + case Left(_) => fail() + case Right(value) => assert(value == expectedAdditionalDataDTO) + } - assert(actualPartitionDTOJson == expectedPartitionDTOJson) +// assert(Right(expectedAdditionalDataDTO)) } - "fromJson" should "deserialize PartitioningDTO with parent partitioning from json string" in { - val partitioningDTOJson = - """ - |{ - |"partitioning":[{"key":"key","value":"val"}], - |"parentPartitioning":[{"key":"parentKey","value":"parentVal"}], - |"authorIfNew":"authorTest" - |} - |""".linearize - - val expectedPartitionDTO = PartitionDTO("key", "val") - val expectedParentPartitionDTO = PartitionDTO("parentKey", "parentVal") - val expectedPartitioningDTO = PartitioningSubmitDTO( - partitioning = Seq(expectedPartitionDTO), - parentPartitioning = Some(Seq(expectedParentPartitionDTO)), - authorIfNew = "authorTest" - ) - - val actualPartitioningDTO = SerializationUtils.fromJson[PartitioningSubmitDTO](partitioningDTOJson) - - assert(actualPartitioningDTO == expectedPartitioningDTO) - } +// "asJson" should "serialize empty AdditionalDataDTO into json string" in { +// val additionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") +// +// val expectedAdditionalDataJson = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" +// val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) +// +// assert(actualAdditionalDataJson == expectedAdditionalDataJson) +// } +// +// "fromJson" should "deserialize empty AdditionalDataDTO from json string" in { +// val additionalDataDTOJsonString = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" +// +// val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") +// +// val actualAdditionalDataDTO = SerializationUtils.fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) +// +// assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) +// } +// +// // AtumContextDTO +// "asJson" should "serialize AtumContextDTO into json string" in { +// val seqPartitionDTO = Seq(PartitionDTO("key", "val")) +// val seqMeasureDTO = Set(MeasureDTO("count", Seq("col"))) +// +// val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) +// +// val expectedAdditionalDataJson = +// """ +// |{ +// |"partitioning":[{"key":"key","value":"val"}], +// |"measures":[{"measureName":"count","measuredColumns":["col"]}], +// |"additionalData":{} +// |}""".linearize +// val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO) +// +// assert(actualAdditionalDataJson == expectedAdditionalDataJson) +// } +// +// "fromJson" should "deserialize AtumContextDTO from json string" in { +// val atumContextDTOJson = +// """ +// |{ +// |"partitioning":[{"key":"key","value":"val"}], +// |"measures":[{"measureName":"count","measuredColumns":["col"]}], +// |"additionalData":{} +// |}""".linearize +// +// val seqPartitionDTO = Seq(PartitionDTO("key", "val")) +// val seqMeasureDTO = Set(MeasureDTO("count", Seq("col"))) +// +// val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) +// +// val actualAtumContextDTO = SerializationUtils.fromJson[AtumContextDTO](atumContextDTOJson) +// +// assert(actualAtumContextDTO == expectedAtumContextDTO) +// } +// +// "asJson" should "serialize AtumContextDTO without measures into json string" in { +// val seqPartitionDTO = Seq(PartitionDTO("key", "val")) +// +// val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO) +// +// val expectedAdditionalDataJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" +// val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO) +// +// assert(actualAdditionalDataJson == expectedAdditionalDataJson) +// } +// +// "fromJson" should "deserialize AtumContextDTO without measures from json string" in { +// val atumContextDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" +// +// val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) +// +// val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) +// val actualAtumContextDTO = SerializationUtils.fromJson[AtumContextDTO](atumContextDTOJson) +// +// assert(actualAtumContextDTO == expectedAtumContextDTO) +// } +// +// // CheckpointDTO +// "asJson" should "serialize CheckpointDTO into json string" in { +// val uuid = UUID.randomUUID() +// val seqPartitionDTO = Seq(PartitionDTO("key", "val")) +// val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneId.of("CET")) +// +// val setMeasurementDTO = Set( +// MeasurementDTO( +// measure = MeasureDTO("count", Seq("col")), result = MeasureResultDTO( +// mainValue = TypedValue("1", ResultValueType.Long) +// ) +// ) +// ) +// +// val checkpointDTO = CheckpointDTO( +// id = uuid, +// name = "checkpoint", +// author = "author", +// measuredByAtumAgent = true, +// partitioning = seqPartitionDTO, +// processStartTime = timeWithZone, +// processEndTime = Some(timeWithZone), +// measurements = setMeasurementDTO +// ) +// +// val expectedCheckpointDTOJson = +// s""" +// |{ +// |"id":"$uuid", +// |"name":"checkpoint", +// |"author":"author", +// |"measuredByAtumAgent":true, +// |"partitioning":[{"key":"key","value":"val"}], +// |"processStartTime":"2023-10-24T10:20:59.005+02:00[CET]", +// |"processEndTime":"2023-10-24T10:20:59.005+02:00[CET]", +// |"measurements":[{"measure":{"measureName":"count","measuredColumns":["col"]}, +// |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] +// |} +// |""".linearize +// val actualCheckpointDTOJson = checkpointDTO.asJson(checkpointDTO) +// +// assert(actualCheckpointDTOJson == expectedCheckpointDTOJson) +// } +// +// "fromJson" should "deserialize CheckpointDTO from json string" in { +// val uuid = UUID.randomUUID() +// val seqPartitionDTO = Seq(PartitionDTO("key", "val")) +// val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneOffset.ofHours(2)) +// +// val checkpointDTOJson = +// s""" +// |{ +// |"id":"$uuid", +// |"name":"checkpoint", +// |"author":"author", +// |"measuredByAtumAgent":true, +// |"partitioning":[{"key":"key","value":"val"}], +// |"processStartTime":"2023-10-24T10:20:59.005+02:00", +// |"processEndTime":"2023-10-24T10:20:59.005+02:00", +// |"measurements":[{"measure":{"measureName":"count","measuredColumns":["col"]}, +// |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] +// |} +// |""".linearize +// +// +// val setMeasurementDTO = Set( +// MeasurementDTO( +// measure = MeasureDTO("count", Seq("col")), result = MeasureResultDTO( +// mainValue = TypedValue("1", ResultValueType.Long) +// ) +// ) +// ) +// +// val expectedCheckpointDTO = CheckpointDTO( +// id = uuid, +// name = "checkpoint", +// author = "author", +// measuredByAtumAgent = true, +// partitioning = seqPartitionDTO, +// processStartTime = timeWithZone, +// processEndTime = Some(timeWithZone), +// measurements = setMeasurementDTO +// ) +// +// val actualCheckpointDTO = SerializationUtils.fromJson[CheckpointDTO](checkpointDTOJson) +// +// assert(actualCheckpointDTO == expectedCheckpointDTO) +// } +// +// // MeasureDTO +// "asJson" should "serialize MeasureDTO into json string" in { +// val measureDTO = MeasureDTO("count", Seq("col")) +// +// val expectedMeasureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" +// val actualMeasureDTOJson = SerializationUtils.asJson(measureDTO) +// +// assert(actualMeasureDTOJson == expectedMeasureDTOJson) +// } +// +// "fromJson" should "deserialize MeasureDTO from json string" in { +// val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" +// +// val expectedMeasureDTO = MeasureDTO("count", Seq("col")) +// val actualMeasureDTO = SerializationUtils.fromJson[MeasureDTO](measureDTOJson) +// +// assert(actualMeasureDTO == expectedMeasureDTO) +// } +// +// // MeasurementDTO +// "asJson" should "serialize MeasurementDTO into json string" in { +// val measureDTO = MeasureDTO("count", Seq("col")) +// val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) +// +// val measurementDTO = MeasurementDTO(measureDTO, measureResultDTO) +// +// val expectedMeasurementDTOJson = +// """ +// |{ +// |"measure":{"measureName":"count","measuredColumns":["col"]}, +// |"result":{"mainValue":{"value":"1","valueType":"Long"}, +// |"supportValues":{}} +// |} +// |""".linearize +// val actualMeasurementDTOJson = SerializationUtils.asJson(measurementDTO) +// +// assert(actualMeasurementDTOJson == expectedMeasurementDTOJson) +// } +// +// "fromJson" should "deserialize MeasurementDTO from json string" in { +// val measurementDTOJson = +// """ +// |{ +// |"measure":{"measureName":"count","measuredColumns":["col"]}, +// |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}} +// |} +// |""".stripMargin +// +// val measureDTO = MeasureDTO("count", Seq("col")) +// val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) +// +// val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) +// val actualMeasurementDTO = SerializationUtils.fromJson[MeasurementDTO](measurementDTOJson) +// +// assert(actualMeasurementDTO == expectedMeasurementDTO) +// } +// +// // MeasureResultDTO +// "asJson" should "serialize MeasureResultDTO into json string" in { +// val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) +// +// val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" +// val actualMeasureResultDTOJson = SerializationUtils.asJson(measureResultDTO) +// +// assert(actualMeasureResultDTOJson == expectedMeasureResultDTOJson) +// } +// +// "fromJson" should "deserialize MeasureResultDTO from json string" in { +// val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" +// +// val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) +// val actualMeasureResultDTO = SerializationUtils.fromJson[MeasureResultDTO](measureResultDTOJson) +// +// assert(actualMeasureResultDTO == expectedMeasureResultDTO) +// } +// +// // PartitionDTO +// "asJson" should "serialize PartitionDTO into json string" in { +// val partitionDTO = PartitionDTO("key", "val") +// +// val expectedPartitionDTOJson = """{"key":"key","value":"val"}""" +// val actualPartitionDTOJson = SerializationUtils.asJson(partitionDTO) +// +// assert(actualPartitionDTOJson == expectedPartitionDTOJson) +// } +// +// "fromJson" should "deserialize PartitionDTO from json string" in { +// val partitionDTOJson = """{"key":"key","value":"val"}""" +// +// val expectedPartitionDTO = PartitionDTO("key", "val") +// val actualPartitionDTO = SerializationUtils.fromJson[PartitionDTO](partitionDTOJson) +// +// assert(actualPartitionDTO == expectedPartitionDTO) +// } +// +// // PartitioningDTO +// "asJson" should "serialize PartitioningDTO into json string" in { +// val partitionDTO = PartitionDTO("key", "val") +// +// val partitioningDTO = PartitioningSubmitDTO( +// partitioning = Seq(partitionDTO), +// parentPartitioning = None, +// authorIfNew = "authorTest" +// ) +// +// val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" +// val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) +// +// assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) +// } +// +// "fromJson" should "deserialize PartitioningDTO from json string" in { +// val partitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"authorIfNew":"authorTest"}""" +// +// val expectedPartitionDTO = PartitionDTO("key", "val") +// val expectedPartitioningDTO = PartitioningSubmitDTO( +// partitioning = Seq(expectedPartitionDTO), +// parentPartitioning = None, +// authorIfNew = "authorTest" +// ) +// +// val actualPartitioningDTO = SerializationUtils.fromJson[PartitioningSubmitDTO](partitioningDTOJson) +// +// assert(actualPartitioningDTO == expectedPartitioningDTO) +// } +// +// "asJson" should "serialize PartitioningDTO with parent partitioning into json string" in { +// val partitionDTO = PartitionDTO("key", "val") +// val parentPartitionDTO = PartitionDTO("parentKey", "parentVal") +// +// val partitioningDTO = PartitioningSubmitDTO( +// partitioning = Seq(partitionDTO), +// parentPartitioning = Some(Seq(parentPartitionDTO)), +// authorIfNew = "authorTest" +// ) +// +// val expectedPartitioningDTOJson = +// """ +// |{ +// |"partitioning":[{"key":"key","value":"val"}], +// |"parentPartitioning":[{"key":"parentKey","value":"parentVal"}], +// |"authorIfNew":"authorTest" +// |} +// |""".linearize +// val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) +// +// assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) +// } +// +// +// "asJson" should "serialize Seq[PartitionDTO] into json string" in { +// val partitionDTO = Seq( +// PartitionDTO("key1", "val1"), +// PartitionDTO("key2", "val2"), +// PartitionDTO("key3", "val3") +// ) +// +// val expectedPartitionDTOJson = """[{"key":"key1","value":"val1"},{"key":"key2","value":"val2"},{"key":"key3","value":"val3"}]""" +// val actualPartitionDTOJson = SerializationUtils.asJson(partitionDTO) +// +// assert(actualPartitionDTOJson == expectedPartitionDTOJson) +// } +// +// "fromJson" should "deserialize PartitioningDTO with parent partitioning from json string" in { +// val partitioningDTOJson = +// """ +// |{ +// |"partitioning":[{"key":"key","value":"val"}], +// |"parentPartitioning":[{"key":"parentKey","value":"parentVal"}], +// |"authorIfNew":"authorTest" +// |} +// |""".linearize +// +// val expectedPartitionDTO = PartitionDTO("key", "val") +// val expectedParentPartitionDTO = PartitionDTO("parentKey", "parentVal") +// val expectedPartitioningDTO = PartitioningSubmitDTO( +// partitioning = Seq(expectedPartitionDTO), +// parentPartitioning = Some(Seq(expectedParentPartitionDTO)), +// authorIfNew = "authorTest" +// ) +// +// val actualPartitioningDTO = SerializationUtils.fromJson[PartitioningSubmitDTO](partitioningDTOJson) +// +// assert(actualPartitioningDTO == expectedPartitioningDTO) +// } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 3a1969ed2..6d8f6d312 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -34,6 +34,17 @@ object DoobieImplicits { implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] .tmap(map => map.map { case (k, v) => k -> Option(v) }) + private def circeJsonListToPGJsonArrayString(jsonList: List[Json]): String = { + val arrayElements = jsonList.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s"\"$escapedJsonString\"" + } + + arrayElements.mkString("{", ",", "}") + } + object Sequence { implicit val get: Get[Seq[String]] = Get[List[String]].map(_.toSeq) @@ -51,14 +62,14 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - val arrayElements = a.map { x => - // Convert to compact JSON string and escape inner quotes - val escapedJsonString = x.noSpaces.replace("\"", "\\\"") - // Wrap in double quotes for the array element - s"\"$escapedJsonString\"" - } +// val arrayElements = a.map { x => +// // Convert to compact JSON string and escape inner quotes +// val escapedJsonString = x.noSpaces.replace("\"", "\\\"") +// // Wrap in double quotes for the array element +// s"\"$escapedJsonString\"" +// } // Join all elements into a single string wrapped in curly braces - o.setValue(arrayElements.mkString("{", ",", "}")) + o.setValue(circeJsonListToPGJsonArrayString(a)) o } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 7d0bd599c..f027ef411 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -32,7 +32,6 @@ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ import za.co.absa.atum.model.dto.MeasureResultDTO._ -import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index e13d1dacd..f38dd610c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} -import za.co.absa.atum.model.utils.JsonImplicits._ + import sttp.tapir.{PublicEndpoint, endpoint} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 6c10db8ec..705e6c319 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -18,7 +18,6 @@ package za.co.absa.atum.server.model import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import io.circe.{DecodingFailure, Json} -import za.co.absa.atum.model.utils.JsonImplicits._ import java.time.ZonedDateTime import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala index 8ab99255d..f3898cc66 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/SuccessResponse.scala @@ -16,6 +16,9 @@ package za.co.absa.atum.server.model +import io.circe._ +import io.circe.generic.semiauto._ + import java.util.UUID object SuccessResponse { @@ -25,7 +28,17 @@ object SuccessResponse { case class SingleSuccessResponse[T](data: T, requestId: UUID = UUID.randomUUID()) extends SuccessResponse + object SingleSuccessResponse { + implicit def encoder[T: Encoder]: Encoder[SingleSuccessResponse[T]] = deriveEncoder + implicit def decoder[T: Decoder]: Decoder[SingleSuccessResponse[T]] = deriveDecoder + } + case class MultiSuccessResponse[T](data: Seq[T], requestId: UUID = UUID.randomUUID()) extends SuccessResponse + object MultiSuccessResponse { + implicit def encoder[T: Encoder]: Encoder[MultiSuccessResponse[T]] = deriveEncoder + implicit def decoder[T: Decoder]: Decoder[MultiSuccessResponse[T]] = deriveDecoder + } + } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 2fa568e28..bc2daab8f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -22,8 +22,6 @@ import za.co.absa.atum.server.model.CheckpointFromDB import java.time.ZonedDateTime import java.util.UUID -import MeasureResultDTO.TypedValue -import MeasureResultDTO.ResultValueType._ trait TestData { @@ -73,17 +71,17 @@ trait TestData { val mainValue: TypedValue = TypedValue( value = "123", - valueType = Long + valueType = ResultValueType.Long ) val supportValue1: TypedValue = TypedValue( value = "123456789", - valueType = Long + valueType = ResultValueType.Long ) val supportValue2: TypedValue = TypedValue( value = "12345.6789", - valueType = BigDecimal + valueType = ResultValueType.BigDecimal ) // Measure Result DTO diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 2c655623f..e3bdf52fc 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -16,8 +16,7 @@ package za.co.absa.atum.server.api.database.runs.functions -import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO, ResultValueType, TypedValue} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 70cab0f8b..b14a1d011 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.http -import io.circe.generic.auto._ +//import io.circe.generic.auto._ import io.circe.syntax.EncoderOps import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub @@ -29,7 +29,6 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index 8e2c002ed..cbc889e76 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.http -import io.circe.generic.auto._ +//import io.circe.generic.auto._ import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub import sttp.client3.{UriContext, basicRequest} @@ -28,7 +28,6 @@ import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala index 7b0ae8a26..601cacbe0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.server.api.http -import io.circe.generic.auto._ +//import io.circe.generic.auto._ import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub import sttp.client3.{UriContext, basicRequest} From 9adff685ceae95cdcbf615899c37d6c9feaea3e5 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 16:36:03 +0200 Subject: [PATCH 15/34] Fixing compilation --- .../agent/dispatcher/HttpDispatcher.scala | 11 +- .../za/co/absa/atum/agent/model/Measure.scala | 2 +- .../absa/atum/agent/model/MeasureResult.scala | 2 +- .../atum/agent/AtumContextUnitTests.scala | 2 +- .../agent/model/AtumMeasureUnitTests.scala | 2 +- .../atum/agent/model/MeasureUnitTests.scala | 2 +- .../model/MeasurementBuilderUnitTests.scala | 3 +- .../absa/atum/model/dto/CheckpointDTO.scala | 4 - .../atum/model/dto/MeasureResultDTO.scala | 59 +++------ .../za/co/absa/atum/model/dto/package.scala | 4 - .../absa/atum/model/utils/JsonImplicits.scala | 104 --------------- .../atum/model/utils/OptionImplicits.scala | 31 ----- .../atum/model/utils/SerializationUtils.scala | 124 ------------------ .../model/utils/ZonedDateTimeSerializer.scala | 34 ----- .../utils/SerializationUtilsUnitTests.scala | 50 +++---- .../runs/functions/WriteCheckpoint.scala | 1 - .../absa/atum/server/api/http/Endpoints.scala | 1 - .../atum/server/model/CheckpointFromDB.scala | 1 - .../atum/server/model/PartitioningForDB.scala | 9 +- .../za/co/absa/atum/server/api/TestData.scala | 12 +- .../WriteCheckpointIntegrationTests.scala | 3 +- .../CreateCheckpointEndpointUnitTests.scala | 1 - .../CreatePartitioningEndpointUnitTests.scala | 1 - 23 files changed, 66 insertions(+), 397 deletions(-) delete mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/OptionImplicits.scala delete mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala delete mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index c773229e3..2414d2880 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -22,8 +22,9 @@ import sttp.client3._ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} -import za.co.absa.atum.model.utils.SerializationUtils import io.circe.generic.auto._ +import io.circe.syntax.EncoderOps +import za.co.absa.atum.model.utils.JsonUtils.fromJson class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ @@ -48,11 +49,11 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def createPartitioning(partitioning: PartitioningSubmitDTO): AtumContextDTO = { val request = commonAtumRequest .post(createPartitioningEndpoint) - .body(SerializationUtils.asJson(partitioning)) + .body(partitioning.asJson.noSpaces) val response = backend.send(request) - SerializationUtils.fromJson[AtumContextDTO]( + fromJson[AtumContextDTO]( handleResponseBody(response) ) } @@ -60,7 +61,7 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def saveCheckpoint(checkpoint: CheckpointDTO): Unit = { val request = commonAtumRequest .post(createCheckpointEndpoint) - .body(SerializationUtils.asJson(checkpoint)) + .body(checkpoint.asJson.noSpaces) val response = backend.send(request) @@ -70,7 +71,7 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def saveAdditionalData(additionalDataSubmitDTO: AdditionalDataSubmitDTO): Unit = { val request = commonAtumRequest .post(createAdditionalDataEndpoint) - .body(SerializationUtils.asJson(additionalDataSubmitDTO)) + .body(additionalDataSubmitDTO.asJson.noSpaces) val response = backend.send(request) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index dd4d599fb..3d2889899 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.types.{DataType, DecimalType, LongType, StringType} import org.apache.spark.sql.{Column, DataFrame} import za.co.absa.atum.agent.core.MeasurementProcessor import za.co.absa.atum.agent.core.MeasurementProcessor.MeasurementFunction -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.utils.ResultValueType /** * Type of different measures to be applied to the columns. diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index f95214401..75bfb6859 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.agent.model import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.utils.ResultValueType /** * This trait defines a contract for a measure result. diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala index 67ec7b9a9..95d926fbe 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{RecordCount, SumOfValuesOfColumn} import za.co.absa.atum.agent.model.{Measure, MeasureResult, MeasurementBuilder, UnknownMeasure} import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.utils.ResultValueType class AtumContextUnitTests extends AnyFlatSpec with Matchers { diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala index 17bd3be2a..93741b0d1 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala @@ -23,7 +23,7 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.agent.AtumContext.{AtumPartitions, DatasetWrapper} import za.co.absa.atum.agent.model.AtumMeasure._ -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType +import za.co.absa.atum.model.utils.ResultValueType import za.co.absa.spark.commons.test.SparkTestBase class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala index aef1a7fe6..df9808d11 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala @@ -21,9 +21,9 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{AbsSumOfValuesOfColumn, RecordCount, SumOfHashesOfColumn, SumOfValuesOfColumn} -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType import za.co.absa.spark.commons.test.SparkTestBase import za.co.absa.atum.agent.AtumContext._ +import za.co.absa.atum.model.utils.ResultValueType class MeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala index f96fefbf6..f2f7e1776 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala @@ -20,7 +20,8 @@ import org.scalatest.flatspec.AnyFlatSpec import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO} import za.co.absa.atum.agent.model.AtumMeasure._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.utils.ResultValueType class MeasurementBuilderUnitTests extends AnyFlatSpec { diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala index 674636954..5b5c0daa4 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointDTO.scala @@ -22,10 +22,6 @@ import io.circe._ import java.time.ZonedDateTime import java.util.UUID -import za.co.absa.atum.model.dto._ -//import za.co.absa.atum.model.dto.MeasurementDTO._ -//import za.co.absa.atum.model.dto.PartitionDTO._ - case class CheckpointDTO( id: UUID, name: String, diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 64d625715..c33ebe19e 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -17,54 +17,25 @@ package za.co.absa.atum.model.dto import io.circe._ +import io.circe.generic.semiauto._ +import za.co.absa.atum.model.utils.ResultValueType -sealed trait ResultValueType - -object ResultValueType { - case object String extends ResultValueType - - case object Long extends ResultValueType - case object BigDecimal extends ResultValueType - case object Double extends ResultValueType - - implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { - case ResultValueType.String => "String" - case ResultValueType.Long => "Long" - case ResultValueType.BigDecimal => "BigDecimal" - case ResultValueType.Double => "Double" - } - - implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { - case "String" => Right(ResultValueType.String) - case "Long" => Right(ResultValueType.Long) - case "BigDecimal" => Right(ResultValueType.BigDecimal) - case "Double" => Right(ResultValueType.Double) - case other => Left(s"Cannot decode $other as ResultValueType") - } - -} case class MeasureResultDTO( - mainValue: TypedValue, - supportValues: Map[String, TypedValue] = Map.empty + mainValue: MeasureResultDTO.TypedValue, + supportValues: Map[String, MeasureResultDTO.TypedValue] = Map.empty ) -case class TypedValue( - value: String, - valueType: ResultValueType - ) - -object TypedValue { - - implicit val encodeTypedValue: Encoder[TypedValue] = - Encoder.forProduct2("value", "valueType")(tv => (tv.value, tv.valueType)) - - implicit val decodeTypedValue: Decoder[TypedValue] = - Decoder.forProduct2("value", "valueType")(TypedValue.apply) - -} object MeasureResultDTO { + case class TypedValue( + value: String, + valueType: ResultValueType + ) + + object TypedValue { + implicit val encodeTypedValue: Encoder[TypedValue] = deriveEncoder + implicit val decodeTypedValue: Decoder[TypedValue] = deriveDecoder + } - implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = - Decoder.forProduct2("mainValue", "supportValues")(MeasureResultDTO.apply) - + implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = deriveEncoder + implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = deriveDecoder } diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala index 810ec42e7..bca1a6c77 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/package.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/package.scala @@ -24,10 +24,6 @@ package object dto { type PartitioningDTO = Seq[PartitionDTO] type AdditionalDataDTO = Map[String, Option[String]] - // Implicit encoders and decoders for PartitioningDTO - implicit val decodePartitioningDTO: Decoder[PartitioningDTO] = Decoder.decodeSeq[PartitionDTO] - implicit val encodePartitioningDTO: Encoder[PartitioningDTO] = Encoder.encodeSeq[PartitionDTO] - // Implicit encoders and decoders for AdditionalDataDTO implicit val decodeAdditionalDataDTO: Decoder[AdditionalDataDTO] = Decoder.decodeMap[String, Option[String]] implicit val encodeAdditionalDataDTO: Encoder[AdditionalDataDTO] = Encoder.encodeMap[String, Option[String]] diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala index efb4446cd..b68b4087a 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala @@ -16,110 +16,6 @@ package za.co.absa.atum.model.utils - -//import io.circe.{Decoder, Encoder} -//import io.circe.generic.semiauto._ -//import io.circe.syntax._ -//import za.co.absa.atum.model.dto._ -// -//import java.time.ZonedDateTime -//import java.time.format.DateTimeFormatter -//import java.util.UUID - -//object JsonImplicits { -// -// // Timestamp format -// val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME -// -// // Implicit encoders and decoders for ZonedDateTime -// implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) -// implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => -// Right(ZonedDateTime.parse(str, timestampFormat)) -// } -// -// // Implicit encoders and decoders for UUID -// implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) -// implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => -// Right(UUID.fromString(str)) -// } -// -// // Implicit encoders and decoders for Option[String] -// implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] -// implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] -// -// // Implicit encoders and decoders for ResultValueType -// sealed trait ResultValueType -// object ResultValueType { -// case object String extends ResultValueType -// case object Long extends ResultValueType -// case object BigDecimal extends ResultValueType -// case object Double extends ResultValueType -// -// implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { -// case ResultValueType.String => "String" -// case ResultValueType.Long => "Long" -// case ResultValueType.BigDecimal => "BigDecimal" -// case ResultValueType.Double => "Double" -// } -// -// implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { -// case "String" => Right(ResultValueType.String) -// case "Long" => Right(ResultValueType.Long) -// case "BigDecimal" => Right(ResultValueType.BigDecimal) -// case "Double" => Right(ResultValueType.Double) -// case other => Left(s"Cannot decode $other as ResultValueType") -// } -// } -// -// // Implicit encoders and decoders for various DTOs -// implicit val decodeTypedValue: Decoder[MeasureResultDTO.TypedValue] = deriveDecoder -// implicit val encodeTypedValue: Encoder[MeasureResultDTO.TypedValue] = deriveEncoder -// -// implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = deriveDecoder -// implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = deriveEncoder -// -// implicit val decodeMeasureDTO: Decoder[MeasureDTO] = deriveDecoder -// implicit val encodeMeasureDTO: Encoder[MeasureDTO] = deriveEncoder -// -// implicit val decodeMeasurementDTO: Decoder[MeasurementDTO] = deriveDecoder -// implicit val encodeMeasurementDTO: Encoder[MeasurementDTO] = deriveEncoder -// -// implicit val decodePartitionDTO: Decoder[PartitionDTO] = deriveDecoder -// implicit val encodePartitionDTO: Encoder[PartitionDTO] = deriveEncoder -// -// implicit val decodeCheckpointDTO: Decoder[CheckpointDTO] = deriveDecoder -// implicit val encodeCheckpointDTO: Encoder[CheckpointDTO] = deriveEncoder -// -// implicit val decodePartitioningSubmitDTO: Decoder[PartitioningSubmitDTO] = deriveDecoder -// implicit val encodePartitioningSubmitDTO: Encoder[PartitioningSubmitDTO] = deriveEncoder -// -// implicit val decodeStringMap: Decoder[Map[String, Option[String]]] = Decoder.decodeMap[String, Option[String]] -// implicit val encodeStringMap: Encoder[Map[String, Option[String]]] = Encoder.encodeMap[String, Option[String]] -// -// implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder -// implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder -// -// implicit val decodeAtumContextDTO: Decoder[AtumContextDTO] = deriveDecoder -// implicit val encodeAtumContextDTO: Encoder[AtumContextDTO] = deriveEncoder -// -// // JSON serialization and deserialization utilities -// def asJson[T: Encoder](obj: T): String = { -// obj.asJson.noSpaces -// } -// -// def asJsonPretty[T: Encoder](obj: T): String = { -// obj.asJson.spaces2 -// } -// -// def fromJson[T: Decoder](jsonStr: String): T = { -// io.circe.parser.decode[T](jsonStr) match { -// case Right(value) => value -// case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") -// } -// } -//} -// - import io.circe.{Encoder, Decoder} import io.circe.syntax._ import io.circe.parser.decode diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/OptionImplicits.scala b/model/src/main/scala/za/co/absa/atum/model/utils/OptionImplicits.scala deleted file mode 100644 index 8591ffcb1..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/utils/OptionImplicits.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.utils - -object OptionImplicits { - implicit class OptionEnhancements[T](val option: Option[T]) extends AnyVal { - /** - * Gets the `option` value or throws the provided exception - * - * @param exception the exception to throw in case the `option` is None - * @return - */ - def getOrThrow(exception: => Throwable): T = { - option.getOrElse(throw exception) - } - } -} diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala deleted file mode 100644 index 6c67e69f6..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.utils - - -//import io.circe.{Decoder, Encoder} -//import io.circe.syntax._ -//import io.circe.parser._ -//import java.time.ZonedDateTime -//import java.time.format.DateTimeFormatter -//import java.util.UUID -// -//object SerializationUtils { -// -// -// val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME -// -// implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) -// implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => -// Right(ZonedDateTime.parse(str, timestampFormat)) -// } -// -// implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) -// implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => -// Right(UUID.fromString(str)) -// } -// -// /** -// * The method returns arbitrary object as a Json string. -// * -// * @return A string representing the object in Json format -// */ -// def asJson[T: Encoder](obj: T): String = { -// obj.asJson.noSpaces -// } -// -// /** -// * The method returns arbitrary object as a pretty Json string. -// * -// * @return A string representing the object in Json format -// */ -// def asJsonPretty[T: Encoder](obj: T): String = { -// obj.asJson.spaces2 -// } -// -// /** -// * The method returns arbitrary object parsed from Json string. -// * -// * @return An object deserialized from the Json string -// */ -// def fromJson[T: Decoder](jsonStr: String): T = { -// decode[T](jsonStr) match { -// case Right(value) => value -// case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") -// } -// } -// -// sealed trait ResultValueType -// object ResultValueType { -// case object String extends ResultValueType -// case object Long extends ResultValueType -// case object BigDecimal extends ResultValueType -// case object Double extends ResultValueType -// -// implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { -// case ResultValueType.String => "String" -// case ResultValueType.Long => "Long" -// case ResultValueType.BigDecimal => "BigDecimal" -// case ResultValueType.Double => "Double" -// } -// -// implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { -// case "String" => Right(ResultValueType.String) -// case "Long" => Right(ResultValueType.Long) -// case "BigDecimal" => Right(ResultValueType.BigDecimal) -// case "Double" => Right(ResultValueType.Double) -// case other => Left(s"Cannot decode $other as ResultValueType") -// } -// } -// -//} - -import io.circe._ -//import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType - -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import java.util.UUID - -object SerializationUtils { -// // Timestamp format -// val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME -// -// // Implicit encoders and decoders for ZonedDateTime -// implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) -// implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => -// Right(ZonedDateTime.parse(str, timestampFormat)) -// } -// -// // Implicit encoders and decoders for UUID -// implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) -// implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => -// Right(UUID.fromString(str)) -// } -// -// // Implicit encoders and decoders for Option[String] -// implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] -// implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] - -} diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala b/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala deleted file mode 100644 index 97f300321..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.utils - -import io.circe.{Decoder, Encoder} -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter - -object ZonedDateTimeSerializer { - implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap( - _.format(DateTimeFormatter.ISO_ZONED_DATE_TIME)) - - implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => - try { - Right(ZonedDateTime.parse(str, DateTimeFormatter.ISO_ZONED_DATE_TIME)) - } catch { - case e: Throwable => Left(e.getMessage) - } - } -} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index f43fa083f..ebddac3c1 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -17,9 +17,11 @@ package za.co.absa.atum.model.utils import io.circe.generic.auto._ +import io.circe.syntax.EncoderOps import org.scalatest.flatspec.AnyFlatSpecLike -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto._ +import za.co.absa.atum.model.utils.JsonUtils.fromJson import za.co.absa.atum.model.utils.SerializationUtilsTest.StringLinearization import java.time.{ZoneId, ZoneOffset, ZonedDateTime} @@ -45,7 +47,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"additionalData":{"key1":"val1","key2":"val2","key3":null}, |"author":"testAuthor"} |""".linearize - val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) + val actualAdditionalDataJson = additionalDataDTO.asJson.noSpaces assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -66,7 +68,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ), "testAuthor" ) - val actualAdditionalDataDTO = SerializationUtils.fromJson[AdditionalDataSubmitDTO](additionalDataDTOJson) + val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJson) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -75,7 +77,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val additionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") val expectedAdditionalDataJson = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" - val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) + val actualAdditionalDataJson = additionalDataDTO.asJson.noSpaces assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -85,7 +87,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - val actualAdditionalDataDTO = SerializationUtils.fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) + val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -104,7 +106,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"measures":[{"measureName":"count","measuredColumns":["col"]}], |"additionalData":{} |}""".linearize - val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO) + val actualAdditionalDataJson = atumContextDTO.asJson.noSpaces assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -123,7 +125,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - val actualAtumContextDTO = SerializationUtils.fromJson[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -134,7 +136,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO) val expectedAdditionalDataJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" - val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO) + val actualAdditionalDataJson = atumContextDTO.asJson.noSpaces assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -145,7 +147,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) - val actualAtumContextDTO = SerializationUtils.fromJson[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -189,7 +191,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] |} |""".linearize - val actualCheckpointDTOJson = SerializationUtils.asJson(checkpointDTO) + val actualCheckpointDTOJson = checkpointDTO.asJson.noSpaces assert(actualCheckpointDTOJson == expectedCheckpointDTOJson) } @@ -234,7 +236,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { measurements = setMeasurementDTO ) - val actualCheckpointDTO = SerializationUtils.fromJson[CheckpointDTO](checkpointDTOJson) + val actualCheckpointDTO = fromJson[CheckpointDTO](checkpointDTOJson) assert(actualCheckpointDTO == expectedCheckpointDTO) } @@ -244,7 +246,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureDTO = MeasureDTO("count", Seq("col")) val expectedMeasureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" - val actualMeasureDTOJson = SerializationUtils.asJson(measureDTO) + val actualMeasureDTOJson = measureDTO.asJson.noSpaces assert(actualMeasureDTOJson == expectedMeasureDTOJson) } @@ -253,7 +255,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" val expectedMeasureDTO = MeasureDTO("count", Seq("col")) - val actualMeasureDTO = SerializationUtils.fromJson[MeasureDTO](measureDTOJson) + val actualMeasureDTO = fromJson[MeasureDTO](measureDTOJson) assert(actualMeasureDTO == expectedMeasureDTO) } @@ -273,7 +275,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"supportValues":{}} |} |""".linearize - val actualMeasurementDTOJson = SerializationUtils.asJson(measurementDTO) + val actualMeasurementDTOJson = measurementDTO.asJson.noSpaces assert(actualMeasurementDTOJson == expectedMeasurementDTOJson) } @@ -291,7 +293,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - val actualMeasurementDTO = SerializationUtils.fromJson[MeasurementDTO](measurementDTOJson) + val actualMeasurementDTO = fromJson[MeasurementDTO](measurementDTOJson) assert(actualMeasurementDTO == expectedMeasurementDTO) } @@ -301,7 +303,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" - val actualMeasureResultDTOJson = SerializationUtils.asJson(measureResultDTO) + val actualMeasureResultDTOJson = measureResultDTO.asJson.noSpaces assert(actualMeasureResultDTOJson == expectedMeasureResultDTOJson) } @@ -310,7 +312,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) - val actualMeasureResultDTO = SerializationUtils.fromJson[MeasureResultDTO](measureResultDTOJson) + val actualMeasureResultDTO = fromJson[MeasureResultDTO](measureResultDTOJson) assert(actualMeasureResultDTO == expectedMeasureResultDTO) } @@ -320,7 +322,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val partitionDTO = PartitionDTO("key", "val") val expectedPartitionDTOJson = """{"key":"key","value":"val"}""" - val actualPartitionDTOJson = SerializationUtils.asJson(partitionDTO) + val actualPartitionDTOJson = partitionDTO.asJson.noSpaces assert(actualPartitionDTOJson == expectedPartitionDTOJson) } @@ -329,7 +331,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val partitionDTOJson = """{"key":"key","value":"val"}""" val expectedPartitionDTO = PartitionDTO("key", "val") - val actualPartitionDTO = SerializationUtils.fromJson[PartitionDTO](partitionDTOJson) + val actualPartitionDTO = fromJson[PartitionDTO](partitionDTOJson) assert(actualPartitionDTO == expectedPartitionDTO) } @@ -345,7 +347,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ) val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" - val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) + val actualPartitioningDTOJson = partitioningDTO.asJson.noSpaces assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } @@ -360,7 +362,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = SerializationUtils.fromJson[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } @@ -383,7 +385,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"authorIfNew":"authorTest" |} |""".linearize - val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) + val actualPartitioningDTOJson = partitioningDTO.asJson.noSpaces assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } @@ -397,7 +399,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ) val expectedPartitionDTOJson = """[{"key":"key1","value":"val1"},{"key":"key2","value":"val2"},{"key":"key3","value":"val3"}]""" - val actualPartitionDTOJson = SerializationUtils.asJson(partitionDTO) + val actualPartitionDTOJson = partitionDTO.asJson.noSpaces assert(actualPartitionDTOJson == expectedPartitionDTOJson) } @@ -420,7 +422,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = SerializationUtils.fromJson[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 7d0bd599c..f027ef411 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -32,7 +32,6 @@ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ import za.co.absa.atum.model.dto.MeasureResultDTO._ -import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index e13d1dacd..3feb9df10 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -26,7 +26,6 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} -import za.co.absa.atum.model.utils.JsonImplicits._ import sttp.tapir.{PublicEndpoint, endpoint} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 6c10db8ec..705e6c319 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -18,7 +18,6 @@ package za.co.absa.atum.server.model import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import io.circe.{DecodingFailure, Json} -import za.co.absa.atum.model.utils.JsonImplicits._ import java.time.ZonedDateTime import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index ad987d659..6e91fcfd4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,17 +16,16 @@ package za.co.absa.atum.server.model - import io.circe.generic.semiauto._ import io.circe.{Decoder, Encoder} import za.co.absa.atum.model.dto.PartitioningDTO import scala.collection.immutable.Seq private[server] case class PartitioningForDB private ( - version: Int = 1, - keys: Seq[String], - keysToValues: Map[String, String] - ) + version: Int = 1, + keys: Seq[String], + keysToValues: Map[String, String] +) object PartitioningForDB { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 2fa568e28..df70be3ce 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.server.model.CheckpointFromDB import java.time.ZonedDateTime import java.util.UUID import MeasureResultDTO.TypedValue -import MeasureResultDTO.ResultValueType._ +import za.co.absa.atum.model.utils.ResultValueType trait TestData { @@ -73,17 +73,17 @@ trait TestData { val mainValue: TypedValue = TypedValue( value = "123", - valueType = Long + valueType = ResultValueType.Long ) val supportValue1: TypedValue = TypedValue( value = "123456789", - valueType = Long + valueType = ResultValueType.Long ) val supportValue2: TypedValue = TypedValue( value = "12345.6789", - valueType = BigDecimal + valueType = ResultValueType.BigDecimal ) // Measure Result DTO @@ -194,7 +194,7 @@ trait TestData { author = "author", measuredByAtumAgent = true, measureName = measureDTO1.measureName, - measuredColumns = measureDTO1.measuredColumns, + measuredColumns = measureDTO1.measuredColumns.toIndexedSeq, measurementValue = parser .parse( """ @@ -229,7 +229,7 @@ trait TestData { author = "author2", measuredByAtumAgent = true, measureName = measureDTO2.measureName, - measuredColumns = measureDTO2.measuredColumns, + measuredColumns = measureDTO2.measuredColumns.toIndexedSeq, checkpointStartTime = checkpointDTO2.processStartTime, checkpointEndTime = checkpointDTO2.processEndTime diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 2c655623f..d6a3185c1 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -17,7 +17,8 @@ package za.co.absa.atum.server.api.database.runs.functions import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.utils.ResultValueType import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 70cab0f8b..6e0b55bdf 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -29,7 +29,6 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index 8e2c002ed..c92d418a0 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -28,7 +28,6 @@ import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.model.utils.JsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo From b124fb119d5069bcdfb5229daacb1cddcd1fca13 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 16:36:54 +0200 Subject: [PATCH 16/34] Defining ResultsValueType separate --- .../atum/model/utils/ResultValueType.scala | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala b/model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala new file mode 100644 index 000000000..4fb04ade2 --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.utils + +import io.circe.{Decoder, Encoder} + +sealed trait ResultValueType +object ResultValueType { + case object String extends ResultValueType + case object Long extends ResultValueType + case object BigDecimal extends ResultValueType + case object Double extends ResultValueType + + implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { + case ResultValueType.String => "String" + case ResultValueType.Long => "Long" + case ResultValueType.BigDecimal => "BigDecimal" + case ResultValueType.Double => "Double" + } + + implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { + case "String" => Right(ResultValueType.String) + case "Long" => Right(ResultValueType.Long) + case "BigDecimal" => Right(ResultValueType.BigDecimal) + case "Double" => Right(ResultValueType.Double) + case other => Left(s"Cannot decode $other as ResultValueType") + } +} From a9e23e46d07715087fb253cbaff43652bb079d3d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 16:52:31 +0200 Subject: [PATCH 17/34] Defining private method for stringified json list --- .../server/api/database/DoobieImplicits.scala | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 3a1969ed2..b55280e8b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -41,6 +41,18 @@ object DoobieImplicits { } + private def circeJsonListToPGJsonArrayString(jsonList: List[Json]): String = { + val arrayElements = jsonList.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s"\"$escapedJsonString\"" + } + + // Join all elements into a single string wrapped in curly braces + arrayElements.mkString("{", ",", "}") + } + object Json { implicit val jsonArrayPut: Put[List[Json]] = { @@ -51,14 +63,7 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - val arrayElements = a.map { x => - // Convert to compact JSON string and escape inner quotes - val escapedJsonString = x.noSpaces.replace("\"", "\\\"") - // Wrap in double quotes for the array element - s"\"$escapedJsonString\"" - } - // Join all elements into a single string wrapped in curly braces - o.setValue(arrayElements.mkString("{", ",", "}")) + o.setValue(circeJsonListToPGJsonArrayString(a)) o } } @@ -116,14 +121,7 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("jsonb[]") - val arrayElements = a.map { x => - // Convert to compact JSON string and escape inner quotes - val escapedJsonString = x.noSpaces.replace("\"", "\\\"") - // Wrap in double quotes for the array element - s"\"$escapedJsonString\"" - } - // Join all elements into a single string wrapped in curly braces - o.setValue(arrayElements.mkString("{", ",", "}")) + o.setValue(circeJsonListToPGJsonArrayString(a)) o } } From cf9cbb4385d0b3c66ba2ba6f4fec2fc0b2edcfc8 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 16:56:45 +0200 Subject: [PATCH 18/34] removing jacksonModuleScala --- project/Dependencies.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 41e855530..5e98646d8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -27,8 +27,6 @@ object Dependencies { val scalaLangJava8Compat = "1.0.2" val balta = "0.1.0" - val jacksonModuleScala = "2.14.2" - val specs2 = "4.10.0" val typesafeConfig = "1.4.2" @@ -91,15 +89,12 @@ object Dependencies { private def jsonSerdeDependencies(scalaVersion: Version): Seq[ModuleID] = { - lazy val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jacksonModuleScala - // Circe dependencies lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson Seq( - jacksonModuleScala, circeCore, circeParser, circeGeneric, From 282d16e7d69f0e13bc111fd7d1016303990d49dc Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 16:59:01 +0200 Subject: [PATCH 19/34] removing scalaVersion parameter --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5e98646d8..447bb75fb 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -87,7 +87,7 @@ object Dependencies { ) } - private def jsonSerdeDependencies(scalaVersion: Version): Seq[ModuleID] = { + private def jsonSerdeDependencies(): Seq[ModuleID] = { // Circe dependencies lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson From cde28ee2d6ffd485647bc27d0eaed0ca6fc16b01 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 18:41:11 +0200 Subject: [PATCH 20/34] removing scalaVersion parameter --- build.sbt | 1 - project/Dependencies.scala | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 9d7686d0e..14f3a0bf1 100644 --- a/build.sbt +++ b/build.sbt @@ -104,4 +104,3 @@ lazy val database = (projectMatrix in file("database")) ): _* ) .addSingleScalaBuild(Setup.serverAndDbScalaVersion, Dependencies.databaseDependencies) - diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 447bb75fb..c6cb6a38d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -87,7 +87,7 @@ object Dependencies { ) } - private def jsonSerdeDependencies(): Seq[ModuleID] = { + private def jsonSerdeDependencies: Seq[ModuleID] = { // Circe dependencies lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson @@ -229,7 +229,7 @@ object Dependencies { typeSafeConfig ) ++ testDependencies ++ - jsonSerdeDependencies(scalaVersion) + jsonSerdeDependencies } def databaseDependencies: Seq[ModuleID] = { From 787d922039766919ce887e6eff99e0e945ae61cb Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 1 Jul 2024 19:43:29 +0200 Subject: [PATCH 21/34] moving ResultsValueType to model and modify imports --- agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala | 2 +- .../main/scala/za/co/absa/atum/agent/model/MeasureResult.scala | 2 +- .../test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala | 2 +- .../za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala | 2 +- .../scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala | 2 +- .../co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala | 2 +- .../za/co/absa/atum/model/{utils => }/ResultValueType.scala | 2 +- .../main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala | 2 +- .../co/absa/atum/model/utils/SerializationUtilsUnitTests.scala | 1 + server/src/test/scala/za/co/absa/atum/server/api/TestData.scala | 2 +- .../runs/functions/WriteCheckpointIntegrationTests.scala | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) rename model/src/main/scala/za/co/absa/atum/model/{utils => }/ResultValueType.scala (97%) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index 3d2889899..90ae99683 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.types.{DataType, DecimalType, LongType, StringType} import org.apache.spark.sql.{Column, DataFrame} import za.co.absa.atum.agent.core.MeasurementProcessor import za.co.absa.atum.agent.core.MeasurementProcessor.MeasurementFunction -import za.co.absa.atum.model.utils.ResultValueType +import za.co.absa.atum.model.ResultValueType /** * Type of different measures to be applied to the columns. diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index 75bfb6859..ecc5233fa 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.agent.model import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException -import za.co.absa.atum.model.utils.ResultValueType +import za.co.absa.atum.model.ResultValueType /** * This trait defines a contract for a measure result. diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala index 95d926fbe..ed21dae88 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala @@ -25,8 +25,8 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{RecordCount, SumOfValuesOfColumn} import za.co.absa.atum.agent.model.{Measure, MeasureResult, MeasurementBuilder, UnknownMeasure} +import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.utils.ResultValueType class AtumContextUnitTests extends AnyFlatSpec with Matchers { diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala index 93741b0d1..ece74f3e6 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala @@ -23,7 +23,7 @@ import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.agent.AtumContext.{AtumPartitions, DatasetWrapper} import za.co.absa.atum.agent.model.AtumMeasure._ -import za.co.absa.atum.model.utils.ResultValueType +import za.co.absa.atum.model.ResultValueType import za.co.absa.spark.commons.test.SparkTestBase class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala index df9808d11..ed0dced0d 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{AbsSumOfValuesOfColumn, RecordCount, SumOfHashesOfColumn, SumOfValuesOfColumn} import za.co.absa.spark.commons.test.SparkTestBase import za.co.absa.atum.agent.AtumContext._ -import za.co.absa.atum.model.utils.ResultValueType +import za.co.absa.atum.model.ResultValueType class MeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala index f2f7e1776..cab35e359 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala @@ -20,8 +20,8 @@ import org.scalatest.flatspec.AnyFlatSpec import za.co.absa.atum.agent.exception.AtumAgentException.MeasurementException import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO} import za.co.absa.atum.agent.model.AtumMeasure._ +import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue -import za.co.absa.atum.model.utils.ResultValueType class MeasurementBuilderUnitTests extends AnyFlatSpec { diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala b/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala similarity index 97% rename from model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala rename to model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala index 4fb04ade2..0fd380884 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/ResultValueType.scala +++ b/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.model.utils +package za.co.absa.atum.model import io.circe.{Decoder, Encoder} diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index c33ebe19e..eac2c6c91 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.model.dto import io.circe._ import io.circe.generic.semiauto._ -import za.co.absa.atum.model.utils.ResultValueType +import za.co.absa.atum.model.ResultValueType case class MeasureResultDTO( mainValue: MeasureResultDTO.TypedValue, diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index ebddac3c1..1e5385299 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -19,6 +19,7 @@ package za.co.absa.atum.model.utils import io.circe.generic.auto._ import io.circe.syntax.EncoderOps import org.scalatest.flatspec.AnyFlatSpecLike +import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.utils.JsonUtils.fromJson diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index df70be3ce..418e4103c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.server.model.CheckpointFromDB import java.time.ZonedDateTime import java.util.UUID import MeasureResultDTO.TypedValue -import za.co.absa.atum.model.utils.ResultValueType +import za.co.absa.atum.model.ResultValueType trait TestData { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index d6a3185c1..a67c9f792 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -16,9 +16,9 @@ package za.co.absa.atum.server.api.database.runs.functions +import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue -import za.co.absa.atum.model.utils.ResultValueType import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider From 305ade564b055c703a2820f224fc66a26bc5821f Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 2 Jul 2024 14:34:30 +0200 Subject: [PATCH 22/34] Redefining ResultValueType custom types --- .../za/co/absa/atum/agent/model/Measure.scala | 24 +++++++------- .../absa/atum/agent/model/MeasureResult.scala | 8 ++--- .../atum/agent/AtumContextUnitTests.scala | 8 ++--- .../agent/model/AtumMeasureUnitTests.scala | 22 ++++++------- .../atum/agent/model/MeasureUnitTests.scala | 12 +++---- .../model/MeasurementBuilderUnitTests.scala | 32 +++++++++---------- .../co/absa/atum/model/ResultValueType.scala | 24 +++++++------- .../utils/SerializationUtilsUnitTests.scala | 12 +++---- .../CreateOrUpdateAdditionalData.scala | 2 +- .../runs/functions/WriteCheckpoint.scala | 2 +- .../za/co/absa/atum/server/api/TestData.scala | 6 ++-- .../WriteCheckpointIntegrationTests.scala | 2 +- 12 files changed, 77 insertions(+), 77 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index 90ae99683..6ffa24860 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -57,7 +57,7 @@ object AtumMeasure { } override def measuredColumns: Seq[String] = Seq.empty - override val resultValueType: ResultValueType = ResultValueType.Long + override val resultValueType: ResultValueType = ResultValueType.LongValue } object RecordCount { private[agent] val measureName: String = "count" @@ -76,7 +76,7 @@ object AtumMeasure { } override def measuredColumns: Seq[String] = measuredCols - override val resultValueType: ResultValueType = ResultValueType.Long + override val resultValueType: ResultValueType = ResultValueType.LongValue } object DistinctRecordCount { private[agent] val measureName: String = "distinctCount" @@ -93,7 +93,7 @@ object AtumMeasure { } override def measuredColumns: Seq[String] = Seq(measuredCol) - override val resultValueType: ResultValueType = ResultValueType.BigDecimal + override val resultValueType: ResultValueType = ResultValueType.BigDecimalValue } object SumOfValuesOfColumn { private[agent] val measureName: String = "aggregatedTotal" @@ -110,7 +110,7 @@ object AtumMeasure { } override def measuredColumns: Seq[String] = Seq(measuredCol) - override val resultValueType: ResultValueType = ResultValueType.BigDecimal + override val resultValueType: ResultValueType = ResultValueType.BigDecimalValue } object AbsSumOfValuesOfColumn { private[agent] val measureName: String = "absAggregatedTotal" @@ -125,7 +125,7 @@ object AtumMeasure { } override def measuredColumns: Seq[String] = Seq(measuredCol) - override val resultValueType: ResultValueType = ResultValueType.String + override val resultValueType: ResultValueType = ResultValueType.StringValue } object SumOfHashesOfColumn { private[agent] val measureName: String = "hashCrc32" @@ -138,10 +138,10 @@ object AtumMeasure { ): Column = { dataType match { case _: LongType => - // This is protection against long overflow, e.g. Long.MaxValue = 9223372036854775807: - // scala> sc.parallelize(List(Long.MaxValue, 1)).toDF.agg(sum("value")).take(1)(0)(0) + // This is protection against long overflow, e.g. LongValue.MaxValue = 9223372036854775807: + // scala> sc.parallelize(List(LongValue.MaxValue, 1)).toDF.agg(sum("value")).take(1)(0)(0) // res11: Any = -9223372036854775808 - // Converting to BigDecimal fixes the issue + // Converting to BigDecimalValue fixes the issue column.cast(DecimalType(38, 0)) case _: StringType => // Support for string type aggregation @@ -170,21 +170,21 @@ object AtumMeasure { /** * This method converts a given value to string. - * It is a workaround for different serializers generating different JSONs for BigDecimal. + * It is a workaround for different serializers generating different JSONs for BigDecimalValue. * See https://stackoverflow.com/questions/61973058/json-serialization-of-bigdecimal-returns-scientific-notation * * @param value A value to convert * @return A string representation of the value */ private def workaroundBigDecimalIssues(value: Any): String = - // If aggregated value is java.math.BigDecimal, convert it to scala.math.BigDecimal + // If aggregated value is java.math.BigDecimalValue, convert it to scala.math.BigDecimalValue value match { case v: java.math.BigDecimal => - // Convert the value to string to workaround different serializers generate different JSONs for BigDecimal + // Convert the value to string to workaround different serializers generate different JSONs for BigDecimalValue v.stripTrailingZeros // removes trailing zeros (2001.500000 -> 2001.5, but can introduce scientific notation (600.000 -> 6E+2) .toPlainString // converts to normal string (6E+2 -> "600") case v: BigDecimal => - // Convert the value to string to workaround different serializers generate different JSONs for BigDecimal + // Convert the value to string to workaround different serializers generate different JSONs for BigDecimalValue new java.math.BigDecimal( v.toString() ).stripTrailingZeros // removes trailing zeros (2001.500000 -> 2001.5, but can introduce scientific notation (600.000 -> 6E+2) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index ecc5233fa..463dc632f 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -80,13 +80,13 @@ object MeasureResult { resultValue match { case l: Long => - MeasureResultProvided[Long](l, ResultValueType.Long) + MeasureResultProvided[Long](l, ResultValueType.LongValue) case d: Double => - MeasureResultProvided[Double](d, ResultValueType.Double) + MeasureResultProvided[Double](d, ResultValueType.DoubleValue) case bd: BigDecimal => - MeasureResultProvided[BigDecimal](bd, ResultValueType.BigDecimal) + MeasureResultProvided[BigDecimal](bd, ResultValueType.BigDecimalValue) case s: String => - MeasureResultProvided[String](s, ResultValueType.String) + MeasureResultProvided[String](s, ResultValueType.StringValue) case unsupportedType => val className = unsupportedType.getClass.getSimpleName diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala index ed21dae88..75585f485 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala @@ -100,7 +100,7 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { assert(argument.getValue.author == authorTest) assert(argument.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) assert(argument.getValue.measurements.head.result.mainValue.value == "3") - assert(argument.getValue.measurements.head.result.mainValue.valueType == ResultValueType.Long) + assert(argument.getValue.measurements.head.result.mainValue.valueType == ResultValueType.LongValue) } "createCheckpointOnProvidedData" should "create a Checkpoint on provided data" in { @@ -115,7 +115,7 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { val measurements: Map[Measure, MeasureResult] = Map( RecordCount("col") -> MeasureResult(1L), SumOfValuesOfColumn("col") -> MeasureResult(BigDecimal(1)), - UnknownMeasure("customMeasureName", Seq("col"), ResultValueType.BigDecimal) -> MeasureResult(BigDecimal(1)) + UnknownMeasure("customMeasureName", Seq("col"), ResultValueType.BigDecimalValue) -> MeasureResult(BigDecimal(1)) ) atumContext.createCheckpointOnProvidedData( @@ -172,7 +172,7 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { assert(argumentFirst.getValue.author == authorTest) assert(argumentFirst.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) assert(argumentFirst.getValue.measurements.head.result.mainValue.value == "4") - assert(argumentFirst.getValue.measurements.head.result.mainValue.valueType == ResultValueType.Long) + assert(argumentFirst.getValue.measurements.head.result.mainValue.valueType == ResultValueType.LongValue) atumContext.addMeasure(SumOfValuesOfColumn("columnForSum")) when(mockAgent.currentUser).thenReturn(authorTest + "Another") // maybe a process changed the author / current user @@ -185,7 +185,7 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { assert(argumentSecond.getValue.author == authorTest + "Another") assert(argumentSecond.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.value == "22.5") - assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.valueType == ResultValueType.BigDecimal) + assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.valueType == ResultValueType.BigDecimalValue) } "addAdditionalData" should "add key/value pair to map for additional data" in { diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala index ece74f3e6..7f2278f91 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala @@ -94,17 +94,17 @@ class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase // Assertions assert(dfPersonCntResult.resultValue == "1000") - assert(dfPersonCntResult.resultValueType == ResultValueType.Long) + assert(dfPersonCntResult.resultValueType == ResultValueType.LongValue) assert(dfFullCntResult.resultValue == "1000") - assert(dfFullCntResult.resultValueType == ResultValueType.Long) + assert(dfFullCntResult.resultValueType == ResultValueType.LongValue) assert(dfFullSalaryAbsSumResult.resultValue == "2987144") - assert(dfFullSalaryAbsSumResult.resultValueType == ResultValueType.BigDecimal) + assert(dfFullSalaryAbsSumResult.resultValueType == ResultValueType.BigDecimalValue) assert(dfFullHashResult.resultValue == "2044144307532") - assert(dfFullHashResult.resultValueType == ResultValueType.String) + assert(dfFullHashResult.resultValueType == ResultValueType.StringValue) assert(dfExtraPersonSalarySumResult.resultValue == "2986144") - assert(dfExtraPersonSalarySumResult.resultValueType == ResultValueType.BigDecimal) + assert(dfExtraPersonSalarySumResult.resultValueType == ResultValueType.BigDecimalValue) assert(dfFullSalarySumResult.resultValue == "2987144") - assert(dfFullSalarySumResult.resultValueType == ResultValueType.BigDecimal) + assert(dfFullSalarySumResult.resultValueType == ResultValueType.BigDecimalValue) } "AbsSumOfValuesOfColumn" should "return expected value" in { @@ -119,7 +119,7 @@ class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase val result = salaryAbsSum.function(df) assert(result.resultValue == "300.3") - assert(result.resultValueType == ResultValueType.BigDecimal) + assert(result.resultValueType == ResultValueType.BigDecimalValue) } "AbsSumOfValuesOfColumn" should "return expected value for null result" in { @@ -134,7 +134,7 @@ class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase val result = salaryAbsSum.function(df) assert(result.resultValue == "0") - assert(result.resultValueType == ResultValueType.BigDecimal) + assert(result.resultValueType == ResultValueType.BigDecimalValue) } "RecordCount" should "return expected value" in { @@ -149,7 +149,7 @@ class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase val result = distinctCount.function(df) assert(result.resultValue == "4") - assert(result.resultValueType == ResultValueType.Long) + assert(result.resultValueType == ResultValueType.LongValue) } "DistinctRecordCount" should "return expected value for multiple columns" in { @@ -164,7 +164,7 @@ class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase val result = distinctCount.function(df) assert(result.resultValue == "3") - assert(result.resultValueType == ResultValueType.Long) + assert(result.resultValueType == ResultValueType.LongValue) } "DistinctRecordCount" should "fail requirements when no control columns given" in { @@ -183,7 +183,7 @@ class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase val result = distinctCount.function(df) assert(result.resultValue == "4") - assert(result.resultValueType == ResultValueType.BigDecimal) + assert(result.resultValueType == ResultValueType.BigDecimalValue) } } diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala index ed0dced0d..d96d0ac1e 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala @@ -92,17 +92,17 @@ class MeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { se // Assertions assert(dfPersonCntResult.resultValue == "1000") - assert(dfPersonCntResult.resultValueType == ResultValueType.Long) + assert(dfPersonCntResult.resultValueType == ResultValueType.LongValue) assert(dfFullCntResult.resultValue == "1000") - assert(dfFullCntResult.resultValueType == ResultValueType.Long) + assert(dfFullCntResult.resultValueType == ResultValueType.LongValue) assert(dfFullSalaryAbsSumResult.resultValue == "2987144") - assert(dfFullSalaryAbsSumResult.resultValueType == ResultValueType.BigDecimal) + assert(dfFullSalaryAbsSumResult.resultValueType == ResultValueType.BigDecimalValue) assert(dfFullHashResult.resultValue == "2044144307532") - assert(dfFullHashResult.resultValueType == ResultValueType.String) + assert(dfFullHashResult.resultValueType == ResultValueType.StringValue) assert(dfExtraPersonSalarySumResult.resultValue == "2986144") - assert(dfExtraPersonSalarySumResult.resultValueType == ResultValueType.BigDecimal) + assert(dfExtraPersonSalarySumResult.resultValueType == ResultValueType.BigDecimalValue) assert(dfFullSalarySumResult.resultValue == "2987144") - assert(dfFullSalarySumResult.resultValueType == ResultValueType.BigDecimal) + assert(dfFullSalarySumResult.resultValueType == ResultValueType.BigDecimalValue) } } diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala index cab35e359..b64060257 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue class MeasurementBuilderUnitTests extends AnyFlatSpec { "buildMeasurementDTO" should - "build MeasurementDTO for BigDecimal type of result value when Measure and MeasureResult provided" in { + "build MeasurementDTO for BigDecimalValue type of result value when Measure and MeasureResult provided" in { val measure = SumOfValuesOfColumn("col") val measureResult = MeasureResult(BigDecimal(1)) @@ -36,7 +36,7 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { val expectedMeasureDTO = MeasureDTO("aggregatedTotal", Seq("col")) val expectedMeasureResultDTO = MeasureResultDTO( - TypedValue("1", ResultValueType.BigDecimal) + TypedValue("1", ResultValueType.BigDecimalValue) ) assert(measurementDTO.measure == expectedMeasureDTO) @@ -44,13 +44,13 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { } "buildMeasurementDTO" should - "build MeasurementDTO for BigDecimal type of result value when Measurement provided" in { + "build MeasurementDTO for BigDecimalValue type of result value when Measurement provided" in { val measure = SumOfValuesOfColumn("col") val measureResult = MeasureResult(BigDecimal(3.14)) val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measure, measureResult) - val expectedTypedValue = TypedValue("3.14", ResultValueType.BigDecimal) + val expectedTypedValue = TypedValue("3.14", ResultValueType.BigDecimalValue) assert(measurementDTO.result.mainValue == expectedTypedValue) } @@ -60,27 +60,27 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { "when Measurement provided" in { val measure = SumOfValuesOfColumn("col") - val measureResult = MeasureResult("stringValue", ResultValueType.BigDecimal) + val measureResult = MeasureResult("string", ResultValueType.BigDecimalValue) val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measure, measureResult) - val expectedTypedValue = TypedValue("stringValue", ResultValueType.BigDecimal) + val expectedTypedValue = TypedValue("string", ResultValueType.BigDecimalValue) assert(measurementDTO.result.mainValue == expectedTypedValue) } "buildMeasurementDTO" should - "build MeasurementDTO for BigDecimal type of result value when measured by Agent" in { + "build MeasurementDTO for BigDecimalValue type of result value when measured by Agent" in { val measure = SumOfValuesOfColumn("col") - val measureResult = MeasureResult("1", ResultValueType.BigDecimal) + val measureResult = MeasureResult("1", ResultValueType.BigDecimalValue) val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measure, measureResult) val expectedMeasureDTO = MeasureDTO("aggregatedTotal", Seq("col")) val expectedMeasureResultDTO = MeasureResultDTO( - TypedValue("1", ResultValueType.BigDecimal) + TypedValue("1", ResultValueType.BigDecimalValue) ) assert(measurementDTO.measure == expectedMeasureDTO) @@ -89,25 +89,25 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { "buildAndValidateMeasurementsDTO" should "build Seq[MeasurementDTO] for multiple measures, all unique" in { val measurements: Map[Measure, MeasureResult] = Map( - DistinctRecordCount(Seq("col")) -> MeasureResult("1", ResultValueType.Long), + DistinctRecordCount(Seq("col")) -> MeasureResult("1", ResultValueType.LongValue), SumOfValuesOfColumn("col1") -> MeasureResult(BigDecimal(1.2)), SumOfValuesOfColumn("col2") -> MeasureResult(BigDecimal(1.3)), - UnknownMeasure("unknownMeasure", Seq("col"), ResultValueType.BigDecimal) -> MeasureResult(BigDecimal(1.1)) + UnknownMeasure("unknownMeasure", Seq("col"), ResultValueType.BigDecimalValue) -> MeasureResult(BigDecimal(1.1)) ) val measurementDTOs = MeasurementBuilder.buildAndValidateMeasurementsDTO(measurements) val expectedMeasurementDTO = Set( MeasurementDTO( - MeasureDTO("distinctCount", Seq("col")), MeasureResultDTO(TypedValue("1", ResultValueType.Long)) + MeasureDTO("distinctCount", Seq("col")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue)) ), MeasurementDTO( - MeasureDTO("aggregatedTotal", Seq("col1")), MeasureResultDTO(TypedValue("1.2", ResultValueType.BigDecimal)) + MeasureDTO("aggregatedTotal", Seq("col1")), MeasureResultDTO(TypedValue("1.2", ResultValueType.BigDecimalValue)) ), MeasurementDTO( - MeasureDTO("aggregatedTotal", Seq("col2")), MeasureResultDTO(TypedValue("1.3", ResultValueType.BigDecimal)) + MeasureDTO("aggregatedTotal", Seq("col2")), MeasureResultDTO(TypedValue("1.3", ResultValueType.BigDecimalValue)) ), MeasurementDTO( - MeasureDTO("unknownMeasure", Seq("col")), MeasureResultDTO(TypedValue("1.1", ResultValueType.BigDecimal)) + MeasureDTO("unknownMeasure", Seq("col")), MeasureResultDTO(TypedValue("1.1", ResultValueType.BigDecimalValue)) ) ) @@ -150,7 +150,7 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { val measure = SumOfValuesOfColumn("col") assertThrows[MeasurementException]( - MeasurementBuilder.buildAndValidateMeasurementsDTO(Map(measure -> MeasureResult("stringValue", ResultValueType.String))) + MeasurementBuilder.buildAndValidateMeasurementsDTO(Map(measure -> MeasureResult("string", ResultValueType.StringValue))) ) } } diff --git a/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala b/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala index 0fd380884..6d2177df1 100644 --- a/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala +++ b/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala @@ -20,23 +20,23 @@ import io.circe.{Decoder, Encoder} sealed trait ResultValueType object ResultValueType { - case object String extends ResultValueType - case object Long extends ResultValueType - case object BigDecimal extends ResultValueType - case object Double extends ResultValueType + case object StringValue extends ResultValueType + case object LongValue extends ResultValueType + case object BigDecimalValue extends ResultValueType + case object DoubleValue extends ResultValueType implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { - case ResultValueType.String => "String" - case ResultValueType.Long => "Long" - case ResultValueType.BigDecimal => "BigDecimal" - case ResultValueType.Double => "Double" + case ResultValueType.StringValue => "String" + case ResultValueType.LongValue => "Long" + case ResultValueType.BigDecimalValue => "BigDecimal" + case ResultValueType.DoubleValue => "Double" } implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { - case "String" => Right(ResultValueType.String) - case "Long" => Right(ResultValueType.Long) - case "BigDecimal" => Right(ResultValueType.BigDecimal) - case "Double" => Right(ResultValueType.Double) + case "String" => Right(ResultValueType.StringValue) + case "Long" => Right(ResultValueType.LongValue) + case "BigDecimal" => Right(ResultValueType.BigDecimalValue) + case "Double" => Right(ResultValueType.DoubleValue) case other => Left(s"Cannot decode $other as ResultValueType") } } diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 1e5385299..5596ec84b 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -162,7 +162,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val setMeasurementDTO = Set( MeasurementDTO( measure = MeasureDTO("count", Seq("col")), result = MeasureResultDTO( - mainValue = TypedValue("1", ResultValueType.Long) + mainValue = TypedValue("1", ResultValueType.LongValue) ) ) ) @@ -221,7 +221,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val setMeasurementDTO = Set( MeasurementDTO( measure = MeasureDTO("count", Seq("col")), result = MeasureResultDTO( - mainValue = TypedValue("1", ResultValueType.Long) + mainValue = TypedValue("1", ResultValueType.LongValue) ) ) ) @@ -264,7 +264,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { // MeasurementDTO "asJson" should "serialize MeasurementDTO into json string" in { val measureDTO = MeasureDTO("count", Seq("col")) - val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) + val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val measurementDTO = MeasurementDTO(measureDTO, measureResultDTO) @@ -291,7 +291,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |""".stripMargin val measureDTO = MeasureDTO("count", Seq("col")) - val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) + val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) val actualMeasurementDTO = fromJson[MeasurementDTO](measurementDTOJson) @@ -301,7 +301,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { // MeasureResultDTO "asJson" should "serialize MeasureResultDTO into json string" in { - val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) + val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" val actualMeasureResultDTOJson = measureResultDTO.asJson.noSpaces @@ -312,7 +312,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { "fromJson" should "deserialize MeasureResultDTO from json string" in { val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" - val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long)) + val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val actualMeasureResultDTO = fromJson[MeasureResultDTO](measureResultDTOJson) assert(actualMeasureResultDTO == expectedMeasureResultDTO) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index f7b8da0d3..f95a253d8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -42,7 +42,7 @@ class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEn val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) val partitioningJsonString = partitioning.asJson.noSpaces - // implicits from Doobie can't handle Map[String, Option[String]] -> HStore, so we converted None to null basically + // implicits from Doobie can't handle Map[StringValue, Option[StringValue]] -> HStore, so we converted None to null basically val additionalDataNormalized = values.additionalData.map{ case (k, v) => (k, v.orNull)} sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index f027ef411..273c59610 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -45,7 +45,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) val partitioningNormalized = partitioning.asJson.noSpaces - // List[String] containing json data has to be properly escaped + // List[StringValue] containing json data has to be properly escaped // It would be safer to use Json data type and derive Put instance val measurementsNormalized = { values.measurements.toList.map(_.asJson) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala index 418e4103c..19acbfdc9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestData.scala @@ -73,17 +73,17 @@ trait TestData { val mainValue: TypedValue = TypedValue( value = "123", - valueType = ResultValueType.Long + valueType = ResultValueType.LongValue ) val supportValue1: TypedValue = TypedValue( value = "123456789", - valueType = ResultValueType.Long + valueType = ResultValueType.LongValue ) val supportValue2: TypedValue = TypedValue( value = "12345.6789", - valueType = ResultValueType.BigDecimal + valueType = ResultValueType.BigDecimalValue ) // Measure Result DTO diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index a67c9f792..f24b05dff 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -45,7 +45,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = - Set(MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.Long)))) + Set(MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue)))) ) for { writeCheckpoint <- ZIO.service[WriteCheckpoint] From c051b42c183c460ec7a16324acf3e8f1a5db5897 Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Thu, 4 Jul 2024 08:52:17 +0200 Subject: [PATCH 23/34] Update server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala Co-authored-by: David Benedeki <14905969+benedeki@users.noreply.github.com> --- .../database/runs/functions/CreateOrUpdateAdditionalData.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index f95a253d8..f7b8da0d3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -42,7 +42,7 @@ class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEn val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) val partitioningJsonString = partitioning.asJson.noSpaces - // implicits from Doobie can't handle Map[StringValue, Option[StringValue]] -> HStore, so we converted None to null basically + // implicits from Doobie can't handle Map[String, Option[String]] -> HStore, so we converted None to null basically val additionalDataNormalized = values.additionalData.map{ case (k, v) => (k, v.orNull)} sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( From 0750cb630c9afcb7c22b28e871eef89b79208b86 Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Thu, 4 Jul 2024 09:29:45 +0200 Subject: [PATCH 24/34] Apply suggestions from code review Co-authored-by: David Benedeki <14905969+benedeki@users.noreply.github.com> --- .../za/co/absa/atum/agent/model/Measure.scala | 14 +++++++------- .../agent/model/MeasurementBuilderUnitTests.scala | 6 +++--- .../za/co/absa/atum/model/ResultValueType.scala | 1 + .../co/absa/atum/model/dto/MeasureResultDTO.scala | 8 ++++---- .../functions/CreateOrUpdateAdditionalData.scala | 1 - 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala index 6ffa24860..1a5e634e2 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala @@ -138,10 +138,10 @@ object AtumMeasure { ): Column = { dataType match { case _: LongType => - // This is protection against long overflow, e.g. LongValue.MaxValue = 9223372036854775807: - // scala> sc.parallelize(List(LongValue.MaxValue, 1)).toDF.agg(sum("value")).take(1)(0)(0) + // This is protection against long overflow, e.g. Long.MaxValue = 9223372036854775807: + // scala> sc.parallelize(List(Long.MaxValue, 1)).toDF.agg(sum("value")).take(1)(0)(0) // res11: Any = -9223372036854775808 - // Converting to BigDecimalValue fixes the issue + // Converting to BigDecimal fixes the issue column.cast(DecimalType(38, 0)) case _: StringType => // Support for string type aggregation @@ -170,21 +170,21 @@ object AtumMeasure { /** * This method converts a given value to string. - * It is a workaround for different serializers generating different JSONs for BigDecimalValue. + * It is a workaround for different serializers generating different JSONs for BigDecimal. * See https://stackoverflow.com/questions/61973058/json-serialization-of-bigdecimal-returns-scientific-notation * * @param value A value to convert * @return A string representation of the value */ private def workaroundBigDecimalIssues(value: Any): String = - // If aggregated value is java.math.BigDecimalValue, convert it to scala.math.BigDecimalValue + // If aggregated value is java.math.BigDecimal, convert it to scala.math.BigDecimal value match { case v: java.math.BigDecimal => - // Convert the value to string to workaround different serializers generate different JSONs for BigDecimalValue + // Convert the value to string to workaround different serializers generate different JSONs for BigDecimal v.stripTrailingZeros // removes trailing zeros (2001.500000 -> 2001.5, but can introduce scientific notation (600.000 -> 6E+2) .toPlainString // converts to normal string (6E+2 -> "600") case v: BigDecimal => - // Convert the value to string to workaround different serializers generate different JSONs for BigDecimalValue + // Convert the value to string to workaround different serializers generate different JSONs for BigDecimal new java.math.BigDecimal( v.toString() ).stripTrailingZeros // removes trailing zeros (2001.500000 -> 2001.5, but can introduce scientific notation (600.000 -> 6E+2) diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala index b64060257..048349ad1 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderUnitTests.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue class MeasurementBuilderUnitTests extends AnyFlatSpec { "buildMeasurementDTO" should - "build MeasurementDTO for BigDecimalValue type of result value when Measure and MeasureResult provided" in { + "build MeasurementDTO for BigDecimal type of result value when Measure and MeasureResult provided" in { val measure = SumOfValuesOfColumn("col") val measureResult = MeasureResult(BigDecimal(1)) @@ -44,7 +44,7 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { } "buildMeasurementDTO" should - "build MeasurementDTO for BigDecimalValue type of result value when Measurement provided" in { + "build MeasurementDTO for BigDecimal type of result value when Measurement provided" in { val measure = SumOfValuesOfColumn("col") val measureResult = MeasureResult(BigDecimal(3.14)) @@ -70,7 +70,7 @@ class MeasurementBuilderUnitTests extends AnyFlatSpec { } "buildMeasurementDTO" should - "build MeasurementDTO for BigDecimalValue type of result value when measured by Agent" in { + "build MeasurementDTO for BigDecimal type of result value when measured by Agent" in { val measure = SumOfValuesOfColumn("col") val measureResult = MeasureResult("1", ResultValueType.BigDecimalValue) diff --git a/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala b/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala index 6d2177df1..f4c2edf70 100644 --- a/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala +++ b/model/src/main/scala/za/co/absa/atum/model/ResultValueType.scala @@ -19,6 +19,7 @@ package za.co.absa.atum.model import io.circe.{Decoder, Encoder} sealed trait ResultValueType + object ResultValueType { case object StringValue extends ResultValueType case object LongValue extends ResultValueType diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index eac2c6c91..29ad02001 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -21,14 +21,14 @@ import io.circe.generic.semiauto._ import za.co.absa.atum.model.ResultValueType case class MeasureResultDTO( - mainValue: MeasureResultDTO.TypedValue, - supportValues: Map[String, MeasureResultDTO.TypedValue] = Map.empty + mainValue: MeasureResultDTO.TypedValue, + supportValues: Map[String, MeasureResultDTO.TypedValue] = Map.empty ) object MeasureResultDTO { case class TypedValue( - value: String, - valueType: ResultValueType + value: String, + valueType: ResultValueType ) object TypedValue { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index f7b8da0d3..822a1cfaa 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.database.runs.functions - import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read From e3483f84faee7e318bda929fb2ee99706e1f7be3 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 4 Jul 2024 09:37:53 +0200 Subject: [PATCH 25/34] Applying GitHub comments --- .../agent/dispatcher/HttpDispatcher.scala | 5 +- .../absa/atum/model/utils/JsonImplicits.scala | 6 +- .../utils/SerializationUtilsUnitTests.scala | 70 +++++++++---------- .../absa/atum/server/api/http/Endpoints.scala | 1 - .../atum/server/model/PartitioningForDB.scala | 1 - .../CreateCheckpointEndpointUnitTests.scala | 1 - .../CreatePartitioningEndpointUnitTests.scala | 1 - .../GetFlowCheckpointsEndpointUnitTests.scala | 1 - 8 files changed, 40 insertions(+), 46 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index 2414d2880..03a78b717 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -22,9 +22,8 @@ import sttp.client3._ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} -import io.circe.generic.auto._ import io.circe.syntax.EncoderOps -import za.co.absa.atum.model.utils.JsonUtils.fromJson +import za.co.absa.atum.model.utils.JsonUtils.fromJsonString class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ @@ -53,7 +52,7 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log val response = backend.send(request) - fromJson[AtumContextDTO]( + fromJsonString[AtumContextDTO]( handleResponseBody(response) ) } diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala index b68b4087a..4db361320 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala @@ -21,15 +21,15 @@ import io.circe.syntax._ import io.circe.parser.decode object JsonUtils { - def asJson[T: Encoder](obj: T): String = { + def asJsonString[T: Encoder](obj: T): String = { obj.asJson.noSpaces } - def asJsonPretty[T: Encoder](obj: T): String = { + def asJsonStringPretty[T: Encoder](obj: T): String = { obj.asJson.spaces2 } - def fromJson[T: Decoder](jsonStr: String): T = { + def fromJsonString[T: Decoder](jsonStr: String): T = { decode[T](jsonStr) match { case Right(value) => value case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 5596ec84b..9c029e891 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -22,7 +22,7 @@ import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto._ -import za.co.absa.atum.model.utils.JsonUtils.fromJson +import za.co.absa.atum.model.utils.JsonUtils.fromJsonString import za.co.absa.atum.model.utils.SerializationUtilsTest.StringLinearization import java.time.{ZoneId, ZoneOffset, ZonedDateTime} @@ -31,7 +31,7 @@ import java.util.UUID class SerializationUtilsUnitTests extends AnyFlatSpecLike { // AdditionalDataDTO - "asJson" should "serialize AdditionalDataDTO into json string" in { + "asJsonString" should "serialize AdditionalDataDTO into json string" in { val additionalDataDTO = AdditionalDataSubmitDTO( Seq(PartitionDTO("key", "val")), Map[String, Option[String]]( @@ -53,7 +53,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJson" should "deserialize AdditionalDataDTO from json string" in { + "fromJsonString" should "deserialize AdditionalDataDTO from json string" in { val additionalDataDTOJson = """ |{"partitioning":[{"key":"key","value":"val"}], @@ -69,12 +69,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ), "testAuthor" ) - val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJson) + val actualAdditionalDataDTO = fromJsonString[AdditionalDataSubmitDTO](additionalDataDTOJson) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } - "asJson" should "serialize empty AdditionalDataDTO into json string" in { + "asJsonString" should "serialize empty AdditionalDataDTO into json string" in { val additionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") val expectedAdditionalDataJson = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" @@ -83,18 +83,18 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJson" should "deserialize empty AdditionalDataDTO from json string" in { + "fromJsonString" should "deserialize empty AdditionalDataDTO from json string" in { val additionalDataDTOJsonString = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) + val actualAdditionalDataDTO = fromJsonString[AdditionalDataSubmitDTO](additionalDataDTOJsonString) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } // AtumContextDTO - "asJson" should "serialize AtumContextDTO into json string" in { + "asJsonString" should "serialize AtumContextDTO into json string" in { val seqPartitionDTO = Seq(PartitionDTO("key", "val")) val seqMeasureDTO = Set(MeasureDTO("count", Seq("col"))) @@ -112,7 +112,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJson" should "deserialize AtumContextDTO from json string" in { + "fromJsonString" should "deserialize AtumContextDTO from json string" in { val atumContextDTOJson = """ |{ @@ -126,12 +126,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = fromJsonString[AtumContextDTO](atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } - "asJson" should "serialize AtumContextDTO without measures into json string" in { + "asJsonString" should "serialize AtumContextDTO without measures into json string" in { val seqPartitionDTO = Seq(PartitionDTO("key", "val")) val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO) @@ -142,19 +142,19 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJson" should "deserialize AtumContextDTO without measures from json string" in { + "fromJsonString" should "deserialize AtumContextDTO without measures from json string" in { val atumContextDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) - val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = fromJsonString[AtumContextDTO](atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } // CheckpointDTO - "asJson" should "serialize CheckpointDTO into json string" in { + "asJsonString" should "serialize CheckpointDTO into json string" in { val uuid = UUID.randomUUID() val seqPartitionDTO = Seq(PartitionDTO("key", "val")) val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneId.of("CET")) @@ -197,7 +197,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualCheckpointDTOJson == expectedCheckpointDTOJson) } - "fromJson" should "deserialize CheckpointDTO from json string" in { + "fromJsonString" should "deserialize CheckpointDTO from json string" in { val uuid = UUID.randomUUID() val seqPartitionDTO = Seq(PartitionDTO("key", "val")) val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneOffset.ofHours(2)) @@ -237,13 +237,13 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { measurements = setMeasurementDTO ) - val actualCheckpointDTO = fromJson[CheckpointDTO](checkpointDTOJson) + val actualCheckpointDTO = fromJsonString[CheckpointDTO](checkpointDTOJson) assert(actualCheckpointDTO == expectedCheckpointDTO) } // MeasureDTO - "asJson" should "serialize MeasureDTO into json string" in { + "asJsonString" should "serialize MeasureDTO into json string" in { val measureDTO = MeasureDTO("count", Seq("col")) val expectedMeasureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" @@ -252,17 +252,17 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualMeasureDTOJson == expectedMeasureDTOJson) } - "fromJson" should "deserialize MeasureDTO from json string" in { + "fromJsonString" should "deserialize MeasureDTO from json string" in { val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" val expectedMeasureDTO = MeasureDTO("count", Seq("col")) - val actualMeasureDTO = fromJson[MeasureDTO](measureDTOJson) + val actualMeasureDTO = fromJsonString[MeasureDTO](measureDTOJson) assert(actualMeasureDTO == expectedMeasureDTO) } // MeasurementDTO - "asJson" should "serialize MeasurementDTO into json string" in { + "asJsonString" should "serialize MeasurementDTO into json string" in { val measureDTO = MeasureDTO("count", Seq("col")) val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) @@ -281,7 +281,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualMeasurementDTOJson == expectedMeasurementDTOJson) } - "fromJson" should "deserialize MeasurementDTO from json string" in { + "fromJsonString" should "deserialize MeasurementDTO from json string" in { val measurementDTOJson = """ |{ @@ -294,13 +294,13 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - val actualMeasurementDTO = fromJson[MeasurementDTO](measurementDTOJson) + val actualMeasurementDTO = fromJsonString[MeasurementDTO](measurementDTOJson) assert(actualMeasurementDTO == expectedMeasurementDTO) } // MeasureResultDTO - "asJson" should "serialize MeasureResultDTO into json string" in { + "asJsonString" should "serialize MeasureResultDTO into json string" in { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" @@ -309,17 +309,17 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualMeasureResultDTOJson == expectedMeasureResultDTOJson) } - "fromJson" should "deserialize MeasureResultDTO from json string" in { + "fromJsonString" should "deserialize MeasureResultDTO from json string" in { val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) - val actualMeasureResultDTO = fromJson[MeasureResultDTO](measureResultDTOJson) + val actualMeasureResultDTO = fromJsonString[MeasureResultDTO](measureResultDTOJson) assert(actualMeasureResultDTO == expectedMeasureResultDTO) } // PartitionDTO - "asJson" should "serialize PartitionDTO into json string" in { + "asJsonString" should "serialize PartitionDTO into json string" in { val partitionDTO = PartitionDTO("key", "val") val expectedPartitionDTOJson = """{"key":"key","value":"val"}""" @@ -328,17 +328,17 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualPartitionDTOJson == expectedPartitionDTOJson) } - "fromJson" should "deserialize PartitionDTO from json string" in { + "fromJsonString" should "deserialize PartitionDTO from json string" in { val partitionDTOJson = """{"key":"key","value":"val"}""" val expectedPartitionDTO = PartitionDTO("key", "val") - val actualPartitionDTO = fromJson[PartitionDTO](partitionDTOJson) + val actualPartitionDTO = fromJsonString[PartitionDTO](partitionDTOJson) assert(actualPartitionDTO == expectedPartitionDTO) } // PartitioningDTO - "asJson" should "serialize PartitioningDTO into json string" in { + "asJsonString" should "serialize PartitioningDTO into json string" in { val partitionDTO = PartitionDTO("key", "val") val partitioningDTO = PartitioningSubmitDTO( @@ -353,7 +353,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } - "fromJson" should "deserialize PartitioningDTO from json string" in { + "fromJsonString" should "deserialize PartitioningDTO from json string" in { val partitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"authorIfNew":"authorTest"}""" val expectedPartitionDTO = PartitionDTO("key", "val") @@ -363,12 +363,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = fromJsonString[PartitioningSubmitDTO](partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } - "asJson" should "serialize PartitioningDTO with parent partitioning into json string" in { + "asJsonString" should "serialize PartitioningDTO with parent partitioning into json string" in { val partitionDTO = PartitionDTO("key", "val") val parentPartitionDTO = PartitionDTO("parentKey", "parentVal") @@ -392,7 +392,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { } - "asJson" should "serialize Seq[PartitionDTO] into json string" in { + "asJsonString" should "serialize Seq[PartitionDTO] into json string" in { val partitionDTO = Seq( PartitionDTO("key1", "val1"), PartitionDTO("key2", "val2"), @@ -405,7 +405,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualPartitionDTOJson == expectedPartitionDTOJson) } - "fromJson" should "deserialize PartitioningDTO with parent partitioning from json string" in { + "fromJsonString" should "deserialize PartitioningDTO with parent partitioning from json string" in { val partitioningDTOJson = """ |{ @@ -423,7 +423,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = fromJsonString[PartitioningSubmitDTO](partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 3feb9df10..f86103204 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.server.api.http -import io.circe.generic.auto.{exportDecoder, exportEncoder} import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.ztapir._ diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index 6e91fcfd4..4b0132d13 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -19,7 +19,6 @@ package za.co.absa.atum.server.model import io.circe.generic.semiauto._ import io.circe.{Decoder, Encoder} import za.co.absa.atum.model.dto.PartitioningDTO -import scala.collection.immutable.Seq private[server] case class PartitioningForDB private ( version: Int = 1, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index b14a1d011..482f6465c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.http -//import io.circe.generic.auto._ import io.circe.syntax.EncoderOps import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index cbc889e76..cb5cc7c96 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.http -//import io.circe.generic.auto._ import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub import sttp.client3.{UriContext, basicRequest} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala index 601cacbe0..7e8514e7c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.http -//import io.circe.generic.auto._ import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub import sttp.client3.{UriContext, basicRequest} From 5e0cf75948258732d3319b895aa541828170ae0c Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 4 Jul 2024 11:54:20 +0200 Subject: [PATCH 26/34] Applying GitHub suggestions --- .../server/api/database/DoobieImplicits.scala | 90 ++++--------------- .../flows/functions/GetFlowCheckpoints.scala | 11 +-- .../CreateOrUpdateAdditionalData.scala | 8 +- .../CreatePartitioningIfNotExists.scala | 18 ++-- .../GetPartitioningAdditionalData.scala | 8 +- .../GetPartitioningCheckpoints.scala | 11 +-- .../functions/GetPartitioningMeasures.scala | 8 +- .../runs/functions/WriteCheckpoint.scala | 18 ++-- 8 files changed, 45 insertions(+), 127 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index af66db146..89c7c027e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -20,7 +20,7 @@ import cats.Show import cats.data.NonEmptyList import doobie.{Get, Put} import doobie.postgres.implicits._ -import io.circe.Json +import io.circe.{Json => CirceJson} import org.postgresql.jdbc.PgArray import org.postgresql.util.PGobject @@ -28,13 +28,12 @@ import scala.util.{Failure, Success, Try} object DoobieImplicits { - private implicit val showPGobject: Show[PGobject] = Show.show(_.getValue.take(250)) private implicit val showPgArray: Show[PgArray] = Show.fromToString implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] .tmap(map => map.map { case (k, v) => k -> Option(v) }) - private def circeJsonListToPGJsonArrayString(jsonList: List[Json]): String = { + private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { val arrayElements = jsonList.map { x => // Convert to compact JSON string and escape inner quotes val escapedJsonString = x.noSpaces.replace("\"", "\\\"") @@ -45,6 +44,13 @@ object DoobieImplicits { arrayElements.mkString("{", ",", "}") } + private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { + Try(pgArray.getArray.asInstanceOf[List[String]].map(CirceJson.fromString)) match { + case Failure(exception) => Left(exception.toString) + case Success(value) => Right(value) + } + } + object Sequence { implicit val get: Get[Seq[String]] = Get[List[String]].map(_.toSeq) @@ -54,7 +60,7 @@ object DoobieImplicits { object Json { - implicit val jsonArrayPut: Put[List[Json]] = { + implicit val jsonArrayPut: Put[List[CirceJson]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("json[]") @@ -67,52 +73,19 @@ object DoobieImplicits { } } - implicit val jsonArrayGetUsingString: Get[List[String]] = { - def parsePgArray(a: PgArray): Either[String, List[String]] = { - Try(a.getArray.asInstanceOf[List[String]]) match { - case Failure(exception) => Left(exception.toString) - case Success(value) => Right(value) - } - } - + implicit val jsonArrayGet: Get[List[CirceJson]] = { Get.Advanced .other[PgArray]( NonEmptyList.of("json[]") ) - .temap(a => parsePgArray(a)) - } - - implicit val jsonPutUsingString: Put[String] = { - Put.Advanced - .other[PGobject]( - NonEmptyList.of("json") - ) - .tcontramap { a => - val o = new PGobject - o.setType("json") - o.setValue(a) - o - } - } - - implicit val jsonGetUsingString: Get[String] = { - Get.Advanced - .other[PGobject]( - NonEmptyList.of("json") - ) - .temap(a => - Try(a.getValue) match { - case Failure(exception) => Left(exception.toString) - case Success(value) => Right(value) - } - ) + .temap(pgArray => pgArrayToListOfCirceJson(pgArray)) } } object Jsonb { - implicit val jsonbArrayPut: Put[List[Json]] = { + implicit val jsonbArrayPut: Put[List[CirceJson]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("jsonb[]") @@ -125,45 +98,12 @@ object DoobieImplicits { } } - implicit val jsonbArrayGetUsingString: Get[List[String]] = { - def parsePgArray(a: PgArray): Either[String, List[String]] = { - Try(a.getArray.asInstanceOf[List[String]]) match { - case Failure(exception) => Left(exception.toString) - case Success(value) => Right(value) - } - } - + implicit val jsonbArrayGet: Get[List[CirceJson]] = { Get.Advanced .other[PgArray]( NonEmptyList.of("jsonb[]") ) - .temap(a => parsePgArray(a)) - } - - implicit val jsonbPutUsingString: Put[String] = { - Put.Advanced - .other[PGobject]( - NonEmptyList.of("jsonb") - ) - .tcontramap { a => - val o = new PGobject - o.setType("jsonb") - o.setValue(a) - o - } - } - - implicit val jsonbGetUsingString: Get[String] = { - Get.Advanced - .other[PGobject]( - NonEmptyList.of("jsonb") - ) - .temap(a => - Try(a.getValue) match { - case Failure(exception) => Left(exception.toString) - case Success(value) => Right(value) - } - ) + .temap(pgArray => pgArrayToListOfCirceJson(pgArray) ) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index b8102987e..8a90ddbaa 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -32,9 +32,9 @@ import zio.interop.catz._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits._ +import doobie.postgres.circe.jsonb.implicits.jsonbPut +import doobie.postgres.circe.jsonb.implicits.jsonbGet import io.circe.syntax.EncoderOps -import io.circe.generic.auto._ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task] { @@ -50,14 +50,11 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] override def sql(values: CheckpointQueryDTO)(implicit read: Read[CheckpointFromDB]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = partitioning.asJson.noSpaces + val partitioningNormalized = partitioning.asJson sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningNormalized - }, + $partitioningNormalized, ${values.limit}, ${values.checkpointName} ) AS ${Fragment.const(alias)};""" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 822a1cfaa..c28c74627 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -32,6 +32,7 @@ import zio.interop.catz._ import io.circe.syntax._ import doobie.postgres.implicits._ +import doobie.postgres.circe.jsonb.implicits.jsonbPut class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task] @@ -39,16 +40,13 @@ class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEn override def sql(values: AdditionalDataSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = partitioning.asJson.noSpaces + val partitioningJson = partitioning.asJson // implicits from Doobie can't handle Map[String, Option[String]] -> HStore, so we converted None to null basically val additionalDataNormalized = values.additionalData.map{ case (k, v) => (k, v.orNull)} sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningJsonString - }, + $partitioningJson, $additionalDataNormalized, ${values.author} ) ${Fragment.const(alias)};""" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 708add14e..65cab3a1e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -32,29 +32,25 @@ import zio._ import zio.interop.catz._ import io.circe.syntax._ +import doobie.postgres.circe.jsonb.implicits.jsonbPut + class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] with StandardStatusHandling { override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = partitioning.asJson.noSpaces + val partitioningJson = partitioning.asJson - val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => + val parentPartitioningJson = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - parentPartitioningForDB.asJson.noSpaces + parentPartitioningForDB.asJson } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningJsonString - }, + $partitioningJson, ${values.authorIfNew}, - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - parentPartitioningJsonString - } + $parentPartitioningJson ) ${Fragment.const(alias)};""" } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 961076ac8..e3d4ac86d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -31,22 +31,20 @@ import zio.{Task, URLayer, ZIO, ZLayer} import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues +import doobie.postgres.circe.jsonb.implicits.jsonbPut class GetPartitioningAdditionalData (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunction[PartitioningDTO, (String, Option[String]), Task] { - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString override val fieldsToSelect: Seq[String] = Seq("ad_name", "ad_value") override def sql(values: PartitioningDTO)(implicit read: Read[(String, Option[String])]): Fragment = { val partitioning: PartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(values) - val partitioningJsonString = partitioning.asJson.noSpaces + val partitioningJson = partitioning.asJson sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - partitioningJsonString - } + $partitioningJson ) ${Fragment.const(alias)};""" } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 52a6832f8..63e3ca459 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -31,9 +31,9 @@ import zio.interop.catz._ import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get -import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits._ +import doobie.postgres.circe.jsonb.implicits.jsonbPut +import doobie.postgres.circe.jsonb.implicits.jsonbGet class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task] { @@ -52,14 +52,11 @@ class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEng override def sql(values: CheckpointQueryDTO)(implicit read: Read[CheckpointFromDB]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = partitioning.asJson.noSpaces + val partitioningNormalized = partitioning.asJson sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningNormalized - }, + $partitioningNormalized, ${values.limit}, ${values.checkpointName} ) AS ${Fragment.const(alias)};""" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index 5d541417b..ed1694f3d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -31,22 +31,20 @@ import zio.interop.catz._ import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import doobie.postgres.circe.jsonb.implicits.jsonbPut class GetPartitioningMeasures (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunction[PartitioningDTO, MeasureDTO, Task] { - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString override val fieldsToSelect: Seq[String] = Seq("measure_name", "measured_columns") override def sql(values: PartitioningDTO)(implicit read: Read[MeasureDTO]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values) - val partitioningJsonString = partitioning.asJson.noSpaces + val partitioningJson = partitioning.asJson sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - partitioningJsonString - } + $partitioningJson ) ${Fragment.const(alias)};""" } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 273c59610..a196038c1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -30,12 +30,12 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.model.dto.MeasureResultDTO._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPut import doobie.postgres.circe.jsonb.implicits.jsonbGet +import doobie.postgres.circe.jsonb.implicits.jsonbPut import doobie.postgres.implicits._ -import io.circe.Json class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task] @@ -43,27 +43,21 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) override def sql(values: CheckpointDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = partitioning.asJson.noSpaces + val partitioningNormalized = partitioning.asJson - // List[StringValue] containing json data has to be properly escaped + // List[String] containing json data has to be properly escaped // It would be safer to use Json data type and derive Put instance val measurementsNormalized = { values.measurements.toList.map(_.asJson) } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningNormalized - }, + $partitioningNormalized, ${values.id}, ${values.name}, ${values.processStartTime}, ${values.processEndTime}, - ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPut - measurementsNormalized - }, + $measurementsNormalized, ${values.measuredByAtumAgent}, ${values.author} ) ${Fragment.const(alias)};""" From a75e988f788867566922f7bcb4a288d4b0da905e Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 4 Jul 2024 16:38:06 +0200 Subject: [PATCH 27/34] Applying GitHub suggestions --- .../agent/dispatcher/HttpDispatcher.scala | 13 ++-- .../absa/atum/model/utils/JsonImplicits.scala | 38 ---------- .../utils/SerializationUtilsUnitTests.scala | 71 +++++++++---------- .../CreateCheckpointEndpointUnitTests.scala | 7 +- 4 files changed, 44 insertions(+), 85 deletions(-) delete mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index 03a78b717..4fdfb8001 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -22,8 +22,7 @@ import sttp.client3._ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} -import io.circe.syntax.EncoderOps -import za.co.absa.atum.model.utils.JsonUtils.fromJsonString +import za.co.absa.atum.model.utils.JsonSyntaxSerialization.{asJsonString, fromJson} class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ @@ -48,19 +47,17 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def createPartitioning(partitioning: PartitioningSubmitDTO): AtumContextDTO = { val request = commonAtumRequest .post(createPartitioningEndpoint) - .body(partitioning.asJson.noSpaces) + .body(asJsonString(partitioning)) val response = backend.send(request) - fromJsonString[AtumContextDTO]( - handleResponseBody(response) - ) + fromJson[AtumContextDTO](handleResponseBody(response)) } override protected[agent] def saveCheckpoint(checkpoint: CheckpointDTO): Unit = { val request = commonAtumRequest .post(createCheckpointEndpoint) - .body(checkpoint.asJson.noSpaces) + .body(asJsonString(checkpoint)) val response = backend.send(request) @@ -70,7 +67,7 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def saveAdditionalData(additionalDataSubmitDTO: AdditionalDataSubmitDTO): Unit = { val request = commonAtumRequest .post(createAdditionalDataEndpoint) - .body(additionalDataSubmitDTO.asJson.noSpaces) + .body(asJsonString(additionalDataSubmitDTO)) val response = backend.send(request) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala deleted file mode 100644 index 4db361320..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonImplicits.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.utils - -import io.circe.{Encoder, Decoder} -import io.circe.syntax._ -import io.circe.parser.decode - -object JsonUtils { - def asJsonString[T: Encoder](obj: T): String = { - obj.asJson.noSpaces - } - - def asJsonStringPretty[T: Encoder](obj: T): String = { - obj.asJson.spaces2 - } - - def fromJsonString[T: Decoder](jsonStr: String): T = { - decode[T](jsonStr) match { - case Right(value) => value - case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") - } - } -} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 9c029e891..23dfbeee5 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -16,13 +16,12 @@ package za.co.absa.atum.model.utils -import io.circe.generic.auto._ import io.circe.syntax.EncoderOps import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto._ -import za.co.absa.atum.model.utils.JsonUtils.fromJsonString +import za.co.absa.atum.model.utils.JsonSyntaxSerialization.{asJsonString, fromJson} import za.co.absa.atum.model.utils.SerializationUtilsTest.StringLinearization import java.time.{ZoneId, ZoneOffset, ZonedDateTime} @@ -48,12 +47,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"additionalData":{"key1":"val1","key2":"val2","key3":null}, |"author":"testAuthor"} |""".linearize - val actualAdditionalDataJson = additionalDataDTO.asJson.noSpaces + val actualAdditionalDataJson = asJsonString(additionalDataDTO) assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJsonString" should "deserialize AdditionalDataDTO from json string" in { + "fromJson" should "deserialize AdditionalDataDTO from json string" in { val additionalDataDTOJson = """ |{"partitioning":[{"key":"key","value":"val"}], @@ -69,7 +68,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ), "testAuthor" ) - val actualAdditionalDataDTO = fromJsonString[AdditionalDataSubmitDTO](additionalDataDTOJson) + val actualAdditionalDataDTO = fromJson(additionalDataDTOJson) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -78,17 +77,17 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val additionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") val expectedAdditionalDataJson = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" - val actualAdditionalDataJson = additionalDataDTO.asJson.noSpaces + val actualAdditionalDataJson = asJsonString(additionalDataDTO) assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJsonString" should "deserialize empty AdditionalDataDTO from json string" in { + "fromJson" should "deserialize empty AdditionalDataDTO from json string" in { val additionalDataDTOJsonString = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - val actualAdditionalDataDTO = fromJsonString[AdditionalDataSubmitDTO](additionalDataDTOJsonString) + val actualAdditionalDataDTO = fromJson(additionalDataDTOJsonString) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -107,12 +106,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"measures":[{"measureName":"count","measuredColumns":["col"]}], |"additionalData":{} |}""".linearize - val actualAdditionalDataJson = atumContextDTO.asJson.noSpaces + val actualAdditionalDataJson = asJsonString(atumContextDTO) assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJsonString" should "deserialize AtumContextDTO from json string" in { + "fromJson" should "deserialize AtumContextDTO from json string" in { val atumContextDTOJson = """ |{ @@ -126,7 +125,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - val actualAtumContextDTO = fromJsonString[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = fromJson(atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -137,18 +136,18 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO) val expectedAdditionalDataJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" - val actualAdditionalDataJson = atumContextDTO.asJson.noSpaces + val actualAdditionalDataJson = asJsonString(atumContextDTO) assert(actualAdditionalDataJson == expectedAdditionalDataJson) } - "fromJsonString" should "deserialize AtumContextDTO without measures from json string" in { + "fromJson" should "deserialize AtumContextDTO without measures from json string" in { val atumContextDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) - val actualAtumContextDTO = fromJsonString[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = fromJson(atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -192,12 +191,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] |} |""".linearize - val actualCheckpointDTOJson = checkpointDTO.asJson.noSpaces + val actualCheckpointDTOJson = asJsonString(checkpointDTO) assert(actualCheckpointDTOJson == expectedCheckpointDTOJson) } - "fromJsonString" should "deserialize CheckpointDTO from json string" in { + "fromJson" should "deserialize CheckpointDTO from json string" in { val uuid = UUID.randomUUID() val seqPartitionDTO = Seq(PartitionDTO("key", "val")) val timeWithZone = ZonedDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneOffset.ofHours(2)) @@ -237,7 +236,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { measurements = setMeasurementDTO ) - val actualCheckpointDTO = fromJsonString[CheckpointDTO](checkpointDTOJson) + val actualCheckpointDTO = fromJson(checkpointDTOJson) assert(actualCheckpointDTO == expectedCheckpointDTO) } @@ -247,16 +246,16 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureDTO = MeasureDTO("count", Seq("col")) val expectedMeasureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" - val actualMeasureDTOJson = measureDTO.asJson.noSpaces + val actualMeasureDTOJson = asJsonString(measureDTO) assert(actualMeasureDTOJson == expectedMeasureDTOJson) } - "fromJsonString" should "deserialize MeasureDTO from json string" in { + "fromJson" should "deserialize MeasureDTO from json string" in { val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" val expectedMeasureDTO = MeasureDTO("count", Seq("col")) - val actualMeasureDTO = fromJsonString[MeasureDTO](measureDTOJson) + val actualMeasureDTO = fromJson(measureDTOJson) assert(actualMeasureDTO == expectedMeasureDTO) } @@ -276,12 +275,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"supportValues":{}} |} |""".linearize - val actualMeasurementDTOJson = measurementDTO.asJson.noSpaces + val actualMeasurementDTOJson = asJsonString(measurementDTO) assert(actualMeasurementDTOJson == expectedMeasurementDTOJson) } - "fromJsonString" should "deserialize MeasurementDTO from json string" in { + "fromJson" should "deserialize MeasurementDTO from json string" in { val measurementDTOJson = """ |{ @@ -294,7 +293,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - val actualMeasurementDTO = fromJsonString[MeasurementDTO](measurementDTOJson) + val actualMeasurementDTO = fromJson(measurementDTOJson) assert(actualMeasurementDTO == expectedMeasurementDTO) } @@ -304,16 +303,16 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" - val actualMeasureResultDTOJson = measureResultDTO.asJson.noSpaces + val actualMeasureResultDTOJson = asJsonString(measureResultDTO) assert(actualMeasureResultDTOJson == expectedMeasureResultDTOJson) } - "fromJsonString" should "deserialize MeasureResultDTO from json string" in { + "fromJson" should "deserialize MeasureResultDTO from json string" in { val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) - val actualMeasureResultDTO = fromJsonString[MeasureResultDTO](measureResultDTOJson) + val actualMeasureResultDTO = fromJson(measureResultDTOJson) assert(actualMeasureResultDTO == expectedMeasureResultDTO) } @@ -323,16 +322,16 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val partitionDTO = PartitionDTO("key", "val") val expectedPartitionDTOJson = """{"key":"key","value":"val"}""" - val actualPartitionDTOJson = partitionDTO.asJson.noSpaces + val actualPartitionDTOJson = asJsonString(partitionDTO) assert(actualPartitionDTOJson == expectedPartitionDTOJson) } - "fromJsonString" should "deserialize PartitionDTO from json string" in { + "fromJson" should "deserialize PartitionDTO from json string" in { val partitionDTOJson = """{"key":"key","value":"val"}""" val expectedPartitionDTO = PartitionDTO("key", "val") - val actualPartitionDTO = fromJsonString[PartitionDTO](partitionDTOJson) + val actualPartitionDTO = fromJson(partitionDTOJson) assert(actualPartitionDTO == expectedPartitionDTO) } @@ -348,12 +347,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ) val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" - val actualPartitioningDTOJson = partitioningDTO.asJson.noSpaces + val actualPartitioningDTOJson = asJsonString(partitioningDTO) assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } - "fromJsonString" should "deserialize PartitioningDTO from json string" in { + "fromJson" should "deserialize PartitioningDTO from json string" in { val partitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"authorIfNew":"authorTest"}""" val expectedPartitionDTO = PartitionDTO("key", "val") @@ -363,7 +362,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJsonString[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = fromJson(partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } @@ -386,7 +385,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"authorIfNew":"authorTest" |} |""".linearize - val actualPartitioningDTOJson = partitioningDTO.asJson.noSpaces + val actualPartitioningDTOJson = asJsonString(partitioningDTO) assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } @@ -400,12 +399,12 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ) val expectedPartitionDTOJson = """[{"key":"key1","value":"val1"},{"key":"key2","value":"val2"},{"key":"key3","value":"val3"}]""" - val actualPartitionDTOJson = partitionDTO.asJson.noSpaces + val actualPartitionDTOJson = asJsonString(partitionDTO) assert(actualPartitionDTOJson == expectedPartitionDTOJson) } - "fromJsonString" should "deserialize PartitioningDTO with parent partitioning from json string" in { + "fromJson" should "deserialize PartitioningDTO with parent partitioning from json string" in { val partitioningDTOJson = """ |{ @@ -423,7 +422,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJsonString[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = fromJson(partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 482f6465c..0e9586fb6 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -25,6 +25,7 @@ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.utils.JsonSyntaxSerialization.asJsonString import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} @@ -62,7 +63,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w suite("CreateCheckpointEndpointSuite")( test("Returns expected CheckpointDTO") { val response = request - .body(checkpointDTO1.asJson.noSpaces) + .body(asJsonString(checkpointDTO1)) .send(backendStub) val body = response.map(_.body) @@ -72,7 +73,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected BadRequest") { val response = request - .body(checkpointDTO2.asJson.noSpaces) + .body(asJsonString(checkpointDTO2)) .send(backendStub) val statusCode = response.map(_.code) @@ -81,7 +82,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected InternalServerError") { val response = request - .body(checkpointDTO3.asJson.noSpaces) + .body(asJsonString(checkpointDTO3)) .send(backendStub) val statusCode = response.map(_.code) From 0e4af21cbe4c237734a5c3bc006ae3fdcefdfa2a Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 4 Jul 2024 16:38:16 +0200 Subject: [PATCH 28/34] Applying GitHub suggestions --- .../model/utils/JsonSyntaxSerialization.scala | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala new file mode 100644 index 000000000..3f566ae3d --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.utils + +import io.circe.parser.decode +import io.circe.syntax._ +import io.circe.{Decoder, Encoder} + +object JsonSyntaxSerialization { + def asJsonString[T: Encoder](obj: T): String = { + obj.asJson.noSpaces + } + + def asJsonStringPretty[T: Encoder](obj: T): String = { + obj.asJson.spaces2 + } + + def fromJson[T: Decoder](jsonStr: String): T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + } + } + +} From 5208392ed995e6a89079e126811775fa40af2b45 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 5 Jul 2024 10:17:03 +0200 Subject: [PATCH 29/34] Fixing some test cases --- .../model/dto/AdditionalDataSubmitDTO.scala | 4 +- .../utils/SerializationUtilsUnitTests.scala | 41 ++++++++++--------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala index 9fe3bf8e8..529d39a5a 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/AdditionalDataSubmitDTO.scala @@ -27,6 +27,6 @@ case class AdditionalDataSubmitDTO ( ) object AdditionalDataSubmitDTO { - implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder - implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder + implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder[AdditionalDataSubmitDTO] + implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder[AdditionalDataSubmitDTO] } diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 23dfbeee5..f9d13baf1 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.model.utils -import io.circe.syntax.EncoderOps import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue @@ -58,17 +57,17 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |{"partitioning":[{"key":"key","value":"val"}], |"additionalData":{"key1":"val1","key2":"val2"}, |"author":"testAuthor"} - |""".linearize + |""".stripMargin val expectedAdditionalDataDTO = AdditionalDataSubmitDTO( - Seq[PartitionDTO](PartitionDTO("key", "val")), - Map[String, Option[String]]( + Seq(PartitionDTO("key", "val")), + Map( "key1" -> Some("val1"), "key2" -> Some("val2") ), "testAuthor" ) - val actualAdditionalDataDTO = fromJson(additionalDataDTOJson) + val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJson) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -83,11 +82,14 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { } "fromJson" should "deserialize empty AdditionalDataDTO from json string" in { - val additionalDataDTOJsonString = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" + val additionalDataDTOJsonString = + """ + |{"partitioning":[],"additionalData":{},"author":"testAuthor"} + |""".stripMargin val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - val actualAdditionalDataDTO = fromJson(additionalDataDTOJsonString) + val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -118,14 +120,14 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"partitioning":[{"key":"key","value":"val"}], |"measures":[{"measureName":"count","measuredColumns":["col"]}], |"additionalData":{} - |}""".linearize + |}""".stripMargin val seqPartitionDTO = Seq(PartitionDTO("key", "val")) val seqMeasureDTO = Set(MeasureDTO("count", Seq("col"))) val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - val actualAtumContextDTO = fromJson(atumContextDTOJson) + val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -147,7 +149,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) - val actualAtumContextDTO = fromJson(atumContextDTOJson) + val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -214,7 +216,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"measurements":[{"measure":{"measureName":"count","measuredColumns":["col"]}, |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] |} - |""".linearize + |""".stripMargin val setMeasurementDTO = Set( @@ -236,7 +238,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { measurements = setMeasurementDTO ) - val actualCheckpointDTO = fromJson(checkpointDTOJson) + val actualCheckpointDTO = fromJson[CheckpointDTO](checkpointDTOJson) assert(actualCheckpointDTO == expectedCheckpointDTO) } @@ -255,7 +257,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" val expectedMeasureDTO = MeasureDTO("count", Seq("col")) - val actualMeasureDTO = fromJson(measureDTOJson) + val actualMeasureDTO = fromJson[MeasureDTO](measureDTOJson) assert(actualMeasureDTO == expectedMeasureDTO) } @@ -293,7 +295,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - val actualMeasurementDTO = fromJson(measurementDTOJson) + val actualMeasurementDTO = fromJson[MeasurementDTO](measurementDTOJson) assert(actualMeasurementDTO == expectedMeasurementDTO) } @@ -312,7 +314,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) - val actualMeasureResultDTO = fromJson(measureResultDTOJson) + val actualMeasureResultDTO = fromJson[MeasureResultDTO](measureResultDTOJson) assert(actualMeasureResultDTO == expectedMeasureResultDTO) } @@ -331,7 +333,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val partitionDTOJson = """{"key":"key","value":"val"}""" val expectedPartitionDTO = PartitionDTO("key", "val") - val actualPartitionDTO = fromJson(partitionDTOJson) + val actualPartitionDTO = fromJson[PartitionDTO](partitionDTOJson) assert(actualPartitionDTO == expectedPartitionDTO) } @@ -362,7 +364,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJson(partitioningDTOJson) + val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } @@ -390,7 +392,6 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } - "asJsonString" should "serialize Seq[PartitionDTO] into json string" in { val partitionDTO = Seq( PartitionDTO("key1", "val1"), @@ -412,7 +413,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"parentPartitioning":[{"key":"parentKey","value":"parentVal"}], |"authorIfNew":"authorTest" |} - |""".linearize + |""".stripMargin val expectedPartitionDTO = PartitionDTO("key", "val") val expectedParentPartitionDTO = PartitionDTO("parentKey", "parentVal") @@ -422,7 +423,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJson(partitioningDTOJson) + val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) assert(actualPartitioningDTO == expectedPartitioningDTO) } From 4b457a3bee0650f0ce183911a61ed0fc4f25d1f9 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 8 Jul 2024 10:23:45 +0200 Subject: [PATCH 30/34] implementing GitHub changes --- .../agent/dispatcher/HttpDispatcher.scala | 10 ++-- ...ation.scala => JsonSyntaxExtensions.scala} | 25 +++++----- .../utils/SerializationUtilsUnitTests.scala | 48 +++++++++---------- .../CreateCheckpointEndpointUnitTests.scala | 9 ++-- 4 files changed, 46 insertions(+), 46 deletions(-) rename model/src/main/scala/za/co/absa/atum/model/utils/{JsonSyntaxSerialization.scala => JsonSyntaxExtensions.scala} (59%) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index 4fdfb8001..31821a05e 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -22,7 +22,7 @@ import sttp.client3._ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} -import za.co.absa.atum.model.utils.JsonSyntaxSerialization.{asJsonString, fromJson} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ @@ -47,17 +47,17 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def createPartitioning(partitioning: PartitioningSubmitDTO): AtumContextDTO = { val request = commonAtumRequest .post(createPartitioningEndpoint) - .body(asJsonString(partitioning)) + .body(partitioning.asJsonString) val response = backend.send(request) - fromJson[AtumContextDTO](handleResponseBody(response)) + handleResponseBody(response).as[AtumContextDTO] } override protected[agent] def saveCheckpoint(checkpoint: CheckpointDTO): Unit = { val request = commonAtumRequest .post(createCheckpointEndpoint) - .body(asJsonString(checkpoint)) + .body(checkpoint.asJsonString) val response = backend.send(request) @@ -67,7 +67,7 @@ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Log override protected[agent] def saveAdditionalData(additionalDataSubmitDTO: AdditionalDataSubmitDTO): Unit = { val request = commonAtumRequest .post(createAdditionalDataEndpoint) - .body(asJsonString(additionalDataSubmitDTO)) + .body(additionalDataSubmitDTO.asJsonString) val response = backend.send(request) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala similarity index 59% rename from model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala rename to model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index 3f566ae3d..199109b0f 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxSerialization.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -20,20 +20,21 @@ import io.circe.parser.decode import io.circe.syntax._ import io.circe.{Decoder, Encoder} -object JsonSyntaxSerialization { - def asJsonString[T: Encoder](obj: T): String = { - obj.asJson.noSpaces - } +object JsonSyntaxExtensions { - def asJsonStringPretty[T: Encoder](obj: T): String = { - obj.asJson.spaces2 - } + implicit class JsonSerializationSyntax[T: Encoder](val obj: T) { + def asJsonString: String = obj.asJson.noSpaces - def fromJson[T: Decoder](jsonStr: String): T = { - decode[T](jsonStr) match { - case Right(value) => value - case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + def asJsonStringPretty: String = obj.asJson.spaces2 + } + + implicit class JsonDeserializationSyntax(val jsonStr: String) { + def as[T: Decoder]: T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + } + } } - } } diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index f9d13baf1..bd11a7ffb 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -20,7 +20,7 @@ import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto._ -import za.co.absa.atum.model.utils.JsonSyntaxSerialization.{asJsonString, fromJson} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.model.utils.SerializationUtilsTest.StringLinearization import java.time.{ZoneId, ZoneOffset, ZonedDateTime} @@ -46,7 +46,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"additionalData":{"key1":"val1","key2":"val2","key3":null}, |"author":"testAuthor"} |""".linearize - val actualAdditionalDataJson = asJsonString(additionalDataDTO) + val actualAdditionalDataJson = additionalDataDTO.asJsonString assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -67,7 +67,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ), "testAuthor" ) - val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJson) + val actualAdditionalDataDTO = additionalDataDTOJson.as[AdditionalDataSubmitDTO] assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -76,7 +76,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val additionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") val expectedAdditionalDataJson = """{"partitioning":[],"additionalData":{},"author":"testAuthor"}""" - val actualAdditionalDataJson = asJsonString(additionalDataDTO) + val actualAdditionalDataJson = additionalDataDTO.asJsonString assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -89,7 +89,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAdditionalDataDTO = AdditionalDataSubmitDTO(Seq.empty, Map.empty, "testAuthor") - val actualAdditionalDataDTO = fromJson[AdditionalDataSubmitDTO](additionalDataDTOJsonString) + val actualAdditionalDataDTO = additionalDataDTOJsonString.as[AdditionalDataSubmitDTO] assert(actualAdditionalDataDTO == expectedAdditionalDataDTO) } @@ -108,7 +108,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"measures":[{"measureName":"count","measuredColumns":["col"]}], |"additionalData":{} |}""".linearize - val actualAdditionalDataJson = asJsonString(atumContextDTO) + val actualAdditionalDataJson = atumContextDTO.asJsonString assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -127,7 +127,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAtumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO, measures = seqMeasureDTO) - val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = atumContextDTOJson.as[AtumContextDTO] assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -138,7 +138,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val atumContextDTO = AtumContextDTO(partitioning = seqPartitionDTO) val expectedAdditionalDataJson = """{"partitioning":[{"key":"key","value":"val"}],"measures":[],"additionalData":{}}""" - val actualAdditionalDataJson = asJsonString(atumContextDTO) + val actualAdditionalDataJson = atumContextDTO.asJsonString assert(actualAdditionalDataJson == expectedAdditionalDataJson) } @@ -149,7 +149,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedSeqPartitionDTO = Seq(PartitionDTO("key", "val")) val expectedAtumContextDTO = AtumContextDTO(partitioning = expectedSeqPartitionDTO) - val actualAtumContextDTO = fromJson[AtumContextDTO](atumContextDTOJson) + val actualAtumContextDTO = atumContextDTOJson.as[AtumContextDTO] assert(actualAtumContextDTO == expectedAtumContextDTO) } @@ -193,7 +193,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"result":{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}}] |} |""".linearize - val actualCheckpointDTOJson = asJsonString(checkpointDTO) + val actualCheckpointDTOJson = checkpointDTO.asJsonString assert(actualCheckpointDTOJson == expectedCheckpointDTOJson) } @@ -238,7 +238,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { measurements = setMeasurementDTO ) - val actualCheckpointDTO = fromJson[CheckpointDTO](checkpointDTOJson) + val actualCheckpointDTO = checkpointDTOJson.as[CheckpointDTO] assert(actualCheckpointDTO == expectedCheckpointDTO) } @@ -248,7 +248,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureDTO = MeasureDTO("count", Seq("col")) val expectedMeasureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" - val actualMeasureDTOJson = asJsonString(measureDTO) + val actualMeasureDTOJson = measureDTO.asJsonString assert(actualMeasureDTOJson == expectedMeasureDTOJson) } @@ -257,7 +257,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureDTOJson = """{"measureName":"count","measuredColumns":["col"]}""" val expectedMeasureDTO = MeasureDTO("count", Seq("col")) - val actualMeasureDTO = fromJson[MeasureDTO](measureDTOJson) + val actualMeasureDTO = measureDTOJson.as[MeasureDTO] assert(actualMeasureDTO == expectedMeasureDTO) } @@ -277,7 +277,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"supportValues":{}} |} |""".linearize - val actualMeasurementDTOJson = asJsonString(measurementDTO) + val actualMeasurementDTOJson = measurementDTO.asJsonString assert(actualMeasurementDTOJson == expectedMeasurementDTOJson) } @@ -295,7 +295,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasurementDTO = MeasurementDTO(measureDTO, measureResultDTO) - val actualMeasurementDTO = fromJson[MeasurementDTO](measurementDTOJson) + val actualMeasurementDTO = measurementDTOJson.as[MeasurementDTO] assert(actualMeasurementDTO == expectedMeasurementDTO) } @@ -305,7 +305,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) val expectedMeasureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" - val actualMeasureResultDTOJson = asJsonString(measureResultDTO) + val actualMeasureResultDTOJson = measureResultDTO.asJsonString assert(actualMeasureResultDTOJson == expectedMeasureResultDTOJson) } @@ -314,7 +314,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val measureResultDTOJson = """{"mainValue":{"value":"1","valueType":"Long"},"supportValues":{}}""" val expectedMeasureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.LongValue)) - val actualMeasureResultDTO = fromJson[MeasureResultDTO](measureResultDTOJson) + val actualMeasureResultDTO = measureResultDTOJson.as[MeasureResultDTO] assert(actualMeasureResultDTO == expectedMeasureResultDTO) } @@ -324,7 +324,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val partitionDTO = PartitionDTO("key", "val") val expectedPartitionDTOJson = """{"key":"key","value":"val"}""" - val actualPartitionDTOJson = asJsonString(partitionDTO) + val actualPartitionDTOJson = partitionDTO.asJsonString assert(actualPartitionDTOJson == expectedPartitionDTOJson) } @@ -333,7 +333,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val partitionDTOJson = """{"key":"key","value":"val"}""" val expectedPartitionDTO = PartitionDTO("key", "val") - val actualPartitionDTO = fromJson[PartitionDTO](partitionDTOJson) + val actualPartitionDTO = partitionDTOJson.as[PartitionDTO] assert(actualPartitionDTO == expectedPartitionDTO) } @@ -349,7 +349,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ) val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" - val actualPartitioningDTOJson = asJsonString(partitioningDTO) + val actualPartitioningDTOJson = partitioningDTO.asJsonString assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } @@ -364,7 +364,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = partitioningDTOJson.as[PartitioningSubmitDTO] assert(actualPartitioningDTO == expectedPartitioningDTO) } @@ -387,7 +387,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { |"authorIfNew":"authorTest" |} |""".linearize - val actualPartitioningDTOJson = asJsonString(partitioningDTO) + val actualPartitioningDTOJson = partitioningDTO.asJsonString assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) } @@ -400,7 +400,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { ) val expectedPartitionDTOJson = """[{"key":"key1","value":"val1"},{"key":"key2","value":"val2"},{"key":"key3","value":"val3"}]""" - val actualPartitionDTOJson = asJsonString(partitionDTO) + val actualPartitionDTOJson = partitionDTO.asJsonString assert(actualPartitionDTOJson == expectedPartitionDTOJson) } @@ -423,7 +423,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val actualPartitioningDTO = fromJson[PartitioningSubmitDTO](partitioningDTOJson) + val actualPartitioningDTO = partitioningDTOJson.as[PartitioningSubmitDTO] assert(actualPartitioningDTO == expectedPartitioningDTO) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index 0e9586fb6..c3ec8f604 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.api.http -import io.circe.syntax.EncoderOps import org.mockito.Mockito.{mock, when} import sttp.client3.testing.SttpBackendStub import sttp.client3.{UriContext, basicRequest} @@ -25,7 +24,7 @@ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.utils.JsonSyntaxSerialization.asJsonString +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} @@ -63,7 +62,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w suite("CreateCheckpointEndpointSuite")( test("Returns expected CheckpointDTO") { val response = request - .body(asJsonString(checkpointDTO1)) + .body(checkpointDTO1.asJsonString) .send(backendStub) val body = response.map(_.body) @@ -73,7 +72,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected BadRequest") { val response = request - .body(asJsonString(checkpointDTO2)) + .body(checkpointDTO2.asJsonString) .send(backendStub) val statusCode = response.map(_.code) @@ -82,7 +81,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected InternalServerError") { val response = request - .body(asJsonString(checkpointDTO3)) + .body(checkpointDTO3.asJsonString) .send(backendStub) val statusCode = response.map(_.code) From 21885c4f695e9d9c029639527a573008be674628 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 8 Jul 2024 10:38:44 +0200 Subject: [PATCH 31/34] removing vals --- .../za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index 199109b0f..29960a421 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -22,13 +22,13 @@ import io.circe.{Decoder, Encoder} object JsonSyntaxExtensions { - implicit class JsonSerializationSyntax[T: Encoder](val obj: T) { + implicit class JsonSerializationSyntax[T: Encoder](obj: T) { def asJsonString: String = obj.asJson.noSpaces def asJsonStringPretty: String = obj.asJson.spaces2 } - implicit class JsonDeserializationSyntax(val jsonStr: String) { + implicit class JsonDeserializationSyntax(jsonStr: String) { def as[T: Decoder]: T = { decode[T](jsonStr) match { case Right(value) => value From 24a50180bb6f4403a6083b711a1cefbccb2066fc Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 8 Jul 2024 10:42:55 +0200 Subject: [PATCH 32/34] removing unsed pretty serialization --- .../za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index 29960a421..4a95546a9 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -24,8 +24,6 @@ object JsonSyntaxExtensions { implicit class JsonSerializationSyntax[T: Encoder](obj: T) { def asJsonString: String = obj.asJson.noSpaces - - def asJsonStringPretty: String = obj.asJson.spaces2 } implicit class JsonDeserializationSyntax(jsonStr: String) { From f0878b058d7c97bc478dea5a77faf03e68cf4b0c Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 8 Jul 2024 11:44:16 +0200 Subject: [PATCH 33/34] removing explicit serialization --- .../api/http/CreateCheckpointEndpointUnitTests.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index c3ec8f604..0621d7ca4 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -24,7 +24,6 @@ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} @@ -62,7 +61,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w suite("CreateCheckpointEndpointSuite")( test("Returns expected CheckpointDTO") { val response = request - .body(checkpointDTO1.asJsonString) + .body(checkpointDTO1) .send(backendStub) val body = response.map(_.body) @@ -72,7 +71,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected BadRequest") { val response = request - .body(checkpointDTO2.asJsonString) + .body(checkpointDTO2) .send(backendStub) val statusCode = response.map(_.code) @@ -81,7 +80,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected InternalServerError") { val response = request - .body(checkpointDTO3.asJsonString) + .body(checkpointDTO3) .send(backendStub) val statusCode = response.map(_.code) From 479ec333fde7f75677f241d18090989d8bcf4289 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 9 Jul 2024 16:19:40 +0200 Subject: [PATCH 34/34] removing jsonb get method --- project/Dependencies.scala | 1 - .../server/api/database/DoobieImplicits.scala | 32 ++++++++++--------- .../flows/functions/GetFlowCheckpoints.scala | 2 +- .../GetPartitioningCheckpoints.scala | 2 +- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c6cb6a38d..b391a87fa 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -106,7 +106,6 @@ object Dependencies { val tapirOrg = "com.softwaremill.sttp.tapir" val http4sOrg = "org.http4s" val faDbOrg = "za.co.absa.fa-db" - val playOrg = "org.playframework" val sbtOrg = "com.github.sbt" val logbackOrg = "ch.qos.logback" val awsSdkOrg = "software.amazon.awssdk" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 89c7c027e..28163b264 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -18,37 +18,46 @@ package za.co.absa.atum.server.api.database import cats.Show import cats.data.NonEmptyList -import doobie.{Get, Put} import doobie.postgres.implicits._ +import doobie.{Get, Put} import io.circe.{Json => CirceJson} import org.postgresql.jdbc.PgArray import org.postgresql.util.PGobject +import io.circe.parser._ -import scala.util.{Failure, Success, Try} +import scala.util.Try -object DoobieImplicits { +package object DoobieImplicits { private implicit val showPgArray: Show[PgArray] = Show.fromToString implicit val getMapWithOptionStringValues: Get[Map[String, Option[String]]] = Get[Map[String, String]] - .tmap(map => map.map { case (k, v) => k -> Option(v) }) + .tmap(map => map.map { case (k, v) => k -> Option(v) }) private def circeJsonListToPGJsonArrayString(jsonList: List[CirceJson]): String = { val arrayElements = jsonList.map { x => // Convert to compact JSON string and escape inner quotes val escapedJsonString = x.noSpaces.replace("\"", "\\\"") // Wrap in double quotes for the array element - s"\"$escapedJsonString\"" + s""""$escapedJsonString"""" } arrayElements.mkString("{", ",", "}") } private def pgArrayToListOfCirceJson(pgArray: PgArray): Either[String, List[CirceJson]] = { - Try(pgArray.getArray.asInstanceOf[List[String]].map(CirceJson.fromString)) match { - case Failure(exception) => Left(exception.toString) - case Success(value) => Right(value) + Try { + Option(pgArray.getArray) match { + case Some(array: Array[_]) => array.collect { + case str: String => parse(str).toTry.get + case other => parse(other.toString).toTry.get + }.toList + case None => List.empty[CirceJson] + case _ => throw new IllegalArgumentException("Unexpected type encountered.") + } } + .toEither + .left.map(_.getMessage) } object Sequence { @@ -98,13 +107,6 @@ object DoobieImplicits { } } - implicit val jsonbArrayGet: Get[List[CirceJson]] = { - Get.Advanced - .other[PgArray]( - NonEmptyList.of("jsonb[]") - ) - .temap(pgArray => pgArrayToListOfCirceJson(pgArray) ) - } } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 8a90ddbaa..a145d55c3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -33,7 +33,7 @@ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut -import doobie.postgres.circe.jsonb.implicits.jsonbGet +import doobie.postgres.circe.json.implicits.jsonGet import io.circe.syntax.EncoderOps class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 63e3ca459..4427c4797 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -33,7 +33,7 @@ import io.circe.syntax._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ import doobie.postgres.circe.jsonb.implicits.jsonbPut -import doobie.postgres.circe.jsonb.implicits.jsonbGet +import doobie.postgres.circe.json.implicits.jsonGet class GetPartitioningCheckpoints (implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunction[CheckpointQueryDTO, CheckpointFromDB, Task] {