From 809044a736ba676abcd920dba23a2f8167363b4e Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 9 Sep 2024 13:38:35 +0200 Subject: [PATCH] intersect instead of iteration --- .../scala/za/co/absa/atum/agent/AtumContext.scala | 11 ++++------- build.sbt | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index fe6075d4a..501d805a3 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -53,14 +53,11 @@ class AtumContext private[agent] ( * @return the sub-partition context */ def subPartitionContext(subPartitions: AtumPartitions): AtumContext = { - val newPartitions = subPartitions.foldLeft(atumPartitions) { case (acc, (k, v)) => - if (acc.contains(k)) { - throw PartitioningUpdateException(s"Partition key '$k' already exists. Updates are not allowed.") - } else { - acc + (k -> v) - } + val overlap = atumPartitions.keys.toSet.intersect(subPartitions.keys.toSet) + if (overlap.nonEmpty) { + throw PartitioningUpdateException(s"Partition keys '$overlap' already exist. Updates are not allowed.") } - agent.getOrCreateAtumSubContext(newPartitions)(this) + agent.getOrCreateAtumSubContext(atumPartitions ++ subPartitions)(this) } private def takeMeasurements(df: DataFrame): Set[MeasurementDTO] = { diff --git a/build.sbt b/build.sbt index 1a9f89ed5..89e77602f 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ import Dependencies.* import Dependencies.Versions.spark3 import VersionAxes.* -ThisBuild / scalaVersion := Setup.scala213.asString // default version TODO +ThisBuild / scalaVersion := Setup.scala212.asString // default version TODO ThisBuild / versionScheme := Some("early-semver")