From 9480cffee457beca1de3b7d695cb6f27c344a0ce Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 3 Jun 2021 16:42:20 +0200 Subject: [PATCH] Improve integration tests (#2511) * Improve integration tests * Update cluster configuration for integration tests Co-authored-by: Simon Dumas --- tests/docker/docker-compose-cassandra.yml | 75 ++++++++--- tests/docker/docker-compose-ci.yml | 33 +++-- tests/docker/docker-compose-postgres.yml | 61 +++++++-- tests/src/test/resources/application.conf | 1 + .../resources/elasticsearch/template.json | 2 +- .../epfl/bluebrain/nexus/tests/BaseSpec.scala | 34 ++++- .../nexus/tests/ElasticsearchDsl.scala | 15 ++- .../bluebrain/nexus/tests/HttpClient.scala | 32 +++-- .../epfl/bluebrain/nexus/tests/Identity.scala | 48 ++++++- .../nexus/tests/admin/OrgsSpec.scala | 19 +-- .../nexus/tests/admin/ProjectsSpec.scala | 24 +--- .../nexus/tests/config/TestsConfig.scala | 2 +- .../bluebrain/nexus/tests/iam/AclDsl.scala | 4 +- .../bluebrain/nexus/tests/iam/AclsSpec.scala | 23 +--- .../nexus/tests/kg/ArchiveSpec.scala | 19 +-- .../nexus/tests/kg/CompositeViewsSpec.scala | 38 +++--- .../nexus/tests/kg/DiskStorageSpec.scala | 1 + .../bluebrain/nexus/tests/kg/EventsSpec.scala | 119 +++++++++--------- .../nexus/tests/kg/RemoteStorageSpec.scala | 6 +- .../nexus/tests/kg/ResourcesSpec.scala | 25 +--- .../nexus/tests/kg/S3StorageSpec.scala | 6 +- .../nexus/tests/kg/StorageSpec.scala | 19 +-- .../nexus/tests/kg/VersionSpec.scala | 3 +- .../bluebrain/nexus/tests/kg/ViewsSpec.scala | 27 ++-- 24 files changed, 357 insertions(+), 279 deletions(-) diff --git a/tests/docker/docker-compose-cassandra.yml b/tests/docker/docker-compose-cassandra.yml index 258dab7426..876591fbc8 100644 --- a/tests/docker/docker-compose-cassandra.yml +++ b/tests/docker/docker-compose-cassandra.yml @@ -4,6 +4,7 @@ services: depends_on: - keycloak - elasticsearch + - blazegraph - cassandra - storage-service - minio @@ -12,7 +13,7 @@ services: KAMON_ENABLED: "false" image: bluebrain/nexus-delta:latest entrypoint: ["bin/wait-for-it.sh", "-s", "-t", "0", "cassandra:9042", "--", "./bin/delta-app", - "-Xmx2G", + "-Xmx512m", "-Dapp.http.interface=0.0.0.0", "-Dapp.http.base-uri=http://delta:8080/v1", "-Dapp.cluster.remote-interface=delta", @@ -21,16 +22,23 @@ services: "-Dapp.database.cassandra.keyspace-autocreate=true", "-Dapp.database.cassandra.tables-autocreate=true", "-Dplugins.elasticsearch.base=http://elasticsearch:9200", + "-Dplugins.elasticsearch.indexing.max-time-window=50ms", "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", - "-Dplugins.composite-views.min-interval-rebuild=15seconds", + "-Dplugins.blazegraph.indexing.max-time-window=50ms", + "-Dplugins.composite-views.min-interval-rebuild=5seconds", + "-Dplugins.composite-views.indexing.max-time-window=50ms", "-Dplugins.storage.storages.remote-disk.enabled=true", "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", "-Dplugins.storage.storages.amazon.enabled=true", "-Dakka.persistence.cassandra.events-by-tag.first-time-bucket=20210503T00:00", - "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=4s", - "-Dakka.persistence.cassandra.query.refresh-interval=1s"] + "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=2s", + "-Dakka.persistence.cassandra.query.refresh-interval=500ms"] ports: - "8080" + deploy: + resources: + limits: + memory: 768M ######################################################### # Uncomment the following lines to run a local cluster # @@ -44,23 +52,30 @@ services: # KAMON_ENABLED: "false" # image: bluebrain/nexus-delta:latest # entrypoint: [ "bin/wait-for-it.sh", "-s", "-t", "0", "delta:8080", "--", "./bin/delta-app", -# "-Xmx2G", +# "-Xmx512m", # "-Dapp.http.interface=0.0.0.0", # "-Dapp.http.base-uri=http://delta:8080/v1", # "-Dapp.cluster.remote-interface=delta2", # "-Dapp.cluster.seeds=delta:25520", # "-Dapp.database.cassandra.contact-points.1=cassandra:9042", # "-Dplugins.elasticsearch.base=http://elasticsearch:9200", +# "-Dplugins.elasticsearch.indexing.max-time-window=50ms", # "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", -# "-Dplugins.composite-views.min-interval-rebuild=15seconds", +# "-Dplugins.blazegraph.indexing.max-time-window=50ms", +# "-Dplugins.composite-views.min-interval-rebuild=5seconds", +# "-Dplugins.composite-views.indexing.max-time-window=50ms", # "-Dplugins.storage.storages.remote-disk.enabled=true", # "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", # "-Dplugins.storage.storages.amazon.enabled=true", # "-Dakka.persistence.cassandra.events-by-tag.first-time-bucket=20210503T00:00", -# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=4s", -# "-Dakka.persistence.cassandra.query.refresh-interval=1s"] +# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=2s", +# "-Dakka.persistence.cassandra.query.refresh-interval=500ms"] # ports: # - "8080" +# deploy: +# resources: +# limits: +# memory: 768M # # delta3: # depends_on: @@ -70,37 +85,50 @@ services: # KAMON_ENABLED: "false" # image: bluebrain/nexus-delta:latest # entrypoint: [ "bin/wait-for-it.sh", "-s", "-t", "0", "delta2:8080", "--", "./bin/delta-app", -# "-Xmx2G", +# "-Xmx512m", # "-Dapp.http.interface=0.0.0.0", # "-Dapp.http.base-uri=http://delta:8080/v1", # "-Dapp.cluster.remote-interface=delta3", # "-Dapp.cluster.seeds=delta:25520", # "-Dapp.database.cassandra.contact-points.1=cassandra:9042", # "-Dplugins.elasticsearch.base=http://elasticsearch:9200", +# "-Dplugins.elasticsearch.indexing.max-time-window=50ms", # "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", -# "-Dplugins.composite-views.min-interval-rebuild=15seconds", +# "-Dplugins.blazegraph.indexing.max-time-window=50ms", +# "-Dplugins.composite-views.min-interval-rebuild=5seconds", +# "-Dplugins.composite-views.indexing.max-time-window=50ms", # "-Dplugins.storage.storages.remote-disk.enabled=true", # "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", # "-Dplugins.storage.storages.amazon.enabled=true", # "-Dakka.persistence.cassandra.events-by-tag.first-time-bucket=20210503T00:00", -# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=4s", -# "-Dakka.persistence.cassandra.query.refresh-interval=1s"] +# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=2s", +# "-Dakka.persistence.cassandra.query.refresh-interval=500ms"] # ports: # - "8080" +# deploy: +# resources: +# limits: +# memory: 768M keycloak: image: jboss/keycloak:11.0.1 environment: + JAVA_OPTS: "-Xms64m -Xmx256m" KEYCLOAK_USER: "admin" KEYCLOAK_PASSWORD: "admin" KEYCLOAK_FRONTEND_URL: "http://keycloak:8080/auth" + DB_VENDOR: H2 ports: - "8080" + deploy: + resources: + limits: + memory: 384M elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0 environment: - ES_JAVA_OPTS: "-Xmx1G" + ES_JAVA_OPTS: "-Xmx256m" discovery.type: "single-node" bootstrap.memory_lock: "true" healthcheck: @@ -110,30 +138,41 @@ services: retries: 3 ports: - "9200" + deploy: + resources: + limits: + memory: 384M blazegraph: image: bluebrain/blazegraph-nexus:2.1.5 environment: - JAVA_OPTS: "-Djava.awt.headless=true -XX:MaxDirectMemorySize=300m -Xms1g -Xmx1g -XX:+UseG1GC" + JAVA_OPTS: "-Djava.awt.headless=true -XX:MaxDirectMemorySize=64m -Xmx256m -XX:+UseG1GC" ports: - "9999" + deploy: + resources: + limits: + memory: 384M cassandra: image: cassandra:3.11.8 environment: - JVM_OPTS: "-Xms1g -Xmx1g -Dcassandra.initial_token=0 -Dcassandra.skip_wait_for_gossip_to_settle=0" - MAX_HEAP_SIZE: "1G" + JVM_OPTS: "-Xms300m -Xmx512m -Dcassandra.initial_token=0 -Dcassandra.skip_wait_for_gossip_to_settle=0" + MAX_HEAP_SIZE: "512m" HEAP_NEWSIZE: "100m" CASSANDRA_BROADCAST_ADDRESS: cassandra + CASSANDRA_NUM_TOKENS: 1 healthcheck: test: ["CMD", "cqlsh","-e describe keyspaces"] interval: 30s timeout: 20s retries: 3 + storage-service: image: bluebrain/nexus-storage:latest entrypoint: [ "./bin/storage", + "-Xmx192m", "-Dapp.instance.interface=0.0.0.0", "-Dapp.http.interface=0.0.0.0", "-Dapp.http.public-uri=http://storage.tests.nexus.ocp.bbp.epfl.ch", @@ -151,6 +190,10 @@ services: - "8080" volumes: - /tmp/storage:/data + deploy: + resources: + limits: + memory: 256M minio: image: minio/minio:RELEASE.2020-09-21T22-31-59Z diff --git a/tests/docker/docker-compose-ci.yml b/tests/docker/docker-compose-ci.yml index 155546f694..0d54fdc429 100644 --- a/tests/docker/docker-compose-ci.yml +++ b/tests/docker/docker-compose-ci.yml @@ -16,21 +16,23 @@ services: "-Xmx4G", "-Dapp.http.interface=0.0.0.0", "-Dapp.http.base-uri=http://delta:8080/v1", + "-Dapp.cluster.remote-interface=delta", + "-Dapp.cluster.seeds=delta:25520", "-Dapp.database.cassandra.contact-points.1=cassandra:9042", "-Dapp.database.cassandra.keyspace-autocreate=true", "-Dapp.database.cassandra.tables-autocreate=true", - "-Dapp.cluster.remote-interface=delta", - "-Dapp.cluster.seeds=delta:25520", -# "-Dakka.cluster.min-nr-of-members=3", "-Dplugins.elasticsearch.base=http://elasticsearch:9200", + "-Dplugins.elasticsearch.indexing.max-time-window=50ms", "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", - "-Dplugins.composite-views.min-interval-rebuild=15seconds", + "-Dplugins.blazegraph.indexing.max-time-window=50ms", + "-Dplugins.composite-views.min-interval-rebuild=5seconds", + "-Dplugins.composite-views.indexing.max-time-window=50ms", "-Dplugins.storage.storages.remote-disk.enabled=true", "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", "-Dplugins.storage.storages.amazon.enabled=true", "-Dakka.persistence.cassandra.events-by-tag.first-time-bucket=20210503T00:00", - "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=4s", - "-Dakka.persistence.cassandra.query.refresh-interval=1s"] + "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=2s", + "-Dakka.persistence.cassandra.query.refresh-interval=500ms"] ports: - "8080" @@ -50,14 +52,17 @@ services: # "-Dakka.cluster.min-nr-of-members=3", # "-Dapp.database.cassandra.contact-points.1=cassandra:9042", # "-Dplugins.elasticsearch.base=http://elasticsearch:9200", +# "-Dplugins.elasticsearch.indexing.max-time-window=50ms", # "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", -# "-Dplugins.composite-views.min-interval-rebuild=15seconds", +# "-Dplugins.blazegraph.indexing.max-time-window=50ms", +# "-Dplugins.composite-views.min-interval-rebuild=5seconds", +# "-Dplugins.composite-views.indexing.max-time-window=50ms", # "-Dplugins.storage.storages.remote-disk.enabled=true", # "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", # "-Dplugins.storage.storages.amazon.enabled=true", # "-Dakka.persistence.cassandra.events-by-tag.first-time-bucket=20210503T00:00", -# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=4s", -# "-Dakka.persistence.cassandra.query.refresh-interval=1s"] +# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=2s", +# "-Dakka.persistence.cassandra.query.refresh-interval=500ms"] # ports: # - "8080" # @@ -77,14 +82,17 @@ services: # "-Dakka.cluster.min-nr-of-members=3", # "-Dapp.database.cassandra.contact-points.1=cassandra:9042", # "-Dplugins.elasticsearch.base=http://elasticsearch:9200", +# "-Dplugins.elasticsearch.indexing.max-time-window=50ms", # "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", -# "-Dplugins.composite-views.min-interval-rebuild=15seconds", +# "-Dplugins.blazegraph.indexing.max-time-window=50ms", +# "-Dplugins.composite-views.min-interval-rebuild=5seconds", +# "-Dplugins.composite-views.indexing.max-time-window=50ms", # "-Dplugins.storage.storages.remote-disk.enabled=true", # "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", # "-Dplugins.storage.storages.amazon.enabled=true", # "-Dakka.persistence.cassandra.events-by-tag.first-time-bucket=20210503T00:00", -# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=4s", -# "-Dakka.persistence.cassandra.query.refresh-interval=1s"] +# "-Dakka.persistence.cassandra.events-by-tag.eventual-consistency-delay=2s", +# "-Dakka.persistence.cassandra.query.refresh-interval=500ms"] # ports: # - "8080" @@ -94,6 +102,7 @@ services: KEYCLOAK_USER: "admin" KEYCLOAK_PASSWORD: "admin" KEYCLOAK_FRONTEND_URL: "http://keycloak:8080/auth" + DB_VENDOR: H2 ports: - "8080" diff --git a/tests/docker/docker-compose-postgres.yml b/tests/docker/docker-compose-postgres.yml index 028573d867..e5c1fe66af 100644 --- a/tests/docker/docker-compose-postgres.yml +++ b/tests/docker/docker-compose-postgres.yml @@ -12,7 +12,7 @@ services: KAMON_ENABLED: "false" image: bluebrain/nexus-delta:latest entrypoint: [ "bin/wait-for-it.sh", "-s", "-t", "0", "postgres:5432", "--", "./bin/delta-app", - "-Xmx2G", + "-Xmx512m", "-Dapp.http.interface=0.0.0.0", "-Dapp.http.base-uri=http://delta:8080/v1", "-Dapp.cluster.remote-interface=delta", @@ -21,14 +21,21 @@ services: "-Dapp.database.postgres.host=postgres", "-Dapp.database.postgres.tables-autocreate=true", "-Dplugins.elasticsearch.base=http://elasticsearch:9200", + "-Dplugins.elasticsearch.indexing.max-time-window=50ms", "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", - "-Dplugins.composite-views.min-interval-rebuild=15seconds", + "-Dplugins.blazegraph.indexing.max-time-window=50ms", + "-Dplugins.composite-views.min-interval-rebuild=5seconds", + "-Dplugins.composite-views.indexing.max-time-window=50ms", "-Dplugins.storage.storages.remote-disk.enabled=true", "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", "-Dplugins.storage.storages.amazon.enabled=true", - "-Djdbc-read-journal.refresh-interval=1s"] + "-Djdbc-read-journal.refresh-interval=500ms"] ports: - "8080" + deploy: + resources: + limits: + memory: 768M ######################################################### # Uncomment the following lines to run a local cluster # @@ -42,7 +49,7 @@ services: # KAMON_ENABLED: "false" # image: bluebrain/nexus-delta:latest # entrypoint: [ "bin/wait-for-it.sh", "-s", "-t", "0", "delta:8080", "--", "./bin/delta-app", - # "-Xmx2G", + # "-Xmx512m", # "-Dapp.http.interface=0.0.0.0", # "-Dapp.http.base-uri=http://delta:8080/v1", # "-Dapp.cluster.remote-interface=delta2", @@ -50,14 +57,21 @@ services: # "-Dapp.database.flavour=postgres", # "-Dapp.database.postgres.host=postgres", # "-Dplugins.elasticsearch.base=http://elasticsearch:9200", + # "-Dplugins.elasticsearch.indexing.max-time-window=50ms", # "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", - # "-Dplugins.composite-views.min-interval-rebuild=15seconds", + # "-Dplugins.blazegraph.indexing.max-time-window=50ms", + # "-Dplugins.composite-views.min-interval-rebuild=5seconds", + # "-Dplugins.composite-views.indexing.max-time-window=50ms", # "-Dplugins.storage.storages.remote-disk.enabled=true", # "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", # "-Dplugins.storage.storages.amazon.enabled=true", - # "-Djdbc-read-journal.refresh-interval=1s"] + # "-Djdbc-read-journal.refresh-interval=500ms"] # ports: # - "8080" + # deploy: + # resources: + # limits: + # memory: 768M # # delta3: # depends_on: @@ -67,7 +81,7 @@ services: # KAMON_ENABLED: "false" # image: bluebrain/nexus-delta:latest # entrypoint: [ "bin/wait-for-it.sh", "-s", "-t", "0", "delta2:8080", "--", "./bin/delta-app", - # "-Xmx2G", + # "-Xmx512m", # "-Dapp.http.interface=0.0.0.0", # "-Dapp.http.base-uri=http://delta:8080/v1", # "-Dapp.cluster.remote-interface=delta3", @@ -75,29 +89,41 @@ services: # "-Dapp.database.flavour=postgres", # "-Dapp.database.postgres.host=postgres", # "-Dplugins.elasticsearch.base=http://elasticsearch:9200", + # "-Dplugins.elasticsearch.indexing.max-time-window=50ms", # "-Dplugins.blazegraph.base=http://blazegraph:9999/blazegraph", - # "-Dplugins.composite-views.min-interval-rebuild=15seconds", + # "-Dplugins.blazegraph.indexing.max-time-window=50ms", + # "-Dplugins.composite-views.min-interval-rebuild=5seconds", + # "-Dplugins.composite-views.indexing.max-time-window=50ms", # "-Dplugins.storage.storages.remote-disk.enabled=true", # "-Dplugins.storage.storages.remote-disk.default-endpoint=http://storage-service:8080/v1", # "-Dplugins.storage.storages.amazon.enabled=true", - # "-Djdbc-read-journal.refresh-interval=1s"] + # "-Djdbc-read-journal.refresh-interval=500ms" # ports: # - "8080" + # deploy: + # resources: + # limits: + # memory: 768M keycloak: image: jboss/keycloak:11.0.1 environment: + JAVA_OPTS: "-Xms64m -Xmx256m" KEYCLOAK_USER: "admin" KEYCLOAK_PASSWORD: "admin" KEYCLOAK_FRONTEND_URL: "http://keycloak:8080/auth" DB_VENDOR: H2 ports: - "8080" + deploy: + resources: + limits: + memory: 384M elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0 environment: - ES_JAVA_OPTS: "-Xmx1G" + ES_JAVA_OPTS: "-Xmx256m" discovery.type: "single-node" bootstrap.memory_lock: "true" healthcheck: @@ -107,13 +133,21 @@ services: retries: 3 ports: - "9200" + deploy: + resources: + limits: + memory: 384M blazegraph: image: bluebrain/blazegraph-nexus:2.1.5 environment: - JAVA_OPTS: "-Djava.awt.headless=true -XX:MaxDirectMemorySize=300m -Xms1g -Xmx1g -XX:+UseG1GC" + JAVA_OPTS: "-Djava.awt.headless=true -XX:MaxDirectMemorySize=64m -Xmx256m -XX:+UseG1GC" ports: - "9999" + deploy: + resources: + limits: + memory: 384M postgres: image: library/postgres:12.2 @@ -124,6 +158,7 @@ services: storage-service: image: bluebrain/nexus-storage:latest entrypoint: [ "./bin/storage", + "-Xmx192m", "-Dapp.instance.interface=0.0.0.0", "-Dapp.http.interface=0.0.0.0", "-Dapp.http.public-uri=http://storage.tests.nexus.ocp.bbp.epfl.ch", @@ -141,6 +176,10 @@ services: - "8080" volumes: - /tmp/storage:/data + deploy: + resources: + limits: + memory: 256M minio: image: minio/minio:RELEASE.2020-09-21T22-31-59Z diff --git a/tests/src/test/resources/application.conf b/tests/src/test/resources/application.conf index 16147f0255..27ecbd9656 100644 --- a/tests/src/test/resources/application.conf +++ b/tests/src/test/resources/application.conf @@ -2,6 +2,7 @@ tests { delta-uri = "http://delta:8080/v1" realm-uri = "http://keycloak:8080/auth/realms" patience = 20 seconds + clean-up = true } storage { diff --git a/tests/src/test/resources/elasticsearch/template.json b/tests/src/test/resources/elasticsearch/template.json index 85de3c8a88..c67cbb57eb 100644 --- a/tests/src/test/resources/elasticsearch/template.json +++ b/tests/src/test/resources/elasticsearch/template.json @@ -5,7 +5,7 @@ "settings" : { "number_of_shards": 1, "number_of_replicas": 0, - "refresh_interval": "1s" + "refresh_interval": "200ms" } } } \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala index bfb29a5acf..14131e9b93 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala @@ -7,9 +7,9 @@ import akka.http.scaladsl.model.headers._ import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.util.ByteString import cats.implicits._ -import ch.epfl.bluebrain.nexus.testkit.{CirceEq, IOValues, TestHelpers} +import ch.epfl.bluebrain.nexus.testkit.{CirceEq, IORef, IOValues, TestHelpers} import ch.epfl.bluebrain.nexus.tests.HttpClient._ -import ch.epfl.bluebrain.nexus.tests.Identity._ +import ch.epfl.bluebrain.nexus.tests.Identity.{allUsers, testClient, testRealm, _} import ch.epfl.bluebrain.nexus.tests.admin.AdminDsl import ch.epfl.bluebrain.nexus.tests.config.ConfigLoader._ import ch.epfl.bluebrain.nexus.tests.config.TestsConfig @@ -25,6 +25,7 @@ import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AsyncWordSpecLike import org.scalatest.{Assertion, BeforeAndAfterAll, OptionValues} +import BaseSpec._ import scala.concurrent.duration._ @@ -84,11 +85,28 @@ trait BaseSpec Identity.ServiceAccount, Permission.minimalPermissions ) - _ <- aclDsl.cleanAclsAnonymous + _ <- initRealm( + testRealm, + Identity.ServiceAccount, + testClient, + allUsers + ) } yield () - setup.runSyncUnsafe() + + val allTasks = for { + isSetupCompleted <- setupCompleted.get + _ <- Task.unless(isSetupCompleted)(setup) + _ <- setupCompleted.set(true) + _ <- aclDsl.cleanAclsAnonymous + } yield () + + allTasks.runSyncUnsafe() + } + override def afterAll(): Unit = + Task.when(config.cleanUp)(elasticsearchDsl.deleteAllIndices().void).runSyncUnsafe() + private def toAuthorizationHeader(token: String) = Authorization( HttpCredentials.createOAuth2BearerToken(token) @@ -161,7 +179,7 @@ trait BaseSpec // Create the realm in Keycloak _ <- keycloakDsl.importRealm(realm, client, users) // Get the tokens and cache them in the map - _ <- users.traverse { user => + _ <- users.parTraverse { user => authenticateUser(user, client) } _ <- authenticateClient(client) @@ -202,3 +220,9 @@ trait BaseSpec genString(length = length, Vector.range('a', 'z') ++ Vector.range('0', '9')) } + +object BaseSpec { + + val setupCompleted: IORef[Boolean] = IORef.unsafe(false) + +} diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/ElasticsearchDsl.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/ElasticsearchDsl.scala index d7e23e5284..eae357b325 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/ElasticsearchDsl.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/ElasticsearchDsl.scala @@ -1,7 +1,7 @@ package ch.epfl.bluebrain.nexus.tests import akka.actor.ActorSystem -import akka.http.scaladsl.model.HttpMethods.PUT +import akka.http.scaladsl.model.HttpMethods.{DELETE, PUT} import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest, StatusCode} import akka.stream.Materializer import ch.epfl.bluebrain.nexus.testkit.TestHelpers @@ -46,4 +46,17 @@ class ElasticsearchDsl(implicit as: ActorSystem, materializer: Materializer) ext } } + def deleteAllIndices(): Task[StatusCode] = + elasticClient( + HttpRequest( + method = DELETE, + uri = s"$elasticUrl/delta_*" + ) + ).tapError { t => + Task { logger.error(s"Error while deleting elasticsearch indices", t) } + }.map { res => + logger.info(s"Deleting elasticsearch indices returned ${res.status}") + res.status + } + } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index c759988b7e..e6fac768b9 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -1,18 +1,14 @@ package ch.epfl.bluebrain.nexus.tests -import java.nio.file.{Files, Path} -import java.util.UUID -import java.util.concurrent.ConcurrentHashMap - import akka.actor.ActorSystem -import akka.http.scaladsl.{Http, HttpExt} import akka.http.scaladsl.model.HttpCharsets._ import akka.http.scaladsl.model.HttpMethods._ import akka.http.scaladsl.model.Multipart.FormData import akka.http.scaladsl.model.Multipart.FormData.BodyPart -import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers.{`Accept-Encoding`, Accept, Authorization, HttpEncodings} +import akka.http.scaladsl.model.{HttpResponse, _} import akka.http.scaladsl.unmarshalling.FromEntityUnmarshaller +import akka.http.scaladsl.{Http, HttpExt} import akka.stream.Materializer import akka.stream.alpakka.sse.scaladsl.EventSource import akka.stream.scaladsl.Sink @@ -24,14 +20,18 @@ import io.circe.parser._ import fs2._ import monix.bio.Task import monix.execution.Scheduler.Implicits.global -import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers +import org.scalatest.{AppendedClues, Assertion} +import java.nio.file.{Files, Path} +import java.util.concurrent.ConcurrentHashMap import scala.collection.immutable.Seq import scala.concurrent.Future import scala.concurrent.duration._ -class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit materializer: Materializer) extends Matchers { +class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit materializer: Materializer) + extends Matchers + with AppendedClues { def apply(req: HttpRequest): Task[HttpResponse] = Task.deferFuture(httpExt.singleRequest(req)) @@ -127,6 +127,16 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit materializer: identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders )(assertResponse: (A, HttpResponse) => Assertion)(implicit um: FromEntityUnmarshaller[A]): Task[Assertion] = { + def buildClue(a: A, response: HttpResponse) = + s""" + |Endpoint: ${method.value} $url + |Identity: $identity + |Token: ${Option(tokensMap.get(identity)).map(_.credentials.token()).getOrElse("None")} + |Status code: ${response.status} + |Response: + |$a + |""".stripMargin + def onFail(e: Throwable) = fail( s"Something went wrong while processing the response for url: ${method.value} $url with identity $identity", @@ -137,7 +147,7 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit materializer: url, body, identity, - assertResponse, + (a: A, response: HttpResponse) => assertResponse(a, response) withClue buildClue(a, response), onFail, extraHeaders ) @@ -249,7 +259,7 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit materializer: def sseEvents( url: String, identity: Identity, - initialLastEventId: UUID, + initialLastEventId: Option[String], take: Long = 100L, takeWithin: FiniteDuration = 30.seconds )(assertResponse: Seq[(Option[String], Option[Json])] => Assertion): Task[Assertion] = { @@ -257,7 +267,7 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit materializer: apply(request.addHeader(tokensMap.get(identity))).runToFuture Task .deferFuture { - EventSource(s"$baseUrl$url", send, initialLastEventId = Some(initialLastEventId.toString)) + EventSource(s"$baseUrl$url", send, initialLastEventId = initialLastEventId) //drop resolver, views and storage events .take(take) .takeWithin(takeWithin) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala index 67949715cf..7a9429c5a8 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala @@ -1,8 +1,10 @@ package ch.epfl.bluebrain.nexus.tests +import ch.epfl.bluebrain.nexus.testkit.TestHelpers + sealed trait Identity extends Product with Serializable -object Identity { +object Identity extends TestHelpers { case object Anonymous extends Identity @@ -28,4 +30,48 @@ object Identity { val Delta: UserCredentials = UserCredentials("delta", "shhh", internal) + val testRealm = Realm("test-" + genString()) + val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) + + object acls { + val Marge = UserCredentials(genString(), genString(), testRealm) + } + + object archives { + val Tweety = UserCredentials(genString(), genString(), testRealm) + } + + object compositeviews { + val Jerry = UserCredentials(genString(), genString(), testRealm) + } + + object events { + val BugsBunny = UserCredentials(genString(), genString(), testRealm) + } + + object orgs { + val Fry = UserCredentials(genString(), genString(), testRealm) + val Leela = UserCredentials(genString(), genString(), testRealm) + } + + object projects { + val Bojack = UserCredentials(genString(), genString(), testRealm) + val PrincessCarolyn = UserCredentials(genString(), genString(), testRealm) + } + + object resources { + val Rick = UserCredentials(genString(), genString(), testRealm) + } + + object storages { + val Coyote = UserCredentials(genString(), genString(), testRealm) + } + + object views { + val ScoobyDoo = UserCredentials(genString(), genString(), testRealm) + } + + lazy val allUsers = + acls.Marge :: archives.Tweety :: compositeviews.Jerry :: events.BugsBunny :: orgs.Fry :: orgs.Leela :: projects.Bojack :: projects.PrincessCarolyn :: resources.Rick :: storages.Coyote :: views.ScoobyDoo :: Nil + } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/OrgsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/OrgsSpec.scala index d1f997486f..8cb6ecdd3e 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/OrgsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/OrgsSpec.scala @@ -2,32 +2,17 @@ package ch.epfl.bluebrain.nexus.tests.admin import akka.http.scaladsl.model.StatusCodes import ch.epfl.bluebrain.nexus.testkit.EitherValuable -import ch.epfl.bluebrain.nexus.tests.Identity.UserCredentials +import ch.epfl.bluebrain.nexus.tests.Identity.orgs.{Fry, Leela} import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.OrgsTag -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, ExpectedResponse, Identity, Realm} +import ch.epfl.bluebrain.nexus.tests.{BaseSpec, ExpectedResponse} import io.circe.Json import monix.execution.Scheduler.Implicits.global class OrgsSpec extends BaseSpec with EitherValuable { - private val testRealm = Realm("orgs" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val Fry = UserCredentials(genString(), genString(), testRealm) - private val Leela = UserCredentials(genString(), genString(), testRealm) - import ch.epfl.bluebrain.nexus.tests.iam.types.Permission._ - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Fry :: Leela :: Nil - ).runSyncUnsafe() - } - private val UnauthorizedAccess = ExpectedResponse( StatusCodes.Forbidden, jsonContentOf("/iam/errors/unauthorized-access.json") diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/ProjectsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/ProjectsSpec.scala index c09e493243..5b3322e6f0 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/ProjectsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/admin/ProjectsSpec.scala @@ -2,20 +2,16 @@ package ch.epfl.bluebrain.nexus.tests.admin import akka.http.scaladsl.model.StatusCodes import cats.implicits._ -import ch.epfl.bluebrain.nexus.tests.Identity.{Authenticated, UserCredentials} +import ch.epfl.bluebrain.nexus.tests.Identity.Authenticated +import ch.epfl.bluebrain.nexus.tests.Identity.projects.{Bojack, PrincessCarolyn} import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.ProjectsTag -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, ExpectedResponse, Identity, Realm} +import ch.epfl.bluebrain.nexus.tests.{BaseSpec, ExpectedResponse, Identity} import io.circe.Json import monix.execution.Scheduler.Implicits.global class ProjectsSpec extends BaseSpec { - private val testRealm = Realm("projects" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val Bojack = UserCredentials(genString(), genString(), testRealm) - private val PrincessCarolyn = UserCredentials(genString(), genString(), testRealm) - import ch.epfl.bluebrain.nexus.tests.iam.types.Permission._ private val UnauthorizedAccess = ExpectedResponse( @@ -33,16 +29,6 @@ class ProjectsSpec extends BaseSpec { jsonContentOf("/admin/errors/project-incorrect-revision.json") ) - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Bojack :: PrincessCarolyn :: Nil - ).runSyncUnsafe() - } - "projects API" should { val orgId = genId() @@ -335,7 +321,7 @@ class ProjectsSpec extends BaseSpec { "Description", Bojack ) - _ <- projectIds.traverse { case (orgId, projId) => + _ <- projectIds.parTraverse { case (orgId, projId) => adminDsl.createProject( orgId, projId, @@ -382,7 +368,7 @@ class ProjectsSpec extends BaseSpec { ) for { - _ <- projectsToList.traverse { case (orgId, projectId) => + _ <- projectsToList.parTraverse { case (orgId, projectId) => aclDsl.addPermission( s"/$orgId/$projectId", PrincessCarolyn, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/config/TestsConfig.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/config/TestsConfig.scala index 44f5eca219..bf886dcafe 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/config/TestsConfig.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/config/TestsConfig.scala @@ -5,7 +5,7 @@ import ch.epfl.bluebrain.nexus.tests.Realm import scala.concurrent.duration.FiniteDuration -case class TestsConfig(deltaUri: Uri, realmUri: Uri, patience: FiniteDuration) { +case class TestsConfig(deltaUri: Uri, realmUri: Uri, patience: FiniteDuration, cleanUp: Boolean) { def realmSuffix(realm: Realm) = s"$realmUri/${realm.name}" } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclDsl.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclDsl.scala index 2162dfebed..23484d0f15 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclDsl.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclDsl.scala @@ -93,7 +93,7 @@ class AclDsl(cl: HttpClient) extends TestHelpers with CirceUnmarshalling with Op .filter(_.acl.nonEmpty) permissions - .traverse { acl => + .parTraverse { acl => val payload = jsonContentOf( "/iam/subtract-permissions.json", "realm" -> target.realm.name, @@ -123,7 +123,7 @@ class AclDsl(cl: HttpClient) extends TestHelpers with CirceUnmarshalling with Op .filter(_.acl.nonEmpty) permissions - .traverse { acl => + .parTraverse { acl => val payload = jsonContentOf( "/iam/subtract-permissions-anon.json", "perms" -> acl.acl.head.permissions.map(_.value).mkString("""","""") diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclsSpec.scala index 5a4b3c631a..71e64409d9 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/iam/AclsSpec.scala @@ -2,28 +2,15 @@ package ch.epfl.bluebrain.nexus.tests.iam import akka.http.scaladsl.model.StatusCodes import cats.implicits._ -import ch.epfl.bluebrain.nexus.tests.Identity.UserCredentials +import ch.epfl.bluebrain.nexus.tests.Identity.acls.Marge +import ch.epfl.bluebrain.nexus.tests.Identity.testRealm import ch.epfl.bluebrain.nexus.tests.Tags.AclsTag import ch.epfl.bluebrain.nexus.tests.iam.types.{AclEntry, AclListing, Permission, User} -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} +import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity} import monix.execution.Scheduler.Implicits.global class AclsSpec extends BaseSpec { - private val testRealm = Realm("acls" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val Marge = UserCredentials(genString(), genString(), testRealm) - - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Marge :: Nil - ).runSyncUnsafe() - } - "manage acls" should { val orgPath1 = genString() val orgPath2 = genString() @@ -81,7 +68,7 @@ class AclsSpec extends BaseSpec { } "add permissions for user on paths with depth1" taggedAs AclsTag in { - orgs.traverse { org => + orgs.parTraverse { org => aclDsl.addPermissions( s"/$org", Marge, @@ -91,7 +78,7 @@ class AclsSpec extends BaseSpec { } "add permissions for user on /orgpath/projectpath1 and /orgpath/projectpath2" taggedAs AclsTag in { - crossProduct.traverse { case (org, project) => + crossProduct.parTraverse { case (org, project) => aclDsl.addPermissions( s"/$org/$project", Marge, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ArchiveSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ArchiveSpec.scala index 9782087f67..4bf23644b0 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ArchiveSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ArchiveSpec.scala @@ -6,11 +6,12 @@ import akka.http.scaladsl.unmarshalling.PredefinedFromEntityUnmarshallers import akka.util.ByteString import ch.epfl.bluebrain.nexus.testkit.{CirceEq, EitherValuable} import ch.epfl.bluebrain.nexus.tests.HttpClient._ -import ch.epfl.bluebrain.nexus.tests.Identity.UserCredentials +import ch.epfl.bluebrain.nexus.tests.Identity.archives.Tweety +import ch.epfl.bluebrain.nexus.tests.Identity.testRealm import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.ArchivesTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Projects, Resources} -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} +import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity} import io.circe.Json import io.circe.parser._ import monix.execution.Scheduler.Implicits.global @@ -23,10 +24,6 @@ import scala.annotation.tailrec class ArchiveSpec extends BaseSpec with CirceEq with EitherValuable { - private val testRealm = Realm("resources" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val Tweety = UserCredentials(genString(), genString(), testRealm) - private val orgId = genId() private val projId = genId() private val projId2 = genId() @@ -76,16 +73,6 @@ class ArchiveSpec extends BaseSpec with CirceEq with EitherValuable { private type PathAndContent = (Path, ByteString) - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Tweety :: Nil - ).runSyncUnsafe() - } - @tailrec private def readEntries(tar: TarArchiveInputStream, entries: List[PathAndContent] = Nil): List[PathAndContent] = { val entry = tar.getNextTarEntry diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala index 0d9069df52..51d439a6ca 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala @@ -2,12 +2,12 @@ package ch.epfl.bluebrain.nexus.tests.kg import akka.http.scaladsl.model.StatusCodes import cats.implicits._ +import ch.epfl.bluebrain.nexus.tests.BaseSpec import ch.epfl.bluebrain.nexus.tests.HttpClient._ -import ch.epfl.bluebrain.nexus.tests.Identity.UserCredentials +import ch.epfl.bluebrain.nexus.tests.Identity.compositeviews.Jerry import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.CompositeViewsTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Events, Organizations, Views} -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} import com.typesafe.scalalogging.Logger import io.circe.Json import io.circe.optics.JsonPath._ @@ -32,20 +32,6 @@ class CompositeViewsSpec extends BaseSpec { private val albumsProject = "albums" private val songsProject = "songs" - private val testRealm = Realm("composite" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val Jerry = UserCredentials(genString(), genString(), testRealm) - - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Jerry :: Nil - ).runSyncUnsafe() - } - "Creating projects" should { "add necessary permissions for user" taggedAs CompositeViewsTag in { aclDsl.addPermissions( @@ -59,9 +45,13 @@ class CompositeViewsSpec extends BaseSpec { val projectPayload = jsonContentOf("/kg/views/composite/project.json") for { _ <- adminDsl.createOrganization(orgId, orgId, Jerry) - _ <- adminDsl.createProject(orgId, bandsProject, projectPayload, Jerry) - _ <- adminDsl.createProject(orgId, albumsProject, projectPayload, Jerry) - _ <- adminDsl.createProject(orgId, songsProject, projectPayload, Jerry) + _ <- Task.parSequence( + List( + adminDsl.createProject(orgId, bandsProject, projectPayload, Jerry), + adminDsl.createProject(orgId, albumsProject, projectPayload, Jerry), + adminDsl.createProject(orgId, songsProject, projectPayload, Jerry) + ) + ) } yield succeed } @@ -90,7 +80,7 @@ class CompositeViewsSpec extends BaseSpec { "Uploading data" should { "upload context" taggedAs CompositeViewsTag in { val context = jsonContentOf("/kg/views/composite/context.json") - List(songsProject, albumsProject, bandsProject).traverse { projectId => + List(songsProject, albumsProject, bandsProject).parTraverse { projectId => deltaClient.post[Json](s"/resources/$orgId/$projectId", context, Jerry) { (_, response) => response.status shouldEqual StatusCodes.Created } @@ -102,7 +92,7 @@ class CompositeViewsSpec extends BaseSpec { .getAll( jsonContentOf("/kg/views/composite/songs1.json") ) - .traverse { song => + .parTraverse { song => deltaClient.post[Json](s"/resources/$orgId/$songsProject", song, Jerry) { (_, response) => response.status shouldEqual StatusCodes.Created } @@ -114,7 +104,7 @@ class CompositeViewsSpec extends BaseSpec { .getAll( jsonContentOf("/kg/views/composite/albums.json") ) - .traverse { album => + .parTraverse { album => deltaClient.post[Json](s"/resources/$orgId/$albumsProject", album, Jerry) { (_, response) => response.status shouldEqual StatusCodes.Created } @@ -126,7 +116,7 @@ class CompositeViewsSpec extends BaseSpec { .getAll( jsonContentOf("/kg/views/composite/bands.json") ) - .traverse { band => + .parTraverse { band => deltaClient.post[Json](s"/resources/$orgId/$bandsProject", band, Jerry) { (_, response) => response.status shouldEqual StatusCodes.Created } @@ -268,7 +258,7 @@ class CompositeViewsSpec extends BaseSpec { .getAll( jsonContentOf("/kg/views/composite/songs2.json") ) - .traverse { song => + .parTraverse { song => deltaClient.post[Json](s"/resources/$orgId/$songsProject", song, Jerry) { (_, response) => response.status shouldEqual StatusCodes.Created } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index 5dd64ebc1e..f1f94fb251 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -1,6 +1,7 @@ package ch.epfl.bluebrain.nexus.tests.kg import akka.http.scaladsl.model.StatusCodes +import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote import ch.epfl.bluebrain.nexus.tests.Optics.filterMetadataKeys import ch.epfl.bluebrain.nexus.tests.Tags.StorageTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala index 60ff98f218..22979004cf 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala @@ -1,44 +1,36 @@ package ch.epfl.bluebrain.nexus.tests.kg import akka.http.scaladsl.model.{ContentTypes, StatusCodes} -import ch.epfl.bluebrain.nexus.tests.Identity.UserCredentials +import ch.epfl.bluebrain.nexus.tests.Identity.events.BugsBunny import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.EventsTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Events, Organizations, Resources} -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} +import ch.epfl.bluebrain.nexus.tests.kg.VersionSpec.VersionBundle +import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity} import com.fasterxml.uuid.Generators -import com.typesafe.scalalogging.Logger import io.circe.Json +import monix.bio.Task import monix.execution.Scheduler.Implicits.global import org.scalatest.Inspectors class EventsSpec extends BaseSpec with Inspectors { - private val logger = Logger[this.type] - - private val orgId = genId() - private val orgId2 = genId() - private val projId = genId() - private val id = s"$orgId/$projId" - private val id2 = s"$orgId2/$projId" - private lazy val timestampUuid = Generators.timeBasedGenerator().generate() - - private[tests] val testRealm = Realm("events" + genString()) - private[tests] val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private[tests] val BugsBunny = UserCredentials(genString(), genString(), testRealm) + private val orgId = genId() + private val orgId2 = genId() + private val projId = genId() + private val id = s"$orgId/$projId" + private val id2 = s"$orgId2/$projId" + private var initialEventId: Option[String] = None override def beforeAll(): Unit = { super.beforeAll() - logger.info(s"TimestampUuid: $timestampUuid") - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - BugsBunny :: Nil - ).runSyncUnsafe() - // force timestampUuid to be evaluated - val _ = timestampUuid.toString - () + deltaClient + .get[VersionBundle]("/version", Identity.ServiceAccount) { (version, _) => + initialEventId = version.dependencies.cassandra.map(_ => Generators.timeBasedGenerator().generate().toString) + succeed + } + .void + .runSyncUnsafe() } "creating projects" should { @@ -175,7 +167,7 @@ class EventsSpec extends BaseSpec with Inspectors { "fetch resource events filtered by project" taggedAs EventsTag in { for { uuids <- adminDsl.getUuids(orgId, projId, BugsBunny) - _ <- deltaClient.sseEvents(s"/resources/$id/events", BugsBunny, timestampUuid, take = 11L) { seq => + _ <- deltaClient.sseEvents(s"/resources/$id/events", BugsBunny, initialEventId, take = 11L) { seq => val projectEvents = seq.drop(5) projectEvents.size shouldEqual 6 projectEvents.flatMap(_._1) should contain theSameElementsInOrderAs List( @@ -205,7 +197,7 @@ class EventsSpec extends BaseSpec with Inspectors { "fetch resource events filtered by organization 1" taggedAs EventsTag in { for { uuids <- adminDsl.getUuids(orgId, projId, BugsBunny) - _ <- deltaClient.sseEvents(s"/resources/$orgId/events", BugsBunny, timestampUuid, take = 12L) { seq => + _ <- deltaClient.sseEvents(s"/resources/$orgId/events", BugsBunny, initialEventId, take = 12L) { seq => val projectEvents = seq.drop(6) projectEvents.size shouldEqual 6 projectEvents.flatMap(_._1) should contain theSameElementsInOrderAs List( @@ -236,7 +228,7 @@ class EventsSpec extends BaseSpec with Inspectors { for { uuids <- adminDsl.getUuids(orgId2, projId, BugsBunny) _ <- - deltaClient.sseEvents(s"/resources/$orgId2/events", BugsBunny, timestampUuid, take = 7L) { seq => + deltaClient.sseEvents(s"/resources/$orgId2/events", BugsBunny, initialEventId, take = 7L) { seq => val projectEvents = seq.drop(6) projectEvents.size shouldEqual 1 projectEvents.flatMap(_._1) should contain theSameElementsInOrderAs List("ResourceCreated") @@ -257,39 +249,44 @@ class EventsSpec extends BaseSpec with Inspectors { } "fetch global events" taggedAs EventsTag in { - for { - uuids <- adminDsl.getUuids(orgId, projId, BugsBunny) - uuids2 <- adminDsl.getUuids(orgId2, projId, BugsBunny) - _ <- deltaClient.sseEvents(s"/resources/events", BugsBunny, timestampUuid, take = 19) { seq => - val projectEvents = seq.drop(12) - projectEvents.size shouldEqual 7 - projectEvents.flatMap(_._1) should contain theSameElementsInOrderAs List( - "ResourceCreated", - "ResourceCreated", - "ResourceUpdated", - "ResourceTagAdded", - "ResourceDeprecated", - "FileCreated", - "FileUpdated" - ) - val json = Json.arr(projectEvents.flatMap(_._2.map(events.filterFields)): _*) - json shouldEqual jsonContentOf( - "/kg/events/events-multi-project.json", - replacements( - BugsBunny, - "resources" -> s"${config.deltaUri}/resources/$id", - "organizationUuid" -> uuids._1, - "projectUuid" -> uuids._2, - "organization2Uuid" -> uuids2._1, - "project2Uuid" -> uuids2._2, - "project" -> s"${config.deltaUri}/projects/$orgId/$projId", - "project2" -> s"${config.deltaUri}/projects/$orgId2/$projId", - "schemaProject" -> s"${config.deltaUri}/projects/$orgId/$projId", - "schemaProject2" -> s"${config.deltaUri}/projects/$orgId2/$projId" - ): _* - ) - } - } yield succeed + // Only for cassandra, it is difficult to get the current sequence value with PostgreSQL + Task + .when(initialEventId.isDefined) { + for { + uuids <- adminDsl.getUuids(orgId, projId, BugsBunny) + uuids2 <- adminDsl.getUuids(orgId2, projId, BugsBunny) + _ <- deltaClient.sseEvents(s"/resources/events", BugsBunny, initialEventId, take = 19) { seq => + val projectEvents = seq.drop(12) + projectEvents.size shouldEqual 7 + projectEvents.flatMap(_._1) should contain theSameElementsInOrderAs List( + "ResourceCreated", + "ResourceCreated", + "ResourceUpdated", + "ResourceTagAdded", + "ResourceDeprecated", + "FileCreated", + "FileUpdated" + ) + val json = Json.arr(projectEvents.flatMap(_._2.map(events.filterFields)): _*) + json shouldEqual jsonContentOf( + "/kg/events/events-multi-project.json", + replacements( + BugsBunny, + "resources" -> s"${config.deltaUri}/resources/$id", + "organizationUuid" -> uuids._1, + "projectUuid" -> uuids._2, + "organization2Uuid" -> uuids2._1, + "project2Uuid" -> uuids2._2, + "project" -> s"${config.deltaUri}/projects/$orgId/$projId", + "project2" -> s"${config.deltaUri}/projects/$orgId2/$projId", + "schemaProject" -> s"${config.deltaUri}/projects/$orgId/$projId", + "schemaProject2" -> s"${config.deltaUri}/projects/$orgId2/$projId" + ): _* + ) + } + } yield () + } + .as(succeed) } } } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index a445c01651..fea818ff3a 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -1,11 +1,9 @@ package ch.epfl.bluebrain.nexus.tests.kg -import java.io.File -import java.nio.file.{Files, Paths} - import akka.http.scaladsl.model.StatusCodes import ch.epfl.bluebrain.nexus.tests.HttpClient._ import ch.epfl.bluebrain.nexus.tests.Identity +import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote import ch.epfl.bluebrain.nexus.tests.Optics.{filterKey, filterMetadataKeys} import ch.epfl.bluebrain.nexus.tests.Tags.StorageTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission @@ -13,6 +11,8 @@ import io.circe.Json import monix.bio.Task import org.scalatest.Assertion +import java.io.File +import java.nio.file.{Files, Paths} import scala.reflect.io.Directory class RemoteStorageSpec extends StorageSpec { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ResourcesSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ResourcesSpec.scala index 317324b2ef..63a16ca806 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ResourcesSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ResourcesSpec.scala @@ -3,11 +3,12 @@ package ch.epfl.bluebrain.nexus.tests.kg import akka.http.scaladsl.model.StatusCodes import cats.implicits._ import ch.epfl.bluebrain.nexus.testkit.{CirceEq, EitherValuable} -import ch.epfl.bluebrain.nexus.tests.Identity.{Delta, UserCredentials} +import ch.epfl.bluebrain.nexus.tests.BaseSpec +import ch.epfl.bluebrain.nexus.tests.Identity.Delta +import ch.epfl.bluebrain.nexus.tests.Identity.resources.Rick import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.ResourcesTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.Organizations -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} import io.circe.Json import monix.bio.Task import monix.execution.Scheduler.Implicits.global @@ -16,26 +17,12 @@ import java.net.URLEncoder class ResourcesSpec extends BaseSpec with EitherValuable with CirceEq { - private val testRealm = Realm("resources" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val Rick = UserCredentials(genString(), genString(), testRealm) - private val orgId = genId() private val projId1 = genId() private val projId2 = genId() private val id1 = s"$orgId/$projId1" private val id2 = s"$orgId/$projId2" - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Rick :: Nil - ).runSyncUnsafe() - } - "creating projects" should { "add necessary permissions for user" taggedAs ResourcesTag in { @@ -300,7 +287,7 @@ class ResourcesSpec extends BaseSpec with EitherValuable with CirceEq { List( s"/resources/$id1/test-schema/test-resource:1", s"/resources/$id1/_/test-resource:1" - ).traverse { url => + ).parTraverse { url => deltaClient.get[Json](url, Rick) { (json, response) => response.status shouldEqual StatusCodes.OK filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) @@ -324,7 +311,7 @@ class ResourcesSpec extends BaseSpec with EitherValuable with CirceEq { List( s"/resources/$id1/test-schema/test-resource:1?rev=1", s"/resources/$id1/_/test-resource:1?rev=1" - ).traverse { url => + ).parTraverse { url => deltaClient.get[Json](url, Rick) { (json, response) => response.status shouldEqual StatusCodes.OK filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) @@ -403,7 +390,7 @@ class ResourcesSpec extends BaseSpec with EitherValuable with CirceEq { "storages" -> jsonContentOf("/kg/listings/default-storage.json", mapping: _*) ) - resources.traverse { case (segment, expected) => + resources.parTraverse { case (segment, expected) => deltaClient.get[Json](s"/$segment/$id1", Rick) { (json, response) => response.status shouldEqual StatusCodes.OK filterSearchMetadata(json) should equalIgnoreArrayOrder(expected) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala index db49e80e0c..ab070f9e41 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala @@ -1,9 +1,7 @@ package ch.epfl.bluebrain.nexus.tests.kg -import java.net.URI -import java.nio.file.Paths - import akka.http.scaladsl.model.StatusCodes +import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote import ch.epfl.bluebrain.nexus.tests.Optics.filterMetadataKeys import ch.epfl.bluebrain.nexus.tests.Tags.StorageTag import ch.epfl.bluebrain.nexus.tests.config.S3Config @@ -16,6 +14,8 @@ import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model._ +import java.net.URI +import java.nio.file.Paths import scala.jdk.CollectionConverters._ class S3StorageSpec extends StorageSpec { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index 2cfaea3346..949012eea3 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -4,15 +4,15 @@ import akka.http.scaladsl.model.headers.{ContentDispositionTypes, HttpEncodings} import akka.http.scaladsl.model.{ContentType, ContentTypes, HttpResponse, StatusCodes} import akka.util.ByteString import ch.epfl.bluebrain.nexus.testkit.CirceEq +import ch.epfl.bluebrain.nexus.tests.BaseSpec import ch.epfl.bluebrain.nexus.tests.HttpClient._ -import ch.epfl.bluebrain.nexus.tests.Identity.UserCredentials +import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.StorageTag import ch.epfl.bluebrain.nexus.tests.config.ConfigLoader._ import ch.epfl.bluebrain.nexus.tests.config.StorageConfig import ch.epfl.bluebrain.nexus.tests.iam.types.Permission import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.Organizations -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} import com.google.common.io.BaseEncoding import com.typesafe.config.ConfigFactory import io.circe.Json @@ -33,21 +33,6 @@ abstract class StorageSpec extends BaseSpec with CirceEq { private[tests] val projId = genId() private[tests] val fullId = s"$orgId/$projId" - private[tests] val testRealm = Realm("storage" + genString()) - private[tests] val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private[tests] val Coyote = UserCredentials(genString(), genString(), testRealm) - - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - Coyote :: Nil - ).runSyncUnsafe() - () - } - def storageName: String def storageType: String diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/VersionSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/VersionSpec.scala index 845e92d2a6..081f9e9c57 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/VersionSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/VersionSpec.scala @@ -38,7 +38,8 @@ object VersionSpec { final case class DependenciesBundle( blazegraph: String, - cassandra: String, + cassandra: Option[String], + postgres: Option[String], elasticsearch: String, remoteStorage: String ) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ViewsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ViewsSpec.scala index 5fb4a3f470..3d0a4601c7 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ViewsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ViewsSpec.scala @@ -3,21 +3,18 @@ package ch.epfl.bluebrain.nexus.tests.kg import akka.http.scaladsl.model.StatusCodes import cats.implicits._ import ch.epfl.bluebrain.nexus.testkit.{CirceEq, EitherValuable} -import ch.epfl.bluebrain.nexus.tests.Identity.{Anonymous, UserCredentials} +import ch.epfl.bluebrain.nexus.tests.BaseSpec +import ch.epfl.bluebrain.nexus.tests.Identity.Anonymous +import ch.epfl.bluebrain.nexus.tests.Identity.views.ScoobyDoo import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.Tags.ViewsTag import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Organizations, Views} -import ch.epfl.bluebrain.nexus.tests.{BaseSpec, Identity, Realm} import io.circe.Json import monix.bio.Task import monix.execution.Scheduler.Implicits.global class ViewsSpec extends BaseSpec with EitherValuable with CirceEq { - private val testRealm = Realm("views" + genString()) - private val testClient = Identity.ClientCredentials(genString(), genString(), testRealm) - private val ScoobyDoo = UserCredentials(genString(), genString(), testRealm) - private val orgId = genId() private val projId = genId() val fullId = s"$orgId/$projId" @@ -27,16 +24,6 @@ class ViewsSpec extends BaseSpec with EitherValuable with CirceEq { val projects = List(fullId, fullId2) - override def beforeAll(): Unit = { - super.beforeAll() - initRealm( - testRealm, - Identity.ServiceAccount, - testClient, - ScoobyDoo :: Nil - ).runSyncUnsafe() - } - "creating projects" should { "add necessary permissions for user" taggedAs ViewsTag in { for { @@ -58,7 +45,7 @@ class ViewsSpec extends BaseSpec with EitherValuable with CirceEq { "create a context" taggedAs ViewsTag in { val payload = jsonContentOf("/kg/views/context.json") - projects.traverse { project => + projects.parTraverse { project => deltaClient.put[Json](s"/resources/$project/resource/test-resource:context", payload, ScoobyDoo) { (_, response) => response.status shouldEqual StatusCodes.Created @@ -78,7 +65,7 @@ class ViewsSpec extends BaseSpec with EitherValuable with CirceEq { "create an ElasticSearch view" taggedAs ViewsTag in { val payload = jsonContentOf("/kg/views/elastic-view.json") - projects.traverse { project => + projects.parTraverse { project => deltaClient.put[Json](s"/views/$project/test-resource:testView", payload, ScoobyDoo) { (_, response) => response.status shouldEqual StatusCodes.Created } @@ -166,7 +153,7 @@ class ViewsSpec extends BaseSpec with EitherValuable with CirceEq { } "post instances" taggedAs ViewsTag in { - (1 to 8).toList.traverse { i => + (1 to 8).toList.parTraverse { i => val payload = jsonContentOf(s"/kg/views/instances/instance$i.json") val id = `@id`.getOption(payload).value val unprefixedId = id.stripPrefix("https://bbp.epfl.ch/nexus/v0/data/bbp/experiment/patchedcell/v0.1.0/") @@ -350,7 +337,7 @@ class ViewsSpec extends BaseSpec with EitherValuable with CirceEq { } "tag resources resource" taggedAs ViewsTag in { - (1 to 5).toList.traverse { i => + (1 to 5).toList.parTraverse { i => val payload = jsonContentOf(s"/kg/views/instances/instance$i.json") val id = `@id`.getOption(payload).value val unprefixedId = id.stripPrefix("https://bbp.epfl.ch/nexus/v0/data/bbp/experiment/patchedcell/v0.1.0/")