Merge branch 'DIRACGrid:integration' into integration

integration_tests.py

@@ -181,8 +181,8 @@ def destroy():
     typer.secho("Shutting down and removing containers", err=True, fg=c.GREEN)
     with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
         os.execvpe(
-            "docker-compose",
-            ["docker-compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
+            "docker",
+            ["docker", "compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
             _make_env({}),
         )
 
@@ -228,10 +228,10 @@ def prepare_environment(
     module_configs = _load_module_configs(modules)
     extra_services = list(chain(*[config["extra-services"] for config in module_configs.values()]))
 
-    typer.secho("Running docker-compose to create containers", fg=c.GREEN)
+    typer.secho("Running docker compose to create containers", fg=c.GREEN)
     with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
         subprocess.run(
-            ["docker-compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client"] + extra_services,
+            ["docker", "compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client"] + extra_services,
             check=True,
             env=docker_compose_env,
         )
@@ -321,7 +321,7 @@ def prepare_environment(
             subprocess.run(command, check=True, shell=True)
 
     docker_compose_fn_final = Path(tempfile.mkdtemp()) / "ci"
-    typer.secho("Running docker-compose to create DiracX containers", fg=c.GREEN)
+    typer.secho("Running docker compose to create DiracX containers", fg=c.GREEN)
     typer.secho(f"Will leave a folder behind: {docker_compose_fn_final}", fg=c.YELLOW)
 
     with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
@@ -333,7 +333,7 @@ def prepare_environment(
         # It is going to start all the diracx containers, including one which waits
         # for the DIRAC installation to be over.
         subprocess.Popen(
-            ["docker-compose", "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
+            ["docker", "compose", "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
             env=docker_compose_env,
             stdin=None,
             stdout=None,
@@ -548,7 +548,7 @@ class TestExit(typer.Exit):
 
 @contextmanager
 def _gen_docker_compose(modules, *, diracx_dist_dir=None):
-    # Load the docker-compose configuration and mount the necessary volumes
+    # Load the docker compose configuration and mount the necessary volumes
     input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml"
     docker_compose = yaml.safe_load(input_fn.read_text())
     # diracx-wait-for-db needs the volume to be able to run the witing script
@@ -590,7 +590,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
 def _check_containers_running(*, is_up=True):
     with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
         running_containers = subprocess.run(
-            ["docker-compose", "-f", docker_compose_fn, "ps", "-q", "-a"],
+            ["docker", "compose", "-f", docker_compose_fn, "ps", "-q", "-a"],
             stdout=subprocess.PIPE,
             env=_make_env({}),
             check=True,

tests/.dirac-ci-config.yaml

@@ -4,7 +4,7 @@ config:
 # List of feature variables which must be passed when preparing
 required-feature-flags: []
 
-# Additional services to use in docker-compose
+# Additional services to use in docker compose
 extra-services: {}
 
 # Additional shell commands to run

tests/CI/README.MD

@@ -18,15 +18,14 @@ To run a test locally using the default configuration, make sure that the local
 
 After the test process has been run, the server and client installation logs, as well as the server/client test logs can be retrieved to $PWD from the docker containers using `source utils.sh && getLogs`.
 
-The command `docker-compose -f tests/CI/docker-compose.yml down -v` stops and removes the spawned containers (cleanup).
+The command `docker compose -f tests/CI/docker-compose.yml down -v` stops and removes the spawned containers (cleanup).
 
 ### System requirements
 
 The officially supported operating system is CERN Centos 7 ([CC7](http://linux.web.cern.ch/linux/centos7/)), should you use another Linux distribution, please install the dependencies according to your distribution.
 
 The following dependencies/softwares are required:
 * Docker v18+ (lower version should work, but no promises are made).
-* Docker-Compose v2.4+ format
 
 ```
 sudo yum install -y docker
@@ -36,7 +35,6 @@ sudo groupadd docker
 sudo usermod -aG docker $USER    # So that you don't need sudo docker
 
 sudo yum install -y python-pip python-devel libffi-devel openssl-devel gcc glibc-devel make curl
-pip install docker-compose
 ```
 
 ## Test environment configuration

tests/CI/envs/elasticsearch:7.9.1.env

@@ -7,4 +7,4 @@ xpack.security.enabled=true
 ELASTIC_PASSWORD="changeme"
 # Elasticsearch allocates 1GB of memory by default. As resources are limited
 # and elasticsearch performance isn't critical in CI, limit this to 256MB
-"ES_JAVA_OPTS=-Xms256m -Xmx256m"
+ES_JAVA_OPTS="-Xms256m -Xmx256m"

tests/CI/exportCSLoop.sh

@@ -19,7 +19,6 @@ curl -L https://raw.githubusercontent.com/DIRACGrid/diracx/main/diracx-cli/tests
 while true; do
     DIRAC_COMPAT_ENABLE_CS_CONVERSION=x dirac internal legacy cs-sync \
         "$DIRACOS/etc/Production.cfg" \
-        /home/dirac/TestCode/diracx/tests/cli/legacy/cs_sync/convert_integration_test.yaml \
         /cs_store/initialRepo/default.yml
     git --git-dir=.git -C /cs_store/initialRepo/ commit -am "export $(date)"
     if [[ "${1}" == "--once" ]]; then

tests/Jenkins/dirac-cfg-setup-diracx.py

@@ -2,6 +2,8 @@
 import argparse
 import os
 
+from diraccfg import CFG
+
 import DIRAC
 from DIRAC.Core.Utilities.ReturnValues import returnValueOrRaise
 
@@ -36,6 +38,39 @@ def main(url: str, disabled_vos: list[str]):
     if disabled_vos:
         returnValueOrRaise(csAPI.setOption("DiracX/DisabledVOs", ",".join(disabled_vos)))
 
+    csSync = {
+        "CsSync": {
+            "VOs": {
+                "Jenkins": {
+                    "DefaultGroup": "jenkins_user",
+                    "IdP": {"ClientID": "995ed3b9-d5bd-49d3-a7f4-7fc7dbd5a0cd", "URL": "https://jenkins.invalid/"},
+                    "UserSubjects": {
+                        "adminusername": "e2cb28ec-1a1e-40ee-a56d-d899b79879ce",
+                        "ciuser": "26dbe36e-cf5c-4c52-a834-29a1c904ef74",
+                        "trialUser": "a95ab678-3fa4-41b9-b863-fe62ce8064ce",
+                    },
+                    "Support": {
+                        "Message": "Contact the help desk",
+                        "Email": "helpdesk@example.invalid",
+                        "Webpage": "https://helpdesk.vo.invalid",
+                    },
+                },
+                "vo": {
+                    "DefaultGroup": "dirac_user",
+                    "IdP": {"ClientID": "072afab5-ed92-46e0-a61d-4ecbc96e0770", "URL": "https://vo.invalid/"},
+                    "UserSubjects": {
+                        "adminusername": "26b14fc9-6d40-4ca5-b014-6234eaf0fb6e",
+                        "ciuser": "d3adc733-6588-4d6f-8581-5986b02d0c87",
+                        "trialUser": "ff2152ff-34f4-4739-b106-3def37e291e3",
+                    },
+                },
+            }
+        }
+    }
+
+    csSyncCFG = CFG().loadFromDict(csSync)
+    returnValueOrRaise(csAPI.mergeCFGUnderSection("/DiracX/", csSyncCFG))
+
     returnValueOrRaise(csAPI.commit())