update version to 3.5.0

Author: binzhang7101

CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## 3.5.0
+[Released](https://bitbucket.org/xnatdev/container-service/src/3.5.0/).
+
+* **Improvement** [CS-946][] Prevent setting mutually distinct k8s PVC mounting options
+* **Bugfix** [CS-966][] Ensure tracking of container IDs in workflow tables in a Kubernetes environment
+* **Bugfix** [CS-968][] Switch the docker API library we use from [docker-client][] to [docker-java][].
+    This should restore CS functionality on docker engine v25 and higher.
+    
+### A Note About Our Docker Library
+Originally we used the [spotify/docker-client][] library to wrap the docker remote API in java method calls. They stopped updating that and put out their final release [v6.1.1][] in 2016.  
+
+We switched the Container Service to use a fork of that client, [dmandalidis/docker-client][] in CS version 3.0.0. Given that this was a fork of the client we already used, it was a simple drop-in replacement with no changes needed.
+
+But that library maintainer did continue to make changes. In 2023 they released a major version upgrade, [v7.0.0][], which dropped support for Java 8. That is the version of Java we use in XNAT (at time of writing) so this change meant we weren't able to update our version of this library. That was fine for a while...  
+...Until version 25 of the docker engine, in which they made an API change which caused an error in the version we used of `docker-client`. The library (presumably) fixed their issue but we weren't able to use that fix because our version of the library was frozen by their decision to drop Java 8 support.
+
+This forced us to switch our library from `docker-client` to [docker-java][]. This was not a drop-in replacement, and did require a migration. All the same docker API endpoints were supported in a 1:1 replacement—which took a little effort but was straightforward—except for one. The `docker-java` library did not support requesting `GenericResources` on a swarm service, which is the mechanism by which we allow commands to specify that they need a GPU. We opened a ticket reporting that lack of support (https://github.com/docker-java/docker-java/issues/2320), but at time of writing there has been no response. I created a fork (https://github.com/johnflavin/docker-java) and fixed the issue myself (https://github.com/docker-java/docker-java/pull/2327), but at time of writing that also has no response. I built a custom version of `docker-java` `3.4.0.1` and pushed that to the XNAT artifactory ([ext-release-local/com/github/docker-java][]).
+
+Long story short, as of CS version `3.5.0` we depend on `docker-java` version `3.4.0.1` for our docker (and swarm) API support.
+
+[CS-946]: https://radiologics.atlassian.net/browse/CS-946
+[CS-966]: https://radiologics.atlassian.net/browse/CS-966
+[CS-968]: https://radiologics.atlassian.net/browse/CS-968
+[docker-client]: https://github.com/spotify/docker-client
+[spotify/docker-client]: https://github.com/spotify/docker-client
+[v6.1.1]: https://github.com/spotify/docker-client/releases/tag/v6.1.1
+[dmandalidis/docker-client]: https://github.com/dmandalidis/docker-client
+[v7.0.0]: https://github.com/dmandalidis/docker-client/tree/v7.0.0
+[docker-java]: https://github.com/docker-java/docker-java
+[ext-release-local/com/github/docker-java]: https://nrgxnat.jfrog.io/ui/repos/tree/General/ext-release-local/com/github/docker-java
+
 ## 3.4.3
 [Released](https://bitbucket.org/xnatdev/container-service/src/3.4.3/).
 

build.gradle

@@ -16,7 +16,7 @@ plugins {
 }
 
 group "org.nrg.xnatx.plugins"
-version "3.4.3"
+version "3.5.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -42,22 +42,22 @@ dependencyManagement {
         }
     }
 }
-def vDockerClient = "6.0.4"
+
+def vDockerJava = "3.4.0.1"
 def vAutoValue = "1.3"
 def vKubernetesClient = "15.0.+"
 def vPowerMock = "1.7.0"
 def vGson = dependencyManagement.importedProperties["gson.version"]
 def vJackson = dependencyManagement.importedProperties["jackson.version"]
 def vSpringSecurity = dependencyManagement.importedProperties["spring-security.version"]
 def vActiveMQ = dependencyManagement.importedProperties["activemq.version"]
-def vJersey = "3.0.6"
+def vBouncyCastle = "1.64" // Included in xnat-web via spring-security-jwt 1.1.1, see XNAT-7907
 
 // Use this configuration to put dependencies into the fat jar
 configurations {
     implementation.extendsFrom(implementAndInclude)
     all {
         resolutionStrategy.force "org.mockito:mockito-core:${vMockito}"
-        exclude group: "org.bouncycastle"
     }
 }
 
@@ -67,22 +67,8 @@ dependencies {
 
     compileOnly "com.google.auto.value:auto-value:${vAutoValue}"
 
-    implementAndInclude ("org.mandas:docker-client:${vDockerClient}") {
-        exclude group: "ch.qos.logback"
-        exclude group: "org.slf4j"
-        exclude group: "com.fasterxml.jackson.core"
-        exclude group: "org.apache.commons", module: "commons-compress"  // included in xnat-web via parent
-    }
-
-    implementAndInclude "org.glassfish.jersey.core:jersey-client:${vJersey}"
-    implementAndInclude "org.glassfish.jersey.inject:jersey-hk2:${vJersey}"
-    implementAndInclude ("org.glassfish.jersey.connectors:jersey-apache-connector:${vJersey}") {
-        exclude group: "org.apache.httpcomponents", module: "httpclient"  // included in xnat-web directly
-    }
-    implementAndInclude ("org.glassfish.jersey.media:jersey-media-json-jackson:${vJersey}") {
-        exclude group: "com.fasterxml.jackson.core"  // Included in xnat-web directly
-        exclude group: "com.fasterxml.jackson.module", module: "jackson-module-jaxb-annotations"  // Included in xnat-web via spawner -> jackson-dataformat-xml
-    }
+    implementAndInclude "com.github.docker-java:docker-java-core:${vDockerJava}"
+    implementAndInclude "com.github.docker-java:docker-java-transport-okhttp:${vDockerJava}"
 
     implementAndInclude ("io.kubernetes:client-java:${vKubernetesClient}") {
         exclude group: "com.google.code.gson"
@@ -92,7 +78,9 @@ dependencies {
         exclude group: "org.apache.commons", module: "commons-compress"
         exclude group: "org.apache.commons", module: "commons-lang3"
         exclude group: "org.slf4j"
+        exclude group: "org.bouncycastle"
         exclude group: "org.yaml", module: "snakeyaml"
+        exclude group: "com.squareup.okhttp3", module: "okhttp" // Included via docker-java-transport-okhttp
     }
     implementAndInclude ("io.kubernetes:client-java-api:${vKubernetesClient}") {  // Explicitly including to exclude transitive deps
         exclude group: "com.google.code.gson"
@@ -177,6 +165,8 @@ dependencies {
 
     implementation group: 'javax.interceptor', name: 'javax.interceptor-api', version: '1.2.2'
 
+    runtimeOnly "org.bouncycastle:bcpkix-jdk15on:$vBouncyCastle"
+    runtimeOnly "org.bouncycastle:bcprov-jdk15on:$vBouncyCastle"
     runtimeOnly "commons-codec:commons-codec"
     runtimeOnly "commons-io:commons-io"
     runtimeOnly "org.apache.commons:commons-compress"
@@ -203,8 +193,18 @@ dependencies {
     testImplementation "com.vladmihalcea:hibernate-types-43:${dependencyManagement.importedProperties["vladmihalcea-hibernate-types.version"]}"
 }
 
-task unitTest( type: Test ) {
-    exclude "**/*IntegrationTest.class"
+test {
+    systemProperty "integration", System.getProperty('integration') == "true"
+}
+
+tasks.register ( 'unitTest', Test ) {
+    systemProperties['integration'] = "false"
+    test
+}
+
+tasks.register ( 'integrationTest', Test ) {
+    systemProperties['integration'] = "true"
+    test
 }
 
 // Pulls in the Jenkins BUILD_NUMBER environment variable if available.

src/main/java/org/nrg/containers/api/ContainerControlApi.java

@@ -1,18 +1,15 @@
 package org.nrg.containers.api;
 
-import org.mandas.docker.client.DockerClient;
-import org.mandas.docker.client.exceptions.ServiceNotFoundException;
-import org.mandas.docker.client.exceptions.TaskNotFoundException;
 import org.nrg.containers.events.model.DockerContainerEvent;
 import org.nrg.containers.exceptions.ContainerBackendException;
 import org.nrg.containers.exceptions.ContainerException;
 import org.nrg.containers.exceptions.DockerServerException;
 import org.nrg.containers.exceptions.NoContainerServerException;
 import org.nrg.containers.exceptions.NoDockerServerException;
-import org.nrg.containers.model.command.auto.Command;
+import org.nrg.containers.exceptions.ServiceNotFoundException;
+import org.nrg.containers.exceptions.TaskNotFoundException;
 import org.nrg.containers.model.command.auto.ResolvedCommand;
 import org.nrg.containers.model.container.auto.Container;
-import org.nrg.containers.model.container.auto.ContainerMessage;
 import org.nrg.containers.model.container.auto.ServiceTask;
 import org.nrg.containers.model.dockerhub.DockerHubBase;
 import org.nrg.containers.model.dockerhub.DockerHubBase.DockerHub;
@@ -24,7 +21,6 @@
 import java.time.OffsetDateTime;
 import java.util.Date;
 import java.util.List;
-import java.util.Map;
 
 public interface ContainerControlApi {
 
@@ -45,44 +41,17 @@ public interface ContainerControlApi {
     Container create(ResolvedCommand resolvedCommand, UserI user) throws NoContainerServerException, ContainerBackendException, ContainerException;
     Container create(Container toCreate, UserI user) throws NoContainerServerException, ContainerBackendException, ContainerException;
     void start(final Container toStart) throws NoContainerServerException, ContainerBackendException;
-    @Deprecated Container createContainerOrSwarmService(final ResolvedCommand dockerCommand, final UserI userI) throws NoDockerServerException, DockerServerException, ContainerException;
-    @Deprecated Container createContainerOrSwarmService(final Container container, final UserI userI) throws NoDockerServerException, DockerServerException, ContainerException;
-    @Deprecated void startContainer(final Container containerOrService) throws NoDockerServerException, DockerServerException;
-
-    List<Command> parseLabels(final String imageName)
-            throws DockerServerException, NoDockerServerException, NotFoundException;
-
-    List<ContainerMessage> getAllContainers() throws NoDockerServerException, DockerServerException;
-    List<ContainerMessage> getContainers(final Map<String, String> params) throws NoDockerServerException, DockerServerException;
-    ContainerMessage getContainer(final String id) throws NotFoundException, NoDockerServerException, DockerServerException;
-    String getContainerStatus(final String id) throws NotFoundException, NoDockerServerException, DockerServerException;
 
     String getLog(Container container, LogType logType) throws ContainerBackendException, NoContainerServerException;
     String getLog(Container container, LogType logType, Boolean withTimestamps, OffsetDateTime since) throws ContainerBackendException, NoContainerServerException;
-    @Deprecated String getStdoutLog(Container container) throws NoDockerServerException, DockerServerException;
-    @Deprecated String getStderrLog(Container container) throws NoDockerServerException, DockerServerException;
-    @Deprecated String getContainerStdoutLog(String containerId, DockerClient.LogsParam... logParams) throws NoDockerServerException, DockerServerException;
-    @Deprecated String getContainerStderrLog(String containerId, DockerClient.LogsParam... logParams) throws NoDockerServerException, DockerServerException;
-    @Deprecated String getServiceStdoutLog(String serviceId, DockerClient.LogsParam... logParams) throws NoDockerServerException, DockerServerException;
-    @Deprecated String getServiceStderrLog(String serviceId, DockerClient.LogsParam... logParams) throws NoDockerServerException, DockerServerException;
-    
+
     void kill(Container container) throws NoContainerServerException, ContainerBackendException, NotFoundException;
     void autoCleanup(Container container) throws NoContainerServerException, ContainerBackendException, NotFoundException;
     void remove(Container container) throws NoContainerServerException, ContainerBackendException, NotFoundException;
-    @Deprecated void killContainer(final String id) throws NoDockerServerException, DockerServerException, NotFoundException;
-    @Deprecated void killService(String id) throws NoDockerServerException, DockerServerException, NotFoundException;
-    @Deprecated void removeContainerOrService(final Container container) throws NoDockerServerException, DockerServerException;
 
-    ServiceTask getTaskForService(Container service) throws NoDockerServerException, DockerServerException, ServiceNotFoundException, TaskNotFoundException;
     ServiceTask getTaskForService(DockerServer dockerServer, Container service) throws DockerServerException, ServiceNotFoundException, TaskNotFoundException;
 
     List<DockerContainerEvent> getContainerEvents(final Date since, final Date until) throws NoDockerServerException, DockerServerException;
-    @Deprecated void throwContainerEvents(final Date since, final Date until) throws NoDockerServerException, DockerServerException;
-    @Deprecated void throwLostTaskEventForService(Container service);
-    @Deprecated void throwTaskEventForService(Container service) throws NoDockerServerException, DockerServerException, ServiceNotFoundException, TaskNotFoundException;
-    @Deprecated void throwTaskEventForService(DockerServer dockerServer, Container service) throws DockerServerException, ServiceNotFoundException, TaskNotFoundException;
-    @Deprecated void throwRestartEventForService(Container service) throws ContainerException;
-    @Deprecated void throwWaitingEventForService(Container service) throws ContainerException;
 
     Integer getFinalizingThrottle();
     boolean isStatusEmailEnabled();