Merge pull request #22 from ldsec/dev

MedCo v1.0.0
ldsec · Mar 31, 2020 · f6fb414 · f6fb414
2 parents 9016220 + a9433cd
commit f6fb414
Show file tree

Hide file tree

Showing 14 changed files with 93 additions and 69 deletions.
diff --git a/compose-profiles/docker-compose-definitions.yml b/compose-profiles/docker-compose-definitions.yml
@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   i2b2:
-    image: medco/i2b2:${I2B2_VERSION:-v0.3.0-MedCo}
+    image: medco/i2b2:${I2B2_VERSION:-v1.0.0-MedCo}
     build:
       context: ../docker-images/i2b2
     environment:
@@ -17,26 +17,21 @@ services:
       - AXIS2_LOGLEVEL=INFO
 
   medco-unlynx:
-    image: medco/medco-unlynx:${MEDCO_UNLYNX_VERSION:-v0.3.1}
-    ports:
-    - "2000"
-    - "2001"
+    image: medco/medco-unlynx:${MEDCO_UNLYNX_VERSION:-v1.0.0}
     environment:
     - NODE_IDX=0
     - UNLYNX_DEBUG_LEVEL=1
     volumes:
     - ../configuration-profiles/dev-local-3nodes:/medco-configuration
 
   nginx:
-    image: medco/nginx:${NGINX_VERSION:-v0.3.1-MedCo}
+    image: medco/nginx:${NGINX_VERSION:-v1.0.0-MedCo}
     build:
       context: ../docker-images/nginx
-    ports:
-      - "80"
-      - "443"
     environment:
       - HTTP_SCHEME=http
       - ALL_TIMEOUTS_SECONDS=600
+      - PROD_CONFIG=false
     volumes:
       - ../docker-images/nginx/www-data:/www-data
       - ../docker-images/nginx/conf.d:/etc/nginx/conf.d
@@ -45,22 +40,20 @@ services:
     image: postgres:9.6
     environment:
     - POSTGRES_PASSWORD=postgres
-    ports:
-    - "5432"
     volumes:
     - medcodb:/var/lib/postgresql/data
     - ../docker-images/postgresql/initdb-data:/docker-entrypoint-initdb.d
 
   pg-admin:
-    image: medco/pgadmin4:${PGADMIN_VERSION:-v0.3.1-MedCo}
+    image: medco/pgadmin4:${PGADMIN_VERSION:-v1.0.0-MedCo}
     build:
       context: ../docker-images/pgadmin
     environment:
       - PGADMIN_DEFAULT_EMAIL=admin
       - PGADMIN_DEFAULT_PASSWORD=admin
 
   keycloak:
-    image: medco/keycloak:${KEYCLOAK_VERSION:-v0.3.1-MedCo}
+    image: medco/keycloak:${KEYCLOAK_VERSION:-v1.0.0-MedCo}
     build:
       context: ../docker-images/keycloak
     environment:
@@ -74,9 +67,7 @@ services:
       - DB_PASSWORD=keycloak
 
   glowing-bear-medco:
-    image: medco/glowing-bear-medco:${GLOWING_BEAR_MEDCO_VERSION:-v0.3.1}
-    ports:
-      - "80"
+    image: medco/glowing-bear-medco:${GLOWING_BEAR_MEDCO_VERSION:-v1.0.0}
     environment:
       - GB_MEDCO_NODE_URL=http://localhost/local-3nodes/medco-0
       - GB_KEYCLOAK_URL=http://localhost/auth
@@ -85,9 +76,7 @@ services:
       - GB_FOOTER_TEXT=
 
   medco-connector:
-    image: medco/medco-connector:${MEDCO_CONNECTOR_VERSION:-v0.3.1}
-    ports:
-      - "1999"
+    image: medco/medco-connector:${MEDCO_CONNECTOR_VERSION:-v1.0.0}
     environment:
       - SERVER_HTTP_WRITE_TIMEOUT_SECONDS=600
       - I2B2_HIVE_URL=http://i2b2:8080/i2b2/services
@@ -116,7 +105,7 @@ services:
       - ../configuration-profiles/dev-local-3nodes:/medco-configuration
 
   medco-cli-client:
-    image: medco/medco-cli-client:${MEDCO_CONNECTOR_VERSION:-v0.3.1}
+    image: medco/medco-cli-client:${MEDCO_CONNECTOR_VERSION:-v1.0.0}
     environment:
       - LOG_LEVEL=3
       - UNLYNX_GROUP_FILE_PATH=/medco-configuration/group.toml
@@ -132,7 +121,7 @@ services:
     network_mode: host
 
   medco-loader:
-    image: medco/medco-loader:${MEDCO_LOADER_VERSION:-v0.3.1}
+    image: medco/medco-loader:${MEDCO_LOADER_VERSION:-v1.0.0}
     environment:
       - LOG_LEVEL=3
       - UNLYNX_GROUP_FILE_PATH=/medco-configuration/group.toml

diff --git a/docker-images/i2b2/Dockerfile b/docker-images/i2b2/Dockerfile
@@ -43,6 +43,7 @@ ENV I2B2_DB_HOST="postgresql" \
 
 # build-time variables
 ENV I2B2_SQL_DIR="/i2b2-sql" \
+    I2B2_COMPRESSED_DATA_DIR="/i2b2-compressed-data" \
     I2B2_DATA_DIR="/i2b2-data" \
     PRE_INIT_SCRIPT_DIR="/pre-init-scripts" \
     I2B2_DATA_VERSION="v1.7.10.0002" \
@@ -61,11 +62,14 @@ USER root
 RUN sed -i 's/Xmx512m/Xmx2048m/g' $JBOSS_HOME/bin/standalone.conf && \
     sed -i 's/MaxMetaspaceSize=256m/MaxMetaspaceSize=1024m/g' $JBOSS_HOME/bin/standalone.conf && \
     chmod +x /usr/local/bin/docker-entrypoint.sh && \
-    mkdir -p "$I2B2_DATA_DIR" "$I2B2_FR_FILES_DIR" && \
+    mkdir -p "$I2B2_COMPRESSED_DATA_DIR" "$I2B2_DATA_DIR" "$I2B2_FR_FILES_DIR" && \
     chown -R jboss:jboss "$I2B2_DATA_DIR" "$JBOSS_HOME" && \
     yum update -y && \
     yum -y install wget git ant postgresql && \
     yum clean all
+RUN git clone --depth 1 --branch "$I2B2_DATA_VERSION" https://github.com/i2b2/i2b2-data.git "$I2B2_DATA_DIR" && \
+    GZIP=-9 tar -cvzf "$I2B2_COMPRESSED_DATA_DIR"/i2b2-data.tar.gz "$I2B2_DATA_DIR" && \
+    rm -rf "$I2B2_DATA_DIR"/{*,.*} || true
 USER jboss
 
 # run

diff --git a/docker-images/i2b2/docker-entrypoint.sh b/docker-images/i2b2/docker-entrypoint.sh
@@ -17,15 +17,17 @@ echo "Initialising i2b2 database"
         CREATE DATABASE ${I2B2_DB_NAME};
 EOSQL
 
-    # get the i2b2 data
-    pushd "$I2B2_DATA_DIR"
-    git clone --depth 1 --branch "$I2B2_DATA_VERSION" https://github.com/i2b2/i2b2-data.git .
-    popd
+    # uncompress the i2b2 data
+    tar -xf "$I2B2_COMPRESSED_DATA_DIR"/"i2b2-data.tar.gz" -C "$I2B2_DATA_DIR" --strip-components 1
 
     # run loading scripts
     for f in "$I2B2_SQL_DIR"/*.sh; do
         bash "$f"
     done
+
+    # delete loaded data
+    rm -rf "$I2B2_DATA_DIR"/{*,.*} || true
+
 fi
 
 # execute pre-init scripts & run wildfly

diff --git a/docker-images/nginx/.gitignore b/docker-images/nginx/.gitignore
@@ -1,2 +1,3 @@
 conf.d/servers.conf
-conf.d/common/server-revproxy.conf.inc
+conf.d/common/server-revproxy-base.conf.inc
+conf.d/common/server-revproxy-dev.conf.inc
diff --git a/docker-images/nginx/Dockerfile b/docker-images/nginx/Dockerfile
@@ -2,7 +2,8 @@ FROM nginx:1.15.10
 
 # run-time variables
 ENV HTTP_SCHEME="http" \
-    ALL_TIMEOUTS_SECONDS="600"
+    ALL_TIMEOUTS_SECONDS="600" \
+    PROD_CONFIG="true"
 
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 RUN chmod a+x /usr/local/bin/docker-entrypoint.sh

diff --git a/docker-images/nginx/conf.d/common/security-headers.inc b/docker-images/nginx/conf.d/common/security-headers.inc
@@ -0,0 +1,17 @@
+# do not send out web server info
+server_tokens off;
+
+# various HTTP headers for enhanced security
+add_header 'X-Frame-Options' 'sameorigin' always;
+add_header 'X-XSS-Protection' '1; mode=block' always;
+add_header 'X-Content-Type-Options' 'nosniff' always;
+add_header 'Content-Security-Policy' 'default-src \'self\' \'unsafe-inline\' \'unsafe-eval\'; connect-src *' always;
+
+# prevent disallowed HTTP methods
+if ($request_method !~* (GET|POST|OPTIONS|HEAD|DELETE|PUT)) {
+return 405;
+}
+
+# TODO
+## Content-Security-Policy: allow connect-src to other MedCo node;
+## Feature-Policy: blacklist unused features (most of them)
diff --git a/.../common/server-revproxy.conf.inc.template → ...on/server-revproxy-base.conf.inc.template b/.../common/server-revproxy.conf.inc.template → ...on/server-revproxy-base.conf.inc.template
@@ -31,18 +31,6 @@ location /auth {
     proxy_pass http://$upstream_host:8080;
 }
 
-location /pgadmin {
-    set $upstream_host pg-admin;
-    proxy_pass http://$upstream_host;
-    proxy_set_header X-Script-Name /pgadmin;
-    proxy_redirect http://$upstream_host/pgadmin /pgadmin;
-}
-
-location /i2b2 {
-    set $upstream_host i2b2;
-    proxy_pass http://$upstream_host:8080;
-}
-
 location /glowing-bear {
     set $upstream_host glowing-bear-medco;
     proxy_pass http://$upstream_host;
@@ -53,27 +41,3 @@ location /medco {
     set $upstream_host medco-connector;
     proxy_pass http://$upstream_host:1999;
 }
-
-
-### --- only used in dev or test profiles
-
-location /local-3nodes/medco-0 {
-    include /etc/nginx/conf.d/common/cors.inc;
-    set $upstream_host medco-connector-srv0;
-    rewrite /local-3nodes/medco-0/(.*) /medco/$1 break;
-    proxy_pass http://$upstream_host:1999;
-}
-
-location /local-3nodes/medco-1 {
-    include /etc/nginx/conf.d/common/cors.inc;
-    set $upstream_host medco-connector-srv1;
-    rewrite /local-3nodes/medco-1/(.*) /medco/$1 break;
-    proxy_pass http://$upstream_host:1999;
-}
-
-location /local-3nodes/medco-2 {
-    include /etc/nginx/conf.d/common/cors.inc;
-    set $upstream_host medco-connector-srv2;
-    rewrite /local-3nodes/medco-2/(.*) /medco/$1 break;
-    proxy_pass http://$upstream_host:1999;
-}
diff --git a/docker-images/nginx/conf.d/common/server-revproxy-dev.conf.inc.template b/docker-images/nginx/conf.d/common/server-revproxy-dev.conf.inc.template
@@ -0,0 +1,32 @@
+location /pgadmin {
+    set $upstream_host pg-admin;
+    proxy_pass http://$upstream_host;
+    proxy_set_header X-Script-Name /pgadmin;
+    proxy_redirect http://$upstream_host/pgadmin /pgadmin;
+}
+
+location /i2b2 {
+    set $upstream_host i2b2;
+    proxy_pass http://$upstream_host:8080;
+}
+
+location /local-3nodes/medco-0 {
+    include /etc/nginx/conf.d/common/cors.inc;
+    set $upstream_host medco-connector-srv0;
+    rewrite /local-3nodes/medco-0/(.*) /medco/$1 break;
+    proxy_pass http://$upstream_host:1999;
+}
+
+location /local-3nodes/medco-1 {
+    include /etc/nginx/conf.d/common/cors.inc;
+    set $upstream_host medco-connector-srv1;
+    rewrite /local-3nodes/medco-1/(.*) /medco/$1 break;
+    proxy_pass http://$upstream_host:1999;
+}
+
+location /local-3nodes/medco-2 {
+    include /etc/nginx/conf.d/common/cors.inc;
+    set $upstream_host medco-connector-srv2;
+    rewrite /local-3nodes/medco-2/(.*) /medco/$1 break;
+    proxy_pass http://$upstream_host:1999;
+}
diff --git a/docker-images/nginx/conf.d/servers.conf.http.inc b/docker-images/nginx/conf.d/servers.conf.http.inc
@@ -3,5 +3,7 @@ server {
     server_name _;
 
     include /etc/nginx/conf.d/common/server-default.conf.inc;
-    include /etc/nginx/conf.d/common/server-revproxy.conf.inc;
+    include /etc/nginx/conf.d/common/server-revproxy-base.conf.inc;
+    include /etc/nginx/conf.d/common/server-revproxy-dev.conf.inc;
+    include /etc/nginx/conf.d/common/security-headers.inc;
 }
diff --git a/docker-images/nginx/conf.d/servers.conf.https.inc b/docker-images/nginx/conf.d/servers.conf.https.inc
@@ -10,6 +10,8 @@ server {
     server_name _;
 
     include /etc/nginx/conf.d/common/server-default.conf.inc;
-    include /etc/nginx/conf.d/common/server-revproxy.conf.inc;
+    include /etc/nginx/conf.d/common/server-revproxy-base.conf.inc;
+    include /etc/nginx/conf.d/common/server-revproxy-dev.conf.inc;
     include /etc/nginx/conf.d/common/server-https.conf.inc;
+    include /etc/nginx/conf.d/common/security-headers.inc;
 }
diff --git a/docker-images/nginx/docker-entrypoint.sh b/docker-images/nginx/docker-entrypoint.sh
@@ -3,8 +3,16 @@ set -Eeuo pipefail
 
 # apply configuration from environment variables
 pushd /etc/nginx/conf.d/
+
 envsubst '$HTTP_SCHEME' < servers.conf.template > servers.conf
-envsubst '$ALL_TIMEOUTS_SECONDS' < common/server-revproxy.conf.inc.template > common/server-revproxy.conf.inc
+envsubst '$ALL_TIMEOUTS_SECONDS' < common/server-revproxy-base.conf.inc.template > common/server-revproxy-base.conf.inc
+
+if [[ ${PROD_CONFIG} == "false" ]]; then
+  cp common/server-revproxy-dev.conf.inc.template common/server-revproxy-dev.conf.inc
+else
+  touch common/server-revproxy-dev.conf.inc
+fi
+
 popd
 
 exec nginx -g 'daemon off;'
diff --git a/resources/data/download.sh b/resources/data/download.sh
@@ -12,6 +12,8 @@ wget -O ${SCRIPT_FOLDER}/genomic/tcga_cbio/mutation_data.csv ${REPO_URL}/genomic
 wget -O ${SCRIPT_FOLDER}/genomic/tcga_cbio/clinical_data.csv ${REPO_URL}/genomic/tcga_cbio/clinical_data.csv?raw=true
 wget -O ${SCRIPT_FOLDER}/genomic/tcga_cbio/8_mutation_data.csv ${REPO_URL}/genomic/tcga_cbio/8_mutation_data.csv?raw=true
 wget -O ${SCRIPT_FOLDER}/genomic/tcga_cbio/8_clinical_data.csv ${REPO_URL}/genomic/tcga_cbio/8_clinical_data.csv?raw=true
+wget -O ${SCRIPT_FOLDER}/genomic/tcga_cbio/mutation_data_fake.csv ${REPO_URL}/genomic/tcga_cbio/mutation_data_fake.csv?raw=true
+wget -O ${SCRIPT_FOLDER}/genomic/tcga_cbio/clinical_data_fake.csv ${REPO_URL}/genomic/tcga_cbio/clinical_data_fake.csv?raw=true
 wget -O ${SCRIPT_FOLDER}/genomic/sensitive.txt ${REPO_URL}/genomic/sensitive.txt?raw=true
 
 # i2b2 demo (v1) dataset

diff --git a/resources/profile-generation-scripts/test-network/step1.sh b/resources/profile-generation-scripts/test-network/step1.sh
@@ -20,7 +20,7 @@ PRIV_KEY="${5-}"
 # convenience variables
 PROFILE_NAME="test-network-${NETWORK_NAME}-node${NODE_IDX}"
 SCRIPT_FOLDER="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-MEDCO_UNLYNX_VER="v0.3.1"
+MEDCO_UNLYNX_VER="v1.0.0"
 CONF_FOLDER="${SCRIPT_FOLDER}/../../../configuration-profiles/${PROFILE_NAME}"
 COMPOSE_FOLDER="${SCRIPT_FOLDER}/../../../compose-profiles/${PROFILE_NAME}"
 if [[ -d ${CONF_FOLDER} ]] || [[ -d ${COMPOSE_FOLDER} ]]; then

diff --git a/resources/profile-generation-scripts/test-network/step2.sh b/resources/profile-generation-scripts/test-network/step2.sh
@@ -14,7 +14,7 @@ SECRETS="${3-}"
 # convenience variables
 PROFILE_NAME="test-network-${NETWORK_NAME}-node${NODE_IDX}"
 SCRIPT_FOLDER="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-MEDCO_UNLYNX_VER="v0.3.1"
+MEDCO_UNLYNX_VER="v1.0.0"
 CONF_FOLDER="${SCRIPT_FOLDER}/../../../configuration-profiles/${PROFILE_NAME}"
 COMPOSE_FOLDER="${SCRIPT_FOLDER}/../../../compose-profiles/${PROFILE_NAME}"
 if [[ ! -d ${CONF_FOLDER} ]] || [[ ! -d ${COMPOSE_FOLDER} ]] || [[ -f ${CONF_FOLDER}/group.toml ]]; then