Merge pull request #16 from ldsec/dev

MedCo v0.3.0

.gitignore

@@ -1,3 +1,5 @@
 .DS_Store
 configuration-profiles/dev/
-resources/data/
+resources/data/
+compose-profiles/test-network-*
+configuration-profiles/test-network-*

.travis.yml

@@ -0,0 +1,30 @@
+sudo: required
+language: minimal
+services:
+  - docker
+  - postgresql
+
+notifications:
+  email: false
+
+env:
+  #MEDCO_DB_NUMBER is the number of databases used in the profile that will be tested
+  - MEDCO_DB_NUMBER=9 MEDCO_DB_HOST=localhost MEDCO_DB_PORT=5432 MEDCO_DB_USER=postgres MEDCO_DB_PASSWORD=postgres MEDCO_DB_NAME=postgres MEDCO_USER_USERNAME=test MEDCO_USER_PASSWORD=test
+
+
+before_install:
+  - cd ~/build/ldsec/medco-deployment/resources/data
+  - bash download.sh
+  - cd ~/build/ldsec/medco-deployment/compose-profiles/dev-local-3nodes
+  - docker-compose build
+  - docker-compose -f docker-compose.tools.yml pull medco-cli-client medco-loader-srv0 medco-loader-srv1 medco-loader-srv2
+
+install:
+  - docker-compose up -d
+  - bash ~/build/ldsec/medco-deployment/resources/utility-scripts/wait_for_startup.sh $MEDCO_DB_NUMBER $MEDCO_DB_HOST $MEDCO_DB_PORT $MEDCO_DB_USER $MEDCO_DB_PASSWORD $MEDCO_DB_NAME
+  - docker-compose -f docker-compose.tools.yml run medco-loader-srv0 v0 --ont_clinical /data/genomic/tcga_cbio/8_clinical_data.csv --sen /data/genomic/sensitive.txt --ont_genomic /data/genomic/tcga_cbio/8_mutation_data.csv --clinical /data/genomic/tcga_cbio/8_clinical_data.csv --genomic /data/genomic/tcga_cbio/8_mutation_data.csv --output /data/
+  - docker-compose -f docker-compose.tools.yml run medco-loader-srv1 v0 --ont_clinical /data/genomic/tcga_cbio/8_clinical_data.csv --sen /data/genomic/sensitive.txt --ont_genomic /data/genomic/tcga_cbio/8_mutation_data.csv --clinical /data/genomic/tcga_cbio/8_clinical_data.csv --genomic /data/genomic/tcga_cbio/8_mutation_data.csv --output /data/
+  - docker-compose -f docker-compose.tools.yml run medco-loader-srv2 v0 --ont_clinical /data/genomic/tcga_cbio/8_clinical_data.csv --sen /data/genomic/sensitive.txt --ont_genomic /data/genomic/tcga_cbio/8_mutation_data.csv --clinical /data/genomic/tcga_cbio/8_clinical_data.csv --genomic /data/genomic/tcga_cbio/8_mutation_data.csv --output /data/
+
+script:
+  - bash ~/build/ldsec/medco-deployment/resources/utility-scripts/test_e2e.sh $MEDCO_USER_USERNAME $MEDCO_USER_PASSWORD

README.md

@@ -1,7 +1,35 @@
-## Documentation
-MedCo documentation is centralized on the following website: 
-[MedCo Deployment](https://medco.epfl.ch/documentation/developer/components/medco-deployment.html).
+## medco-deployment
+*medco-deployment* contains the scripts to deploy MedCo in different scenarios.
+
+A description of the available deployment profiles, along with a detailed guide on how to use them, is available [here](https://ldsec.gitbook.io/medco-documentation/system-administrator-guide/deployment).
+
+## Source code organization
+- *compose-profiles*: docker-compose files and parameters for different deployment profiles
+    - *[dev-local-3nodes](https://ldsec.gitbook.io/medco-documentation/system-administrator-guide/deployment/local-development-deployment)*: profile that deploys 3 MedCo nodes on a single machine for development purposes
+    - *[test-local-3nodes](https://ldsec.gitbook.io/medco-documentation/system-administrator-guide/deployment/local-test-deployment)*: profile that deploys 3 MedCo nodes on a single machine for test purposes
+- *configuration-profiles*: configuration files for the different deployment profiles (cryptographic keys, certificates, etc.)
+    - *dev-local-3nodes*: configuration files for the *dev-local-3nodes* profile
+    - *test-local-3nodes*: configuration files for the *test-local-3nodes* profile
+- *docker-images*: configuration files for the docker images that are used in the different deployment profiles
+    - *i2b2*: configuration files for 12b2
+    - *keycloak*: configuration files for keycloak
+    - *nginx*: configuration files for nginx
+    - *pgadmin*: configuration files for pgadmin
+    - *postgresql*: configuration files for postgresql
+- *resources*: additional configuration and utility files
+    - *configuration*: keycloak configuration files
+    - *data*: script to download the test datasets
+    - *profile-generation-scripts*: scripts to generate various deployment profiles files
+        - *test-network*: scripts to generate the deployment profiles files for the [Network Test Deployment](https://ldsec.gitbook.io/medco-documentation/system-administrator-guide/deployment/network-test-deployment) profile
+    - *utility-scripts*: additional utility scripts
+
+## Useful information
+*medco-deployment* is part of the MedCo system.
+
+You can find more information about the MedCo project [here](https://medco.epfl.ch/).
+
+For further details, support, and contacts, you can check the [MedCo Technical Documentation](https://ldsec.gitbook.io/medco-documentation/).
 
 ## License
-*medco-deployment* is licensed under a End User Software License Agreement ('EULA') for non-commercial use. 
+*medco-deployment* is licensed under a End User Software License Agreement ('EULA') for non-commercial use.
 If you need more information, please contact us.

compose-profiles/dev-local-3nodes/.env

@@ -1,9 +1,7 @@
 I2B2_VERSION=dev
 MEDCO_UNLYNX_VERSION=dev
 NGINX_VERSION=dev
-PHP_FPM_VERSION=dev
 PGADMIN_VERSION=dev
 KEYCLOAK_VERSION=dev
-PICSURE_VERSION=dev
 MEDCO_CONNECTOR_VERSION=dev
-MEDCO_LOADER_VERSION=dev
+MEDCO_LOADER_VERSION=dev

compose-profiles/dev-local-3nodes/docker-compose.tools.yml

@@ -13,7 +13,8 @@ services:
       service: medco-loader
     environment:
       - LOG_LEVEL=5
-      - DB_NAME=i2b2medcosrv0
+      - I2B2_DB_NAME=i2b2medcosrv0
+      - GA_DB_NAME=gamedcosrv0
       - UNLYNX_GROUP_FILE_IDX=0
 
   medco-loader-srv1:
@@ -22,7 +23,8 @@ services:
       service: medco-loader
     environment:
       - LOG_LEVEL=5
-      - DB_NAME=i2b2medcosrv1
+      - I2B2_DB_NAME=i2b2medcosrv1
+      - GA_DB_NAME=gamedcosrv1
       - UNLYNX_GROUP_FILE_IDX=1
 
   medco-loader-srv2:
@@ -31,5 +33,6 @@ services:
       service: medco-loader
     environment:
       - LOG_LEVEL=5
-      - DB_NAME=i2b2medcosrv2
+      - I2B2_DB_NAME=i2b2medcosrv2
+      - GA_DB_NAME=gamedcosrv2
       - UNLYNX_GROUP_FILE_IDX=2

compose-profiles/dev-local-3nodes/docker-compose.yml

@@ -44,7 +44,12 @@ services:
       - I2B2_LOGIN_USER=e2etest
       - I2B2_LOGIN_PASSWORD=e2etest
       - LOG_LEVEL=5
-      - UNLYNX_GROUP_FILE_IDX=0
+      - MEDCO_NODE_IDX=0
+      - GA_DB_NAME=gamedcosrv0
+      - OIDC_JWKS_URLS=http://keycloak:8080/auth/realms/master/protocol/openid-connect/certs,https://medco-demo.epfl.ch/auth/realms/master/protocol/openid-connect/certs
+      - OIDC_JWT_ISSUERS=http://localhost/auth/realms/master,https://medco-demo.epfl.ch/auth/realms/master
+      - OIDC_CLIENT_IDS=medco,medco
+      - OIDC_JWT_USER_ID_CLAIMS=preferred_username,preferred_username
     networks:
       intra-node-srv0:
       inter-nodes:
@@ -94,7 +99,12 @@ services:
       - I2B2_LOGIN_USER=e2etest
       - I2B2_LOGIN_PASSWORD=e2etest
       - LOG_LEVEL=5
-      - UNLYNX_GROUP_FILE_IDX=1
+      - MEDCO_NODE_IDX=1
+      - GA_DB_NAME=gamedcosrv1
+      - OIDC_JWKS_URLS=http://keycloak:8080/auth/realms/master/protocol/openid-connect/certs,https://medco-demo.epfl.ch/auth/realms/master/protocol/openid-connect/certs
+      - OIDC_JWT_ISSUERS=http://localhost/auth/realms/master,https://medco-demo.epfl.ch/auth/realms/master
+      - OIDC_CLIENT_IDS=medco,medco
+      - OIDC_JWT_USER_ID_CLAIMS=preferred_username,preferred_username
     networks:
       intra-node-srv1:
       inter-nodes:
@@ -144,7 +154,12 @@ services:
       - I2B2_LOGIN_USER=e2etest
       - I2B2_LOGIN_PASSWORD=e2etest
       - LOG_LEVEL=5
-      - UNLYNX_GROUP_FILE_IDX=2
+      - MEDCO_NODE_IDX=2
+      - GA_DB_NAME=gamedcosrv2
+      - OIDC_JWKS_URLS=http://keycloak:8080/auth/realms/master/protocol/openid-connect/certs,https://medco-demo.epfl.ch/auth/realms/master/protocol/openid-connect/certs
+      - OIDC_JWT_ISSUERS=http://localhost/auth/realms/master,https://medco-demo.epfl.ch/auth/realms/master
+      - OIDC_CLIENT_IDS=medco,medco
+      - OIDC_JWT_USER_ID_CLAIMS=preferred_username,preferred_username
     networks:
       intra-node-srv2:
       inter-nodes:
@@ -161,19 +176,6 @@ services:
       inter-nodes:
         ipv4_address: 172.31.0.200
 
-  php-fpm:
-    extends:
-      file: ../docker-compose-definitions.yml
-      service: php-fpm
-    environment:
-      - I2B2_DB_NAME=i2b2medcosrv0
-      - I2B2_DOMAIN_NAME=i2b2medcosrv0
-      - I2B2_URL=http://i2b2-medco-srv0:8080/i2b2/services
-    networks:
-      intra-node-srv0:
-      inter-nodes:
-        ipv4_address: 172.31.0.201
-
   postgresql:
     extends:
       file: ../docker-compose-definitions.yml
@@ -206,19 +208,6 @@ services:
       inter-nodes:
         ipv4_address: 172.31.0.204
 
-  picsure:
-    extends:
-      file: ../docker-compose-definitions.yml
-      service: picsure
-    ports:
-      - "8082:8080"
-      - "9992:9990"
-    depends_on:
-      - postgresql
-    networks:
-      inter-nodes:
-        ipv4_address: 172.31.0.205
-
 networks:
   inter-nodes:
     driver: bridge