datasets: initial

Author: tharvik

.github/workflows/lint-test-build.yml

@@ -6,18 +6,21 @@ on:
 
 env:
   node_version: 16
+  datasets-cache-key: datasets-${{ hashFiles('datasets/**') }}
 
 jobs:
-  download-training-data:
+  download-datasets:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
-      - run: ./get_training_data.sh
-        working-directory: ./
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
+      - run: ./datasets/populate
 
   lint-lib-core:
     needs: [build-lib-core, build-lib-node]
@@ -213,19 +216,17 @@ jobs:
 
   test-lib-core:
     needs:
-      [
-        build-lib-core,
-        build-lib-node,
-        build-server-docker,
-        download-training-data,
-      ]
+      [build-lib-core, build-lib-node, build-server-docker, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}
@@ -235,14 +236,17 @@ jobs:
       - run: ./with_server npm --workspace=./discojs/discojs-core test
 
   test-lib-node:
-    needs: [build-lib-core, build-server-docker, download-training-data]
+    needs: [build-lib-core, build-server-docker, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}
@@ -252,14 +256,17 @@ jobs:
       - run: ./with_server npm --workspace=./discojs/discojs-node test
 
   test-lib-web:
-    needs: [build-lib-core, build-server-docker, download-training-data]
+    needs: [build-lib-core, build-server-docker, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}
@@ -269,14 +276,17 @@ jobs:
       - run: ./with_server npm --workspace=./discojs/discojs-web test
 
   test-server:
-    needs: [build-lib-core, build-lib-node, download-training-data]
+    needs: [build-lib-core, build-lib-node, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}
@@ -286,14 +296,17 @@ jobs:
       - run: npm --workspace=./server test
 
   test-web-client:
-    needs: [build-lib-core, build-lib-web, download-training-data]
+    needs: [build-lib-core, build-lib-web, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}
@@ -310,15 +323,17 @@ jobs:
           config: baseUrl=http://localhost:8081/#/
 
   test-cli:
-    needs:
-      [build-lib-core, build-lib-node, build-server, download-training-data]
+    needs: [build-lib-core, build-lib-node, build-server, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}
@@ -328,15 +343,17 @@ jobs:
       - run: npm --workspace=./cli start -- -t cifar10 -u 1 -e 1
 
   test-docs-examples:
-    needs:
-      [build-lib-core, build-lib-node, build-server, download-training-data]
+    needs: [build-lib-core, build-lib-node, build-server, download-datasets]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: true
+          submodules: true
       - uses: actions/cache@v3
         with:
-          path: example_training_data
-          key: training_data
+          path: datasets
+          key: ${{ env.datasets-cache-key }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.node_version }}

.gitignore

@@ -1,150 +1,11 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
+# dependencies
+/node_modules/
 
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
+# built
 dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
 
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-
-UI/public/.DS_Store
-UI/.DS_Store
-
-*.DS_Store
-
-node_modules/
-
-# model files on server
-weights.bin
-model.json
-
-# example training data
-example_training_data/
-example_training_data.tar.gz
-
-# IDE files
+# system specifics files
 .metals/
 .idea/
 .vscode/
+*.DS_Store

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "datasets/wikitext"]
+	path = datasets/wikitext
+	url = https://huggingface.co/datasets/wikitext

DEV.md

@@ -75,8 +75,13 @@ npm --version
 
 **3.** Clone the repository
 
+As we are using some huge files in the repository, you'll need to install [`git-lfs`](https://git-lfs.com).
+Be sure to run `git lfs install` after installing it.
+
+We are using git submodules, so we need to also clone theses.
+
 ```
-git clone git@github.com:epfml/disco.git
+git clone --recurse-submodules git@github.com:epfml/disco.git
 cd disco
 ```
 
@@ -101,7 +106,7 @@ npm -ws run build
 **6.** Download and extract the sample training datasets. These datasets are used in the automated tests.
 
 ```
-./get_training_data.sh
+./datasets/populate
 ```
 
 **7.** Launch DISCO

cli/src/data.ts

@@ -12,7 +12,7 @@ function filesFromFolder (dir: string, folder: string, fractionToKeep: number):
 }
 
 async function simplefaceData (task: Task): Promise<data.DataSplit> {
-  const dir = '../example_training_data/simple_face/'
+  const dir = '../datasets/simple_face/'
   const youngFolders = ['child']
   const oldFolders = ['adult']
 
@@ -39,15 +39,15 @@ async function simplefaceData (task: Task): Promise<data.DataSplit> {
 }
 
 async function cifar10Data (cifar10: Task): Promise<data.DataSplit> {
-  const dir = '../example_training_data/CIFAR10/'
+  const dir = '../datasets/CIFAR10/'
   const files = (await fs_promises.readdir(dir)).map((file) => path.join(dir, file))
   const labels = Range(0, 24).map((label) => (label % 10).toString()).toArray()
 
   return await new NodeImageLoader(cifar10).loadAll(files, { labels })
 }
 
 async function titanicData (titanic: Task): Promise<data.DataSplit> {
-  const dir = '../example_training_data/titanic_train.csv'
+  const dir = '../datasets/titanic_train.csv'
 
   const data = await (new NodeTabularLoader(titanic, ',').loadAll(
     ['file://'.concat(dir)],