Merge branch 'develop', prepare 3.3.0

Author: yamalight

src/db/secrets.js

@@ -0,0 +1,34 @@
+// npm packages
+const path = require('path');
+const Loki = require('lokijs');
+
+// our packages
+const {baseFolder} = require('../config');
+
+// init in-memory adapter for login requests
+const memAdapter = new Loki.LokiMemoryAdapter();
+const fsAdapter = new Loki.LokiFsAdapter();
+
+// init persistent secrets db
+let secretsCollection = {};
+let secretResolve = () => {};
+const secretsInited = new Promise(r => {
+  secretResolve = r;
+});
+const secretDb = new Loki(path.join(baseFolder, 'secrets.db'), {
+  adapter: process.env.NODE_ENV !== 'testing' ? fsAdapter : memAdapter,
+  autoload: true,
+  autoloadCallback() {
+    // get of create secrets collection
+    secretsCollection = secretDb.getCollection('secrets');
+    if (secretsCollection === null) {
+      secretsCollection = secretDb.addCollection('secrets');
+    }
+    secretResolve();
+  },
+  autosave: process.env.NODE_ENV !== 'testing',
+});
+
+exports.secretDb = secretDb;
+exports.getSecretsCollection = () => secretsCollection;
+exports.secretsInited = secretsInited;

src/docker/start.js

@@ -2,8 +2,18 @@
 const docker = require('./docker');
 const {initNetwork, createNetwork} = require('../docker/network');
 const {getProjectConfig, nameFromImage, projectFromConfig, writeStatus} = require('../util');
+const {getSecretsCollection} = require('../db/secrets');
 const {getConfig} = require('../config');
 
+// try to find secret with current value name and return secret value if present
+const valueOrSecret = (value, secrets) => {
+  const secret = secrets.find(s => `@${s.name}` === value);
+  if (secret) {
+    return secret.value;
+  }
+  return value;
+};
+
 exports.startFromParams = async ({
   image,
   deploymentName,
@@ -181,8 +191,10 @@ exports.start = async ({image, username, resultStream, existing = []}) => {
   // construct host
   const host = config.domain === undefined ? defaultDomain : config.domain;
 
-  // generate env vars
-  const Env = config.env ? Object.keys(config.env).map(key => `${key}=${config.env[key]}`) : [];
+  // replace env vars values with secrets if needed
+  const secrets = getSecretsCollection().find({user: username});
+  // generate env vars (with secrets)
+  const Env = config.env ? Object.keys(config.env).map(key => `${key}=${valueOrSecret(config.env[key], secrets)}`) : [];
 
   // generate project name
   const project = projectFromConfig({username, config});
@@ -252,7 +264,7 @@ exports.start = async ({image, username, resultStream, existing = []}) => {
 
   // if basic auth is set - add it to config
   if (config.basicAuth && config.basicAuth.length) {
-    Labels['traefik.frontend.auth.basic.users'] = config.basicAuth
+    Labels['traefik.frontend.auth.basic.users'] = config.basicAuth;
   }
 
   // if running in swarm mode - run traefik as swarm service

src/routes/index.js

@@ -7,6 +7,7 @@ const update = require('./update');
 const version = require('./version');
 const templates = require('./templates');
 const setup = require('./setup');
+const secrets = require('./secrets');
 
 module.exports = (fastify, opts, next) => {
   // enable auth for all routes
@@ -20,6 +21,7 @@ module.exports = (fastify, opts, next) => {
   version(fastify);
   templates(fastify);
   setup(fastify);
+  secrets(fastify);
 
   next();
 };

src/routes/secrets.js

@@ -0,0 +1,62 @@
+// our modules
+const {secretsInited, getSecretsCollection} = require('../db/secrets');
+
+module.exports = fastify => {
+  fastify.route({
+    method: 'GET',
+    path: '/secrets',
+    async handler(request, reply) {
+      // get username
+      const {username} = request.user;
+
+      // wait for db to init if required
+      await secretsInited;
+      // find user secrets
+      const secrets = getSecretsCollection()
+        .find({user: username})
+        .map(({value, ...s}) => s);
+
+      reply.send({secrets});
+    },
+  });
+
+  fastify.route({
+    method: 'POST',
+    path: '/secrets',
+    async handler(request, reply) {
+      // get username
+      const {username} = request.user;
+      // get secret data
+      const {secretName, secretValue} = request.body;
+
+      // wait for db to init if required
+      await secretsInited;
+      // create new secret for current user
+      const secret = {user: username, name: secretName, value: secretValue};
+      getSecretsCollection().insert(secret);
+
+      reply.send(secret);
+    },
+  });
+
+  fastify.route({
+    method: 'DELETE',
+    path: '/secrets',
+    async handler(request, reply) {
+      // generate new deploy token
+      const {secretName} = request.body;
+      const {user} = request;
+      const existingSecret = getSecretsCollection().findOne({user: user.username, name: secretName});
+      if (!existingSecret) {
+        reply.code(200).send({removed: false, reason: 'Secret does not exist'});
+        return;
+      }
+      // wait for db to init if required
+      await secretsInited;
+      // remove token from collection
+      getSecretsCollection().remove(existingSecret);
+      // send back to user
+      reply.code(204).send();
+    },
+  });
+};

test/deploy.test.js

@@ -95,7 +95,6 @@ test('Should deploy simple docker project', async done => {
 
   const containerData = docker.getContainer(containerInfo.Id);
   const container = await containerData.inspect();
-  // console.log(JSON.stringify(container));
   expect(container.NetworkSettings.Networks.exoframe.Aliases.includes('test')).toBeTruthy();
   expect(container.HostConfig.RestartPolicy).toMatchObject({Name: 'no', MaximumRetryCount: 0});
 

test/fixtures/secrets-project/Dockerfile

@@ -0,0 +1,3 @@
+FROM busybox
+
+CMD ["sleep", "300"]

test/fixtures/secrets-project/exoframe.json

@@ -0,0 +1,9 @@
+{
+  "name": "test-secrets-deploy",
+  "restart": "no",
+  "project": "secrets-project",
+  "env": {
+    "test": "@test-secret"
+  },
+  "hostname": "secrets"
+}

test/secrets.test.js

@@ -0,0 +1,157 @@
+/* eslint-env jest */
+// mock config for testing
+jest.mock('../src/config', () => require('./__mocks__/config'));
+const config = require('../src/config');
+// switch config to normal
+config.__load('normal');
+
+// npm packages
+const getPort = require('get-port');
+const path = require('path');
+const tar = require('tar-fs');
+
+// our packages
+const authToken = require('./fixtures/authToken');
+const {startServer} = require('../src');
+const {getSecretsCollection} = require('../src/db/secrets');
+const docker = require('../src/docker/docker');
+
+// create tar streams
+const streamDocker = tar.pack(path.join(__dirname, 'fixtures', 'secrets-project'));
+
+// test secret
+const testSecret = {
+  secretName: 'test-secret',
+  secretValue: 'test-secret-value',
+};
+
+// container vars
+let fastify;
+
+// set timeout to 60s
+jest.setTimeout(60000);
+
+beforeAll(async () => {
+  // start server
+  const port = await getPort();
+  fastify = await startServer(port);
+  return fastify;
+});
+
+afterAll(() => fastify.close());
+
+test('Should create new secret', async done => {
+  // options base
+  const options = {
+    method: 'POST',
+    url: '/secrets',
+    headers: {
+      Authorization: `Bearer ${authToken}`,
+    },
+    payload: testSecret,
+  };
+
+  const response = await fastify.inject(options);
+  const result = JSON.parse(response.payload);
+
+  // check response
+  expect(response.statusCode).toEqual(200);
+  expect(result.name).toEqual(testSecret.secretName);
+  expect(result.value).toEqual(testSecret.secretValue);
+  expect(result.user).toEqual('admin');
+
+  done();
+});
+
+test('Should get list with new secret', async done => {
+  // options base
+  const options = {
+    method: 'GET',
+    url: '/secrets',
+    headers: {
+      Authorization: `Bearer ${authToken}`,
+    },
+  };
+
+  const response = await fastify.inject(options);
+  const result = JSON.parse(response.payload);
+
+  // check response
+  expect(response.statusCode).toEqual(200);
+  expect(result.secrets).toBeDefined();
+  expect(result.secrets.length).toEqual(1);
+  expect(result.secrets[0].user).toEqual('admin');
+  expect(result.secrets[0].name).toEqual(testSecret.secretName);
+  expect(result.secrets[0].value).toBeUndefined();
+
+  done();
+});
+
+test('Should deploy simple docker project with secret', async done => {
+  const options = {
+    method: 'POST',
+    url: '/deploy',
+    headers: {
+      Authorization: `Bearer ${authToken}`,
+      'Content-Type': 'application/octet-stream',
+    },
+    payload: streamDocker,
+  };
+
+  const response = await fastify.inject(options);
+  // parse result into lines
+  const result = response.payload
+    .split('\n')
+    .filter(l => l && l.length)
+    .map(line => JSON.parse(line));
+
+  // find deployments
+  const completeDeployments = result.find(it => it.deployments && it.deployments.length).deployments;
+
+  // check response
+  expect(response.statusCode).toEqual(200);
+  expect(completeDeployments.length).toEqual(1);
+  expect(completeDeployments[0].Name.startsWith('/exo-admin-test-secrets-deploy-')).toBeTruthy();
+
+  // check docker services
+  const allContainers = await docker.listContainers();
+  const containerInfo = allContainers.find(c => c.Names.includes(completeDeployments[0].Name));
+  expect(containerInfo).toBeDefined();
+
+  const containerData = docker.getContainer(containerInfo.Id);
+  const container = await containerData.inspect();
+
+  // check secrets replacement in env vars
+  const [key, value] = container.Config.Env.map(v => v.split('=')).find(([key]) => key === 'test');
+  expect(key).toEqual('test');
+  expect(value).toEqual(testSecret.secretValue);
+
+  // cleanup
+  const instance = docker.getContainer(containerInfo.Id);
+  await instance.remove({force: true});
+
+  done();
+});
+
+test('Should delete new secret', async done => {
+  // options base
+  const options = {
+    method: 'DELETE',
+    url: '/secrets',
+    headers: {
+      Authorization: `Bearer ${authToken}`,
+    },
+    payload: {
+      secretName: testSecret.secretName,
+    },
+  };
+
+  // check response
+  const response = await fastify.inject(options);
+  expect(response.statusCode).toEqual(204);
+
+  // make sure it's no longer in db
+  expect(getSecretsCollection().find()).toEqual([]);
+
+  done();
+});