diff --git a/Dockerfile b/Dockerfile index 8545b52..4aca4a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,6 @@ ENV MONGO_ELECTION_TIMEOUT 30 COPY etc/* /etc/ COPY bin/* /usr/local/bin/ -ENTRYPOINT ["containerpilot", "mongod"] +ENTRYPOINT ["containerpilot", "mongod", "--config", "/etc/mongod.conf"] -# Define CMD so the name of the replicaset can be overridden in the compose file CMD ["--replSet=joyent"] diff --git a/README.md b/README.md index c954c87..d22f666 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ A running cluster includes the following components: - [MongoDB](https://www.mongodb.com/community): we're using MongoDB 3.4 and setting up a [replica set](https://docs.mongodb.com/manual/replication/) - [Consul](https://www.consul.io/): used to coordinate replication and failover + ## Running the cluster Starting a new cluster is easy once you have [your `_env` file set with the configuration details](#configuration) @@ -25,6 +26,9 @@ In a few moments you'll have a running MongoDB ready for a replica set. Both the Pass these variables via an `_env` file. - `LOG_LEVEL`: control the amount of logging from ContainerPilot +- `MONGO_USER`: the user that will be created as the default admin user +- `MONGO_PASSWORD`: password to use for the admin user +- `MONGO_KEY`: secret key contents to use for replica member authentication - when the primary node is sent a `SIGTERM` it will [step down](https://docs.mongodb.com/manual/reference/command/replSetStepDown/) as primary; the following control those timeouts - `MONGO_SECONDARY_CATCHUP_PERIOD`: the number of seconds that the mongod will wait for an electable secondary to catch up to the primary - `MONGO_STEPDOWN_TIME`: the number of seconds to step down the primary, during which time the stepdown member is ineligible for becoming primary diff --git a/bin/manage.py b/bin/manage.py index f86896e..a9ff733 100644 --- a/bin/manage.py +++ b/bin/manage.py @@ -10,6 +10,7 @@ import time from functools import wraps +from urllib import quote_plus import consul as pyconsul #import manta @@ -65,6 +66,9 @@ def get_environ(key, default): SESSION_NAME = get_environ('SESSION_NAME', 'mongodb-replica-set-lock') SESSION_TTL = int(get_environ('SESSION_TTL', 60)) +MONGO_USER = get_environ('MONGO_USER', 'admin') +MONGO_PASSWORD = get_environ('MONGO_PASSWORD', 'admin') + # consts for node state PRIMARY = 'mongodb-replicaset' #SECONDARY = 'mongodb-secondary' @@ -99,8 +103,7 @@ def pre_stop(): because we are about to be shut down """ - ip = get_ip() - local_mongo = MongoClient(ip, connect=False) + local_mongo = get_local_mongo() # since we are shutting down, it is ok to stop if mongo is already non-responsive if not is_mongo_up(local_mongo): @@ -142,7 +145,7 @@ def pre_stop(): return False timeout += 1 # use a replica client so that we get "primary" data - mongo_client = MongoClient(ip, connect=False, replicaset=repl_status['set'], serverSelectionTimeoutMS=500) + mongo_client = MongoClient(get_local_mongo_uri(), connect=False, replicaset=repl_status['set'], serverSelectionTimeoutMS=500) # is_mongo_up will sleep on failure, so we don't need a "time.sleep(1)" if is_mongo_up(mongo_client, 1): primary = mongo_client.primary @@ -162,8 +165,7 @@ def health(): # TODO periodic mongodumps to Manta hostname = socket.gethostname() - ip = get_ip() - local_mongo = MongoClient(ip, connect=False) + local_mongo = get_local_mongo() # check that mongo is responsive if not is_mongo_up(local_mongo): @@ -212,7 +214,7 @@ def on_change(): ''' hostname = socket.gethostname() ip = get_ip() - local_mongo = MongoClient(ip, connect=False) + local_mongo = get_local_mongo() try: repl_status = local_mongo.admin.command('replSetGetStatus') @@ -313,6 +315,7 @@ def mongo_update_replset_config(local_mongo, hostname): for new_mongo in new_mongos: new_id = max(ids) + 1 ids.append(new_id) + members.append({'_id': new_id, 'host': new_mongo}) # TODO voting membership @@ -332,6 +335,19 @@ def mongo_update_replset_config(local_mongo, hostname): log.exception(e) sys.exit(1) + +def get_local_mongo_uri(): + ip = get_ip() + uri = "mongodb://%s:%s@%s" % (quote_plus(MONGO_USER), quote_plus(MONGO_PASSWORD), ip) + + return uri + +def get_local_mongo(): + local_mongo = MongoClient(get_local_mongo_uri(), connect=False) + + return local_mongo + + def consul_to_mongo_hostname(service): # if name.startswith(SECONDARY + '-'): # prefix = SECONDARY + '-' diff --git a/bin/setup_mongo.sh b/bin/setup_mongo.sh new file mode 100755 index 0000000..8652bac --- /dev/null +++ b/bin/setup_mongo.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + + +if [ -d "/data/db/_mongosetup" ]; then + echo "/data/db/_mongosetup exists, mongo already setup, exiting" + exit 0 +fi + +echo "Start MongoDB without access control and only for local connections" +mongod --fork --bind_ip 127.0.0.1 --logpath /dev/stdout + +echo "Create the user administrator." +# The createUser will error if the user already exists. +mongo admin --eval "db.createUser({ user: '${MONGO_USER}', pwd: '${MONGO_PASSWORD}', roles: [ { role: 'dbAdminAnyDatabase', db: 'admin' }, { role: 'clusterAdmin', db: 'admin' } ] });" + + +echo "Shutdown the MongoDB service." +mongod --shutdown + + +echo "Creating keyFile for replication." +echo -e ${MONGO_KEY} > /etc/mongod.key +chmod 400 /etc/mongod.key + +echo "Create directory to designate that setup is complete." +mkdir -p /data/db/_mongosetup diff --git a/etc/containerpilot.json b/etc/containerpilot.json index e957333..7e45f55 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -1,6 +1,6 @@ { "consul": "localhost:8500", - "preStart": "python /usr/local/bin/manage.py", + "preStart": "./usr/local/bin/setup_mongo.sh && python /usr/local/bin/manage.py", "preStop": "python /usr/local/bin/manage.py pre_stop", "services": [ { diff --git a/etc/mongod.conf b/etc/mongod.conf new file mode 100644 index 0000000..0768e88 --- /dev/null +++ b/etc/mongod.conf @@ -0,0 +1,3 @@ +security: + keyFile: "/etc/mongod.key" + authorization: "enabled" diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..656d7d5 --- /dev/null +++ b/setup.sh @@ -0,0 +1,217 @@ +#!/bin/bash +set -e -o pipefail + +help() { + echo + echo 'Usage ./setup.sh ~/path/to/MANTA_PRIVATE_KEY ~/path/to/MONGO_KEYFILE' + echo + echo 'Checks that your Triton and Docker environment is sane and configures' + echo 'an environment file to use.' + echo + echo 'MANTA_PRIVATE_KEY is the filesystem path to an SSH private key' + echo 'used to connect to Manta for the database backups.' + echo + echo 'MONGO_KEYFILE is the filesystem path to a file that contains a secret' + echo 'value between 6 and 1024 characters for authenticating replica members' + echo + echo 'Additional details must be configured in the _env file, but this script will properly' + echo 'encode the SSH key details for use with this MongoDB image.' + echo +} + + +# populated by `check` function whenever we're using Triton +TRITON_USER= +TRITON_DC= +TRITON_ACCOUNT= + +# --------------------------------------------------- +# Top-level commands + +# Check for correct configuration and setup _env file +envcheck() { + + if [ -z "$1" ]; then + tput rev # reverse + tput bold # bold + echo 'Please provide a path to a SSH private key to access Manta.' + tput sgr0 # clear + + help + exit 1 + fi + + if [ ! -f "$1" ]; then + tput rev # reverse + tput bold # bold + echo 'SSH private key for Manta is unreadable.' + tput sgr0 # clear + + help + exit 1 + fi + + if [ -z "$2" ]; then + tput rev # reverse + tput bold # bold + echo 'Please provide a path to a key file for MongoDB replica members.' + tput sgr0 # clear + + help + exit 1 + fi + + if [ ! -f "$2" ]; then + tput rev # reverse + tput bold # bold + echo 'MongoDB replica key file is unreadable.' + tput sgr0 # clear + + help + exit 1 + fi + + # Assign args to named vars + MANTA_PRIVATE_KEY_PATH=$1 + MONGO_KEYFILE_PATH=$2 + + command -v docker >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Docker is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://docs.joyent.com/public-cloud/api-access/docker' + exit 1 + } + command -v json >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Error! JSON CLI tool is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://apidocs.joyent.com/cloudapi/#getting-started' + exit 1 + } + + command -v triton >/dev/null 2>&1 || { + echo + tput rev # reverse + tput bold # bold + echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.' + tput sgr0 # clear + echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool' + exit 1 + } + + # make sure Docker client is pointed to the same place as the Triton client + local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') + local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') + TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') + TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') + TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') + if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then + echo + tput rev # reverse + tput bold # bold + echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' + tput sgr0 # clear + echo + echo "Docker user: ${docker_user}" + echo "Triton user: ${TRITON_USER}" + echo "Docker data center: ${docker_dc}" + echo "Triton data center: ${TRITON_DC}" + exit 1 + fi + + local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') + if [ ! "true" == "$triton_cns_enabled" ]; then + echo + tput rev # reverse + tput bold # bold + echo 'Error! Triton CNS is required and not enabled.' + tput sgr0 # clear + echo + exit 1 + fi + + # setup environment file + if [ ! -f "_env" ]; then + echo '# Environment variables for MongoDB service' > _env + echo 'MONGO_USER=dbuser' >> _env + echo 'MONGO_PASSWORD='$(cat /dev/urandom | LC_ALL=C tr -dc 'A-Za-z0-9' | head -c 7) >> _env + echo MONGO_KEY=$(cut -c1-100 ${MONGO_KEYFILE_PATH} | tr '\n' '1' | tr '-' '1') >> _env + echo >> _env + + echo '# Environment variables for backups to Manta' >> _env + echo 'MANTA_URL=https://us-east.manta.joyent.com' >> _env + echo 'MANTA_BUCKET= # an existing Manta bucket' >> _env + echo 'MANTA_USER= # a user with access to that bucket' >> _env + echo 'MANTA_SUBUSER=' >> _env + echo 'MANTA_ROLE=' >> _env + + # MANTA_KEY_ID must be the md5 formatted key fingerprint. A SHA256 will result in errors. + set +o pipefail + # The -E option was added to ssh-keygen recently; if it doesn't work, then + # assume we're using an older version of ssh-keygen that only outputs MD5 fingerprints + ssh-keygen -yl -E md5 -f ${MANTA_PRIVATE_KEY_PATH} > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo MANTA_KEY_ID=$(ssh-keygen -yl -E md5 -f ${MANTA_PRIVATE_KEY_PATH} | awk '{print substr($2,5)}') >> _env + else + echo MANTA_KEY_ID=$(ssh-keygen -yl -f ${MANTA_PRIVATE_KEY_PATH} | awk '{print $2}') >> _env + fi + set -o pipefail + + # munge the private key so that we can pass it into an env var sanely + # and then unmunge it in our startup script + echo MANTA_PRIVATE_KEY=$(cat ${MANTA_PRIVATE_KEY_PATH} | tr '\n' '#') >> _env + echo >> _env + + echo '# Consul discovery via Triton CNS' >> _env + echo CONSUL=mongodb-consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env + echo >> _env + + echo 'Edit the _env file with your desired MONGO_* and MANTA_* config' + else + echo 'Existing _env file found, exiting' + exit + fi +} + +get_root_password() { + echo $(docker logs ${COMPOSE_PROJECT_NAME:-mongodb}_mongodb_1 2>&1 | \ + awk '/Generated root password/{print $NF}' | \ + awk '{$1=$1};1' + ) | pbcopy +} + + + +# --------------------------------------------------- +# parse arguments + +# Get function list +funcs=($(declare -F -p | cut -d " " -f 3)) + +until + if [ ! -z "$1" ]; then + # check if the first arg is a function in this file, or use a default + if [[ " ${funcs[@]} " =~ " $1 " ]]; then + cmd=$1 + shift 1 + else + cmd="envcheck" + fi + + $cmd "$@" + if [ $? == 127 ]; then + help + fi + + exit + else + help + fi +do + echo +done