diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d4921b0..9ed4b9f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +### citustools v0.6.4 (August 15, 2017) ### + +* Bumps Homebrew formula + +* Adds automation script for valgrind tests + +* Starts to use REL_10_STABLE branch for PostgreSQL tests + ### citustools v0.6.3 (July 11, 2017) ### * Addresses citus-style compatibility issues with uncrustify 0.65 diff --git a/HomebrewFormula/citustools.rb b/HomebrewFormula/citustools.rb index 3e00e603..592e2ac4 100644 --- a/HomebrewFormula/citustools.rb +++ b/HomebrewFormula/citustools.rb @@ -12,8 +12,8 @@ def message class Citustools < Formula desc "Tools and config used in Citus Data projects." homepage "https://github.com/citusdata/tools" - url "https://github.com/citusdata/tools/archive/v0.6.0.tar.gz" - sha256 "85644f4910e17ed378748d930cf86e98b9316243467c8a6f009de8001a5bdbff" + url "https://github.com/citusdata/tools/archive/v0.6.3.tar.gz" + sha256 "5e2dc61136be940f436911ad17908bb48d8c6362ed8407d32c405b5a8e982d72" depends_on "uncrustify" depends_on Docker diff --git a/Makefile b/Makefile index ed258f62..bba21aaa 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export mandir := $(datarootdir)/man export sysconfdir := $(prefix)/etc export pkgsysconfdir := $(sysconfdir)/$(PACKAGE_NAME) -DIRNAMES = packaging uncrustify +DIRNAMES = packaging uncrustify valgrind ifeq ($(TRAVIS), true) DIRNAMES += travis endif diff --git a/travis/install_custom_pg b/travis/install_custom_pg index 2c37f662..0d78a3f3 100755 --- a/travis/install_custom_pg +++ b/travis/install_custom_pg @@ -16,10 +16,12 @@ cd ~ if [ "$(ls -A postgresql)" ]; then git -C postgresql pull else - if [ "${PGVERSION}" -eq '10' ]; then - gitref='master' + if [ "${PGVERSION}" == '11' ]; then + gitref="master" + elif [ "${PGVERSION}" == '9.6' ]; then + gitref="REL9_6_STABLE" else - gitref="REL${PGVERSION//./_}_STABLE" + gitref="REL_${PGVERSION}_STABLE" fi git clone -b "${gitref}" --depth 1 git://git.postgresql.org/git/postgresql.git diff --git a/valgrind/Makefile b/valgrind/Makefile new file mode 100644 index 00000000..9c9bd08d --- /dev/null +++ b/valgrind/Makefile @@ -0,0 +1,15 @@ +# needed variables will be passed in via top-level Makefile + +INSTALL := install -c +INSTALL_SCRIPT := $(INSTALL) -m 755 +VALGRIND_SCRIPTS := $(filter-out README.md Makefile,$(wildcard *)) + +all: + +clean: + rm -f $(VALGRIND_SCRIPTS) + +install: all + $(INSTALL_SCRIPT) $(VALGRIND_SCRIPTS) $(DESTDIR)$(bindir) + +.PHONY: clean installdirs install diff --git a/valgrind/README.md b/valgrind/README.md new file mode 100644 index 00000000..e2d26ca7 --- /dev/null +++ b/valgrind/README.md @@ -0,0 +1,13 @@ +# valgrind-test-automation + +Tools for automating Valgrind tests with Citus. This automation suite will open an instance in AWS and runs Citus with Valgrind. When tests are completed, it will send a mail to burak@citusdata.com and metin@citusdata.com with the valgrind logs. Logs will contain memory related problems and the call stack where each problem is found. If there are no problems, no report will be attached. Only a success message will be sent. + +# Usage + +You first need to install and configure aws command line client. Refer [here](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) to see how to install aws command line client. After installation, configure it with `aws configure` command. Then to use aws instance for valgrind tests, run; + +```sh +./launch-test-instance.sh +``` + +This command will create a special key pair and start an m3.xlarge instance with that key pair. Created instance will be tagged with Name=ValgrindTest. Then, scripts will build PostgreSQL and Citus from source and runs Citus' regression tests with Valgrind. After tests are completed, valgrind logs will be sent to burak@citusdata.com and metin@citusdata.com for now and instance will terminate itself. It is expected that the tests will take about 5 hours to complete. diff --git a/valgrind/download-test-scripts b/valgrind/download-test-scripts new file mode 100755 index 00000000..38cb6543 --- /dev/null +++ b/valgrind/download-test-scripts @@ -0,0 +1,16 @@ +#!/bin/bash + +# we send this script to aws while creating the instance, this script will run +# as final step of the instance creation and download necessary scripts to run +# valgrind tests from our repository. + +set -euo pipefail + +# download and install required packages +apt-get update +apt-get install git make -y + +# download the test scripts +git clone https://github.com/citusdata/tools.git +cd tools +make install diff --git a/valgrind/launch-test-instance b/valgrind/launch-test-instance new file mode 100755 index 00000000..7a996d4e --- /dev/null +++ b/valgrind/launch-test-instance @@ -0,0 +1,47 @@ +#!/bin/bash + +set -euo pipefail + +# create a key pair just for valgrind tests and store it in valgrind-test.pem +echo "Creating key pair..." +key_name=valgrind_$RANDOM +aws ec2 create-key-pair --key-name $key_name --query 'KeyMaterial' --output text > $key_name.pem +chmod 600 $key_name.pem + +# start an instance with ami-f4cc1de2 image(i.e. Ubuntu Xenial 16.04) +echo "Starting an instance..." +valgrind_instance_id=$(aws ec2 run-instances \ + --image-id ami-f4cc1de2 \ + --count 1 \ + --instance-type m3.xlarge \ + --key-name $key_name \ + --instance-initiated-shutdown-behavior terminate \ + --user-data file://download-test-scripts \ + --query 'Instances[0].InstanceId' \ + --output text) + +# tag the instance as ValgrindTest +echo "Tagging the instance..." +aws ec2 create-tags \ + --resources "${valgrind_instance_id}" \ + --tags Key=Name,Value=ValgrindTest + +# wait for instance creation complete +echo "Waiting for completion of instance creation... (This may take a several minutes)" +aws ec2 wait instance-status-ok \ + --instance-id "${valgrind_instance_id}" + +# get the instance ip address +echo "Getting the ip address of the instance..." +valgrind_instance_ip=$(aws ec2 describe-instances \ + --instance-ids "${valgrind_instance_id}" \ + --query 'Reservations[0].Instances[0].PublicIpAddress' \ + --output text) + +# run valgrind tests +echo "Running the valgrind tests..." +echo "This will take hours, test results will be sent via e-mail." +ssh -o StrictHostKeyChecking=no -i $key_name.pem ubuntu@$valgrind_instance_ip "screen -d -m run-valgrind-tests" + +#delete the key pair after we are done with tests +aws ec2 delete-key-pair --key-name $key_name diff --git a/valgrind/run-valgrind-tests b/valgrind/run-valgrind-tests new file mode 100755 index 00000000..a379b5bf --- /dev/null +++ b/valgrind/run-valgrind-tests @@ -0,0 +1,63 @@ +#!/bin/bash + +set -euo pipefail + +# download and install required packages +sudo apt-get update +sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \ + build-essential \ + libreadline6 \ + libreadline6-dev \ + zlib1g-dev \ + flex \ + bison \ + libssl-dev \ + valgrind \ + mailutils + +# set environment variables +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 +export LANGUAGE=en_US.UTF-8 +export PG_CONFIG=/usr/local/pgsql/bin/pg_config + +# download and install PostgreSQL +git clone -b "REL9_6_STABLE" --depth 1 git://git.postgresql.org/git/postgresql.git +cd postgresql/ +./configure --enable-cassert --enable-debug CFLAGS="-ggdb -Og -DUSE_VALGRIND" + +# we will use this to parallelize PostgreSQL compilation +procs="$(nproc)" +mjobs="$((procs + 1))" +make -j "${mjobs}" -s +sudo make install +export PATH=/usr/local/pgsql/bin:$PATH + +# download and install Citus +cd .. +git clone https://github.com/citusdata/citus.git +cd citus/ +./configure +make clean +make -j8 -s +sudo make install + +# this is necessary to start tests +sudo chown ubuntu /usr/local/pgsql/bin/ -R + +# run valgrind tests +cd src/test/regress +make check-multi-vg VALGRIND_LOG_FILE=logs.txt || true + +# surprisingly this hits inbox +if [ -s logs.txt ]; then + mail -aFrom:valgrind-test@citusdata.com -s "[Valgrind Test Results] - Failure" -A logs.txt burak@citusdata.com metin@citusdata.com < /dev/null +else + mail -aFrom:valgrind-test@citusdata.com -s "[Valgrind Test Results] - Success" burak@citusdata.com metin@citusdata.com < /dev/null +fi + +# just to ensure everything is completed in the test instance +sleep 30 + +# shut-down the instance, this will also terminate the instance because we set instance-initiated-shutdown-behavior to terminate +sudo shutdown -h now