Merge branch 'release/0.6.4'

CHANGELOG.md

@@ -1,3 +1,11 @@
+### citustools v0.6.4 (August 15, 2017) ###
+
+* Bumps Homebrew formula
+
+* Adds automation script for valgrind tests
+
+* Starts to use REL_10_STABLE branch for PostgreSQL tests
+
 ### citustools v0.6.3 (July 11, 2017) ###
 
 * Addresses citus-style compatibility issues with uncrustify 0.65

HomebrewFormula/citustools.rb

@@ -12,8 +12,8 @@ def message
 class Citustools < Formula
   desc "Tools and config used in Citus Data projects."
   homepage "https://github.com/citusdata/tools"
-  url "https://github.com/citusdata/tools/archive/v0.6.0.tar.gz"
-  sha256 "85644f4910e17ed378748d930cf86e98b9316243467c8a6f009de8001a5bdbff"
+  url "https://github.com/citusdata/tools/archive/v0.6.3.tar.gz"
+  sha256 "5e2dc61136be940f436911ad17908bb48d8c6362ed8407d32c405b5a8e982d72"
 
   depends_on "uncrustify"
   depends_on Docker

Makefile

@@ -7,7 +7,7 @@ export mandir := $(datarootdir)/man
 export sysconfdir := $(prefix)/etc
 export pkgsysconfdir := $(sysconfdir)/$(PACKAGE_NAME)
 
-DIRNAMES = packaging uncrustify
+DIRNAMES = packaging uncrustify valgrind
 ifeq ($(TRAVIS), true)
 	DIRNAMES += travis
 endif

travis/install_custom_pg

@@ -16,10 +16,12 @@ cd ~
 if [ "$(ls -A postgresql)" ]; then
   git -C postgresql pull
 else
-  if [ "${PGVERSION}" -eq '10' ]; then
-    gitref='master'
+  if [ "${PGVERSION}" == '11' ]; then
+    gitref="master"
+  elif [ "${PGVERSION}" == '9.6' ]; then
+    gitref="REL9_6_STABLE"
   else
-    gitref="REL${PGVERSION//./_}_STABLE"
+    gitref="REL_${PGVERSION}_STABLE"
   fi
 
   git clone -b "${gitref}" --depth 1 git://git.postgresql.org/git/postgresql.git

valgrind/Makefile

@@ -0,0 +1,15 @@
+# needed variables will be passed in via top-level Makefile
+
+INSTALL := install -c
+INSTALL_SCRIPT := $(INSTALL) -m 755
+VALGRIND_SCRIPTS := $(filter-out README.md Makefile,$(wildcard *))
+
+all:
+
+clean:
+	rm -f $(VALGRIND_SCRIPTS)
+
+install: all
+	$(INSTALL_SCRIPT) $(VALGRIND_SCRIPTS) $(DESTDIR)$(bindir)
+
+.PHONY: clean installdirs install

valgrind/README.md

@@ -0,0 +1,13 @@
+# valgrind-test-automation
+
+Tools for automating Valgrind tests with Citus. This automation suite will open an instance in AWS and runs Citus with Valgrind. When tests are completed, it will send a mail to burak@citusdata.com and metin@citusdata.com with the valgrind logs. Logs will contain memory related problems and the call stack where each problem is found. If there are no problems, no report will be attached. Only a success message will be sent.
+
+# Usage
+
+You first need to install and configure aws command line client. Refer [here](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) to see how to install aws command line client. After installation, configure it with `aws configure` command. Then to use aws instance for valgrind tests, run;
+
+```sh
+./launch-test-instance.sh
+```
+
+This command will create a special key pair and start an m3.xlarge instance with that key pair. Created instance will be tagged with Name=ValgrindTest. Then, scripts will build PostgreSQL and Citus from source and runs Citus' regression tests with Valgrind. After tests are completed, valgrind logs will be sent to burak@citusdata.com and metin@citusdata.com for now and instance will terminate itself. It is expected that the tests will take about 5 hours to complete.

valgrind/download-test-scripts

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# we send this script to aws while creating the instance, this script will run
+# as final step of the instance creation and download necessary scripts to run
+# valgrind tests from our repository.
+
+set -euo pipefail
+
+# download and install required packages
+apt-get update
+apt-get install git make -y
+
+# download the test scripts
+git clone https://github.com/citusdata/tools.git
+cd tools
+make install

valgrind/launch-test-instance

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# create a key pair just for valgrind tests and store it in valgrind-test.pem
+echo "Creating key pair..."
+key_name=valgrind_$RANDOM
+aws ec2 create-key-pair --key-name $key_name --query 'KeyMaterial' --output text > $key_name.pem
+chmod 600 $key_name.pem
+
+# start an instance with ami-f4cc1de2 image(i.e. Ubuntu Xenial 16.04)
+echo "Starting an instance..."
+valgrind_instance_id=$(aws ec2 run-instances \
+    --image-id ami-f4cc1de2 \
+    --count 1 \
+    --instance-type m3.xlarge \
+    --key-name $key_name \
+    --instance-initiated-shutdown-behavior terminate \
+    --user-data file://download-test-scripts \
+    --query 'Instances[0].InstanceId' \
+    --output text)
+
+# tag the instance as ValgrindTest
+echo "Tagging the instance..."
+aws ec2 create-tags \
+    --resources "${valgrind_instance_id}" \
+    --tags Key=Name,Value=ValgrindTest
+
+# wait for instance creation complete
+echo "Waiting for completion of instance creation... (This may take a several minutes)"
+aws ec2 wait instance-status-ok \
+    --instance-id "${valgrind_instance_id}"
+
+# get the instance ip address
+echo "Getting the ip address of the instance..."
+valgrind_instance_ip=$(aws ec2 describe-instances \
+    --instance-ids "${valgrind_instance_id}" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' \
+    --output text)
+
+# run valgrind tests
+echo "Running the valgrind tests..."
+echo "This will take hours, test results will be sent via e-mail."
+ssh -o StrictHostKeyChecking=no -i $key_name.pem ubuntu@$valgrind_instance_ip "screen -d -m run-valgrind-tests"
+
+#delete the key pair after we are done with tests
+aws ec2 delete-key-pair --key-name $key_name

valgrind/run-valgrind-tests

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# download and install required packages
+sudo apt-get update
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+    build-essential \
+    libreadline6 \
+    libreadline6-dev \
+    zlib1g-dev \
+    flex \
+    bison \
+    libssl-dev \
+    valgrind \
+    mailutils
+
+# set environment variables
+export LC_ALL=en_US.UTF-8
+export LANG=en_US.UTF-8
+export LANGUAGE=en_US.UTF-8
+export PG_CONFIG=/usr/local/pgsql/bin/pg_config
+
+# download and install PostgreSQL
+git clone -b "REL9_6_STABLE" --depth 1 git://git.postgresql.org/git/postgresql.git
+cd postgresql/
+./configure --enable-cassert --enable-debug CFLAGS="-ggdb -Og -DUSE_VALGRIND"
+
+# we will use this to parallelize PostgreSQL compilation
+procs="$(nproc)"
+mjobs="$((procs + 1))"
+make -j "${mjobs}" -s
+sudo make install
+export PATH=/usr/local/pgsql/bin:$PATH
+
+# download and install Citus
+cd ..
+git clone https://github.com/citusdata/citus.git
+cd citus/
+./configure
+make clean
+make -j8 -s
+sudo make install
+
+# this is necessary to start tests
+sudo chown ubuntu /usr/local/pgsql/bin/ -R
+
+# run valgrind tests
+cd src/test/regress
+make check-multi-vg VALGRIND_LOG_FILE=logs.txt || true
+
+# surprisingly this hits inbox
+if [ -s logs.txt ]; then
+	mail -aFrom:valgrind-test@citusdata.com -s "[Valgrind Test Results] - Failure" -A logs.txt burak@citusdata.com metin@citusdata.com < /dev/null
+else
+	mail -aFrom:valgrind-test@citusdata.com -s "[Valgrind Test Results] - Success" burak@citusdata.com metin@citusdata.com < /dev/null
+fi
+
+# just to ensure everything is completed in the test instance
+sleep 30
+
+# shut-down the instance, this will also terminate the instance because we set instance-initiated-shutdown-behavior to terminate
+sudo shutdown -h now