diff --git a/.gitignore b/.gitignore index 72425848..5127c413 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ # = Project-Specific = # ==================== -# generate man pages +# generated man pages *.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index fb4eed22..89f610a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +### citustools v0.2.0 (May 13, 2016) ### + +* Adds wrapper to simplify generating OS packages for Citus projects + +* Some perlcritic and perltidy cleanup here and there + ### citustools v0.1.0 (February 16, 2016) ### * Initial release diff --git a/HomebrewFormula/citustools.rb b/HomebrewFormula/citustools.rb new file mode 100644 index 00000000..a11e2c08 --- /dev/null +++ b/HomebrewFormula/citustools.rb @@ -0,0 +1,28 @@ +class Docker < Requirement + fatal true + default_formula "docker" + + satisfy { which "docker" } + + def message + "Docker is required for this package." + end +end + +class Citustools < Formula + desc "Tools and config used in Citus Data projects." + homepage "https://github.com/citusdata/tools" + url "https://github.com/citusdata/tools/archive/v0.1.0.tar.gz" + sha256 "dc773c21989aa4d716b653ed7542d333f63f14a10d470f9a24fe12fac836b262" + + depends_on "uncrustify" + depends_on Docker + + def install + system "make", "install", "prefix=#{prefix}", "sysconfdir=#{etc}" + end + + test do + system "true" + end +end diff --git a/Makefile b/Makefile index 6f863ca4..09243982 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ export sysconfdir := $(prefix)/etc export pkgsysconfdir := $(sysconfdir)/$(PACKAGE_NAME) # logic from http://stackoverflow.com/a/11206700 -SUBDIRS := $(addsuffix /., uncrustify) +SUBDIRS := $(addsuffix /., packaging uncrustify) TARGETS := all clean install SUBDIRS_TARGETS := $(foreach t,$(TARGETS),$(addsuffix $t,$(SUBDIRS))) diff --git a/packaging/Makefile b/packaging/Makefile new file mode 100644 index 00000000..22b191f6 --- /dev/null +++ b/packaging/Makefile @@ -0,0 +1,27 @@ +# needed variables will be passed in via top-level Makefile + +INSTALL := install -c +INSTALL_DATA := $(INSTALL) -m 644 +INSTALL_SCRIPT := $(INSTALL) -m 755 + +POD2MAN := pod2man --center "Citus Data Tools" -r "Citus Data" +MANPAGES := citus_package.1 + +all: man + +man: $(MANPAGES) + +%.1: % + $(POD2MAN) --quotes=none --section 1 $< $@ + +clean: + rm -f *.1 + +installdirs: + $(INSTALL) -d $(DESTDIR)$(bindir) $(DESTDIR)$(pkgsysconfdir) $(DESTDIR)$(mandir)/man1 + +install: all installdirs + $(INSTALL_SCRIPT) citus_package $(DESTDIR)$(bindir) + $(INSTALL_DATA) $(MANPAGES) $(DESTDIR)$(mandir)/man1 + +.PHONY: all man clean installdirs install diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 00000000..b93adfb3 --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,15 @@ +# Packaging + +`citus_package` encapsulates complex packaging logic to ensure team members can easily build release, nightly, and custom packages for any Citus project on any supported OS. Under the hood, it's using [Docker][1] to guarantee some level of repeatability. + +## Getting Started + +`citus_package` requires `docker` v1.10 or greater. + +`make install` to install the script and a man page. `man citus_package` for more details. + +## Usage + +Ensure your `GITHUB_TOKEN` environment variable is properly set (see the man page if you're not sure how to do that). Make sure Docker is running, then you're off to the races! For example, build a `citus` nightly on CentOS 7, Debian Jessie and Ubuntu Xenial like so: `citus_package -p el/7 -p debian/jessie -p ubuntu/xenial citus nightly` + +[1]: https://www.docker.com diff --git a/packaging/citus_package b/packaging/citus_package new file mode 100755 index 00000000..27aad6fc --- /dev/null +++ b/packaging/citus_package @@ -0,0 +1,338 @@ +#!/usr/bin/perl -w + +use strict; +use warnings; +use Getopt::Long qw(:config no_auto_abbrev no_ignore_case); +use POSIX qw(setlocale LC_ALL); +use File::Temp qw(tempdir); +use List::Util qw(any none); +use Cwd qw(getcwd); +BEGIN { $Pod::Usage::Formatter = 'Pod::Text::Termcap'; } +use Pod::Usage qw(pod2usage); + +# untaint environment +local $ENV{'PATH'} = + '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; +delete @ENV{ 'IFS', 'CDPATH', 'ENV', 'BASH_ENV' }; + +use constant BAD_USAGE => 64; ## no critic (ProhibitConstantPragma) +use constant BAD_INPUT => 65; ## no critic (ProhibitConstantPragma) +use constant NO_SERVICE => 69; ## no critic (ProhibitConstantPragma) +use constant BAD_CONFIG => 78; ## no critic (ProhibitConstantPragma) + +my %supported_platforms = ( + debian => [ "jessie", "wheezy" ], + el => [ "7", "6" ], + fedora => [ "23", "22" ], + ol => [ "7", "6" ], + ubuntu => [ "xenial", "wily", "trusty", "precise" ] +); + +my @rh_flavors = qw(el fedora ol); + +my %docker_names = ( + debian => "debian", + el => "centos", + fedora => "fedora", + ol => "oraclelinux", + ubuntu => "ubuntu" +); + +sub verify_platforms { + my (@platforms) = @_; + + if ( @platforms == 0 ) { + pod2usage( + -msg => "You must specify at least one platform.", + -exitval => BAD_USAGE, + -verbose => 1 + ); + } + + foreach my $platform (@platforms) { + my ( $os, $release ) = split( '/', $platform, 2 ); + + if ( exists $supported_platforms{$os} ) { + my @releases = @{ $supported_platforms{$os} }; + if ( none { $_ eq $release } @releases ) { + pod2usage( + -msg => "Unrecognized $os release: $release", + -exitval => BAD_INPUT, + -verbose => 99, + -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS" + ); + } + } + else { + pod2usage( + -msg => "Unrecognized OS: $os", + -exitval => BAD_INPUT, + -verbose => 99, + -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS" + ); + } + } + + return; +} + +sub get_and_verify_token { + unless ( exists $ENV{GITHUB_TOKEN} ) { + pod2usage( + -msg => "You must have a GITHUB_TOKEN set.", + -exitval => BAD_CONFIG, + -verbose => 99, + -sections => "ENVIRONMENT" + ); + } + + my $github_token = $ENV{GITHUB_TOKEN}; + if ( $ENV{GITHUB_TOKEN} =~ /^(\w+)$/ ) { + $github_token = $1; + } + else { + pod2usage( + -msg => "Malformed GITHUB_TOKEN: $github_token", + -exitval => BAD_INPUT, + -verbose => 99, + -sections => "ENVIRONMENT" + ); + } + + my $cmd = "curl -sf -H 'Authorization: token $github_token' " + . 'https://api.github.com/'; + my $result = `$cmd > /dev/null 2>&1`; + my $exit_code = $? >> 8; + + if ( $exit_code == 22 ) { + pod2usage( + -msg => "Your token was rejected by GitHub.", + -exitval => BAD_INPUT, + -verbose => 99, + -sections => "ENVIRONMENT" + ); + } + + return $github_token; +} + +sub verify_docker_running { + my $result = `docker info > /dev/null 2>&1`; + my $exit_code = $? >> 8; + + unless ( $exit_code == 0 ) { + warn "Cannot connect to the Docker daemon. Is Docker running?\n"; + exit NO_SERVICE; + } + + return; +} + +my ( @platforms, $project, $build_type, $opt_help ); + +GetOptions( 'p|platform=s' => \@platforms, 'help!' => \$opt_help ) + or pod2usage( + -msg => "See '$0 --help' for more information.", + -exitval => BAD_USAGE + ); + +pod2usage( -verbose => 1 ) if $opt_help; + +verify_platforms(@platforms); + +if ( @ARGV != 2 ) { + pod2usage( + -msg => "You must specify a project and build type.", + -exitval => BAD_USAGE + ); +} + +( $project, $build_type ) = @ARGV; + +if ( $project =~ /^(citus|enterprise|rebalancer)$/ ) { + $project = $1; +} +else { + pod2usage( + -msg => "Unrecognized project: $project", + -exitval => BAD_INPUT + ); +} + +verify_docker_running(); + +my $github_token = get_and_verify_token(); +my $homedir = ( getpwuid($<) )[7]; +my $tempdir = tempdir( ".citus_package.XXXXX", DIR => $homedir, CLEANUP => 1 ); +my $currentdir = getcwd(); + +foreach my $platform (@platforms) { + my ( $os, $release ); + + if ( $platform =~ /^(\w+)\/(\w+)$/ ) { + $os = $1; + $release = $2; + } + + my $docker_name = $docker_names{$os}; + my $docker_platform = "$docker_name-$release"; + my $outputdir = $tempdir . '/' . $docker_platform; + my @pg_versions = + ( any { $_ eq $os } @rh_flavors ) ? qw (pg94 pg95) : qw (all); + + foreach my $pg (@pg_versions) { + my @docker_args = ( + qw(run --rm -v), + "$outputdir:/packages", + '-e', + "GITHUB_TOKEN=$github_token", + "citusdata/packaging:$docker_platform-$pg", + $project, + $build_type + ); + + system( 'docker', @docker_args ); + + if ( $? == -1 ) { + die "failed to execute: $!\n"; + } + elsif ( $? & 127 ) { + die "child died with signal %d, %s coredump\n", + ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without'; + } + else { + my $exit_code = $? >> 8; + die "docker run failed. see output for details.\n" if $exit_code; + } + } +} + +system( 'mv', ( ( glob "$tempdir/*" ), $currentdir ) ); + +__END__ + +=head1 NAME + +citus_package - easily create OS packages for Citus projects + +=head1 SYNOPSIS + +B [I] I I + +=head1 DESCRIPTION + + +Packages a Citus project for one or more platforms and places the results in +platform-specific directories within the working directory. B +uses Docker under the hood to ensure repeatable builds, so a working Docker +installation is the only prerequisite. + +Given a Citus I and I, B will build one +package for a single platform, specified using the B<--platform> option. This +option can be provided multiple times in order to build a package for many +platforms at once. + +The I argument has two special values: I and I. A +release build is based on the latest release version (extracted from the build +files contained within the C GitHub repository), pulling +code from the corresponding git tag, which must be have a GitHub-verified +signature. A nightly build is based on the latest commit to the "active" branch +for a given project, which is usually C, but can differ by project. + +All other I values are passed directly to GitHub, which is free to +interpret them how it sees fit, e.g. branch names, tags, or commit identifiers. + +B uses the GitHub API to gather information about the project +it is building. As such, a valid C environment variable must be +set. See the L section for details. + +=head1 OPTIONS + +=over 4 + +=item B<-p> I, B<--platform=>I + +Platform: required. Provide more than once for multi-platform builds + +=back + +=head1 ENVIRONMENT + +For B to do its job, the C environment variable +must be populated with a valid GitHub personal access token. It is recommended +that you add a line to your shell profile to ensure this variable is always +correctly set. + +To generate a new access token, ensure you're logged into GitHub, then navigate +to your account settings. Choose "Personal access tokens" from the sidebar, +press the "Generate new token" button and name your token (like "packaging"). +Ensure the top-level C and C boxes are checked and press the +"Generate token" button. + +B Paste it +into your e.g. C<.bash_profile> or C<.zshrc> to ensure your shells will have +access to your new token. + +=head1 SUPPORTED PROJECTS + +=over 4 + +=item I Citus (Open-Source) + +=item I Citus Enterprise + +=item I Shard Rebalancer + +=back + +=head1 SUPPORTED PLATFORMS + +=over 4 + +=item I Debian 8 "Jessie" + +=item I Debian 7 "Wheezy" + +=item I Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux) + +=item I Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux) + +=item I Fedora 23 + +=item I Fedora 22 + +=item I
    Oracle Linux 7.0 + +=item I
      Oracle Linux 6.0 + +=item I Ubuntu (16.04 LTS Xenial Xerus) + +=item I Ubuntu 15.10 (Wily Werewolf) + +=item I Ubuntu 14.04 LTS (Trusty Tahr) + +=item I Ubuntu 12.04 LTS (Precise Pangolin) + +=back + +=head1 TODO + +Eventually support a different output folder. + +=head1 SEE ALSO + +=over 4 + +=item L + +=item L + +=item L + +=item L + +=back + +=head1 AUTHOR + +Jason Petersen Ljason@citusdata.comE> diff --git a/travis/pg_travis_multi_test.sh b/travis/pg_travis_multi_test.sh index e0d7a5a7..fc88614c 100755 --- a/travis/pg_travis_multi_test.sh +++ b/travis/pg_travis_multi_test.sh @@ -3,6 +3,8 @@ set -eux status=0 +testtargets="check-multi check-multi-fdw check-worker" +testtargets="${testtargets} $*" # Configure, build, and install extension ./configure PG_CONFIG=/usr/lib/postgresql/$PGVERSION/bin/pg_config @@ -13,7 +15,7 @@ sudo make install cd src/test/regress # Run tests. DBs owned by non-standard owner put socket in /tmp -make check-multi check-multi-fdw check-worker || status=$? +make ${testtargets} || status=$? # Print diff if it exists if test -f regression.diffs; then cat regression.diffs; fi diff --git a/travis/setup_apt.sh b/travis/setup_apt.sh index fdbf7766..7f200815 100755 --- a/travis/setup_apt.sh +++ b/travis/setup_apt.sh @@ -7,6 +7,9 @@ set -eux # import the PostgreSQL repository key sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys ACCC4CF8 +# wtf, Google? +sudo rm /etc/apt/sources.list.d/google-chrome* + # add the PostgreSQL 9.5 repository sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list' diff --git a/uncrustify/citus_indent b/uncrustify/citus_indent index 10c9ba23..fe72320b 100755 --- a/uncrustify/citus_indent +++ b/uncrustify/citus_indent @@ -6,7 +6,7 @@ use Getopt::Long qw(:config no_auto_abbrev no_ignore_case prefix=--); use POSIX qw(setlocale LC_ALL); # untaint environment -$ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; +local $ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; my ($quiet, $check);