From 4fa4b23ea685ae719a7afb6879da87ec7af8496b Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Wed, 27 Apr 2016 21:33:29 -0700 Subject: [PATCH 01/10] Permit passing extra tests to multi test script Simplest change needed to add custom tests in certain places. --- travis/pg_travis_multi_test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travis/pg_travis_multi_test.sh b/travis/pg_travis_multi_test.sh index e0d7a5a7..fc88614c 100755 --- a/travis/pg_travis_multi_test.sh +++ b/travis/pg_travis_multi_test.sh @@ -3,6 +3,8 @@ set -eux status=0 +testtargets="check-multi check-multi-fdw check-worker" +testtargets="${testtargets} $*" # Configure, build, and install extension ./configure PG_CONFIG=/usr/lib/postgresql/$PGVERSION/bin/pg_config @@ -13,7 +15,7 @@ sudo make install cd src/test/regress # Run tests. DBs owned by non-standard owner put socket in /tmp -make check-multi check-multi-fdw check-worker || status=$? +make ${testtargets} || status=$? # Print diff if it exists if test -f regression.diffs; then cat regression.diffs; fi From ecfa2c68449ba8ed4a8eb2bdabc9a27b2ca309e3 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Thu, 12 May 2016 15:34:26 -0600 Subject: [PATCH 02/10] Move citustools formula Turns out we have have it alongside the code. Hooray. --- HomebrewFormula/citustools.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 HomebrewFormula/citustools.rb diff --git a/HomebrewFormula/citustools.rb b/HomebrewFormula/citustools.rb new file mode 100644 index 00000000..b6332230 --- /dev/null +++ b/HomebrewFormula/citustools.rb @@ -0,0 +1,16 @@ +class Citustools < Formula + desc "Tools and config used in Citus Data projects." + homepage "https://github.com/citusdata/tools" + url "https://github.com/citusdata/tools/archive/v0.1.0.tar.gz" + sha256 "dc773c21989aa4d716b653ed7542d333f63f14a10d470f9a24fe12fac836b262" + + depends_on "uncrustify" + + def install + system "make", "install", "prefix=#{prefix}", "sysconfdir=#{etc}" + end + + test do + system "true" + end +end From 3e15ae73eec432dc55120434af4ee0e18b834c6e Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Thu, 12 May 2016 15:52:16 -0600 Subject: [PATCH 03/10] Remove Google repos from source lists First got "no public key available for the following key IDs". Tried to hack around that and got a hash mismatch error instead. Eventually gave up and realized I could cut Google out entirely. Fixed. See: https://bugs.chromium.org/p/chromium/issues/detail?id=596074 --- travis/setup_apt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/travis/setup_apt.sh b/travis/setup_apt.sh index fdbf7766..7f200815 100755 --- a/travis/setup_apt.sh +++ b/travis/setup_apt.sh @@ -7,6 +7,9 @@ set -eux # import the PostgreSQL repository key sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys ACCC4CF8 +# wtf, Google? +sudo rm /etc/apt/sources.list.d/google-chrome* + # add the PostgreSQL 9.5 repository sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list' From f6710670ad6ff2868f6e61a95a4df17b86f3f385 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 11:37:11 -0600 Subject: [PATCH 04/10] Add citus_package Finally have an easy-to-use wrapper. --- .gitignore | 2 +- Makefile | 2 +- packaging/Makefile | 27 ++++++ packaging/README.md | 17 ++++ packaging/citus_package | 209 ++++++++++++++++++++++++++++++++++++++++ uncrustify/citus_indent | 2 +- 6 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 packaging/Makefile create mode 100644 packaging/README.md create mode 100755 packaging/citus_package diff --git a/.gitignore b/.gitignore index 72425848..5127c413 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ # = Project-Specific = # ==================== -# generate man pages +# generated man pages *.1 diff --git a/Makefile b/Makefile index 6f863ca4..09243982 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ export sysconfdir := $(prefix)/etc export pkgsysconfdir := $(sysconfdir)/$(PACKAGE_NAME) # logic from http://stackoverflow.com/a/11206700 -SUBDIRS := $(addsuffix /., uncrustify) +SUBDIRS := $(addsuffix /., packaging uncrustify) TARGETS := all clean install SUBDIRS_TARGETS := $(foreach t,$(TARGETS),$(addsuffix $t,$(SUBDIRS))) diff --git a/packaging/Makefile b/packaging/Makefile new file mode 100644 index 00000000..22b191f6 --- /dev/null +++ b/packaging/Makefile @@ -0,0 +1,27 @@ +# needed variables will be passed in via top-level Makefile + +INSTALL := install -c +INSTALL_DATA := $(INSTALL) -m 644 +INSTALL_SCRIPT := $(INSTALL) -m 755 + +POD2MAN := pod2man --center "Citus Data Tools" -r "Citus Data" +MANPAGES := citus_package.1 + +all: man + +man: $(MANPAGES) + +%.1: % + $(POD2MAN) --quotes=none --section 1 $< $@ + +clean: + rm -f *.1 + +installdirs: + $(INSTALL) -d $(DESTDIR)$(bindir) $(DESTDIR)$(pkgsysconfdir) $(DESTDIR)$(mandir)/man1 + +install: all installdirs + $(INSTALL_SCRIPT) citus_package $(DESTDIR)$(bindir) + $(INSTALL_DATA) $(MANPAGES) $(DESTDIR)$(mandir)/man1 + +.PHONY: all man clean installdirs install diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 00000000..02f16889 --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,17 @@ +# Packaging + +`citus_indent` wraps [`uncrustify`][1], a popular C source code beautifier. When invoked, it immediately applies Citus C style on any git-tracked C files under the current working directory, though a `--check` flag is implemented to check style without modifying any files. + +## Getting Started + +`citus_indent` requires `uncrustify` v0.60 or greater. + +`make install` to install the script, the Citus style configuration file, and a man page. `man citus_indent` for more details. + +## Usage + +Apply the `citus-style` git attribute to any files that need the Citus C style applied. After that, just ensure you're within the project's directory hierarchy and run `citus_indent` to format all files. Add style changes with `git add -p`. + +`citus_indent --check` is useful for scripts: it will not modify any files and simply exits with a non-zero status if any files marked with `citus-style` are non-compliant. + +[1]: http://uncrustify.sourceforge.net diff --git a/packaging/citus_package b/packaging/citus_package new file mode 100755 index 00000000..81ae9b53 --- /dev/null +++ b/packaging/citus_package @@ -0,0 +1,209 @@ +#!/usr/bin/perl -w + +use strict; +use warnings; +use Getopt::Long qw(:config no_auto_abbrev no_ignore_case); +use POSIX qw(setlocale LC_ALL); +use File::Temp qw(tempdir); +use List::Util qw(any none); +use Cwd qw(getcwd); + +# untaint environment +local $ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; +delete @ENV{ 'IFS', 'CDPATH', 'ENV', 'BASH_ENV' }; + +use constant BAD_USAGE => 64; ## no critic (ProhibitConstantPragma) + +my %supported_platforms = ( + debian => [ "jessie", "wheezy" ], + el => [ "7", "6" ], + fedora => [ "23", "22" ], + ol => [ "7", "6" ], + ubuntu => [ "xenial", "wily", "trusty", "precise" ] +); + +my @rh_flavors = qw(el fedora ol); + +my %docker_names = ( + debian => "debian", + el => "centos", + fedora => "fedora", + ol => "oraclelinux", + ubuntu => "ubuntu" +); + +sub verify_platforms { + my (@platforms) = @_; + + if ( @platforms == 0 ) { + print "You must specify at least one platform.\n"; + exit BAD_USAGE; + } + + foreach my $platform (@platforms) { + my ( $os, $release ) = split( '/', $platform, 2 ); + + if ( exists $supported_platforms{$os} ) { + my @releases = @{ $supported_platforms{$os} }; + if ( none { $_ eq $release } @releases ) { + print "Unrecognized $os release: $release\n"; + exit BAD_USAGE; + } + } + else { + print "Unrecognized OS: $os\n"; + exit BAD_USAGE; + } + } + + return; +} + +my ( @platforms, $project, $build_type ); + +exit BAD_USAGE unless GetOptions( 'p|platform=s' => \@platforms ); + +verify_platforms(@platforms); + +if ( @ARGV != 2 ) { + print "You must specify a project and build type.\n"; + exit BAD_USAGE; +} + +( $project, $build_type ) = @ARGV; + +if ( $project =~ /^(citus|enterprise|rebalancer)$/ ) { + $project = $1; +} +else { + print "Unrecognized project: $project\n"; + exit BAD_USAGE; +} + +my $homedir = ( getpwuid($<) )[7]; +my $tempdir = tempdir( ".citus_package.XXXXX", DIR => $homedir, CLEANUP => 1 ); +my $currentdir = getcwd(); + +foreach my $platform (@platforms) { + my ( $os, $release ); + + if ( $platform =~ /^(\w+)\/(\w+)$/ ) { + $os = $1; + $release = $2; + } + + my $docker_name = $docker_names{$os}; + my $docker_platform = "$docker_name-$release"; + my $outputdir = $tempdir . '/' . $docker_platform; + my @pg_versions = + ( any { $_ eq $os } @rh_flavors ) ? qw (pg94 pg95) : qw (all); + + foreach my $pg (@pg_versions) { + my @docker_args = ( + qw(run --rm -v), + "$outputdir:/packages", + '-e', + "GITHUB_TOKEN=$ENV{GITHUB_TOKEN}", + "citusdata/packaging:$docker_platform-$pg", + $project, + $build_type + ); + + system( 'docker', @docker_args ); + } +} + +system( 'mv', ( ( glob "$tempdir/*" ), $currentdir ) ); + +__END__ + +=head1 NAME + +citus_package - easily create OS packages for Citus projects + +=head1 SYNOPSIS + +B [I] I I + +=head1 DESCRIPTION + + +Packages a Citus project for one or more platforms and places the results in +platform-specific directories within the working directory. B +uses Docker under the hood to ensure repeatable builds, so a working Docker +installation is the only prerequisite. + +Given a Citus I and I, B will build one +package for a single platform, specified using the B<--platform> option. This +option can be provided multiple times in order to build a package for many +platforms at once. + +The I argument has two special values: I and I. A +release build is based on the latest release version (extracted from the build +files contained within the C GitHub repository), pulling +code from the corresponding git tag, which must be have a GitHub-verified +signature. A nightly build is based on the latest commit to the "active" branch +for a given project, which is usually C, but can differ by project. + +All other I values are passed directly to GitHub, which is free to +interpret them how it sees fit, e.g. branch names, tags, or commit identifiers. + +=head1 OPTIONS + +=over 4 + +=item B<-p> I, B<--platform=>I + +Platform: required. Provide more than once for multi-platform builds + +=back + +=head1 SUPPORTED PROJECTS + +=over 4 + +=item B Citus (Open-Source) + +=item B Citus Enterprise + +=item B Shard Rebalancer + +=back + +=head1 SUPPORTED PLATFORMS + +=over 4 + +=item B Debian 8 "Jessie" + +=item B Debian 7 "Wheezy" + +=item B Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux) + +=item B Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux) + +=item B Fedora 23 + +=item B Fedora 22 + +=item B
    Oracle Linux 7.0 + +=item B
      Oracle Linux 6.0 + +=item B Ubuntu (16.04 LTS Xenial Xerus) + +=item B Ubuntu 15.10 (Wily Werewolf) + +=item B Ubuntu 14.04 LTS (Trusty Tahr) + +=item B Ubuntu 12.04 LTS (Precise Pangolin) + +=back + +=head1 TODO + +Eventually support a different output folder. + +=head1 AUTHOR + +Jason Petersen Ljason@citusdata.comE> diff --git a/uncrustify/citus_indent b/uncrustify/citus_indent index 10c9ba23..fe72320b 100755 --- a/uncrustify/citus_indent +++ b/uncrustify/citus_indent @@ -6,7 +6,7 @@ use Getopt::Long qw(:config no_auto_abbrev no_ignore_case prefix=--); use POSIX qw(setlocale LC_ALL); # untaint environment -$ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; +local $ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; my ($quiet, $check); From 419b98c789aa5959c002de6ed08e0b766dcb7c3d Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 11:56:23 -0600 Subject: [PATCH 05/10] Clean up usage messages For usability. --- packaging/citus_package | 52 ++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/packaging/citus_package b/packaging/citus_package index 81ae9b53..e56d3964 100755 --- a/packaging/citus_package +++ b/packaging/citus_package @@ -7,12 +7,15 @@ use POSIX qw(setlocale LC_ALL); use File::Temp qw(tempdir); use List::Util qw(any none); use Cwd qw(getcwd); +BEGIN { $Pod::Usage::Formatter = 'Pod::Text::Termcap'; } +use Pod::Usage qw(pod2usage); # untaint environment -local $ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; +local $ENV{'PATH'} = + '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; delete @ENV{ 'IFS', 'CDPATH', 'ENV', 'BASH_ENV' }; -use constant BAD_USAGE => 64; ## no critic (ProhibitConstantPragma) +use constant BAD_USAGE => 64; ## no critic (ProhibitConstantPragma) my %supported_platforms = ( debian => [ "jessie", "wheezy" ], @@ -36,8 +39,11 @@ sub verify_platforms { my (@platforms) = @_; if ( @platforms == 0 ) { - print "You must specify at least one platform.\n"; - exit BAD_USAGE; + pod2usage( + -msg => "You must specify at least one platform.", + -exitval => BAD_USAGE, + -verbose => 1 + ); } foreach my $platform (@platforms) { @@ -46,28 +52,44 @@ sub verify_platforms { if ( exists $supported_platforms{$os} ) { my @releases = @{ $supported_platforms{$os} }; if ( none { $_ eq $release } @releases ) { - print "Unrecognized $os release: $release\n"; - exit BAD_USAGE; + pod2usage( + -msg => "Unrecognized $os release: $release", + -exitval => BAD_USAGE, + -verbose => 99, + -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS" + ); } } else { - print "Unrecognized OS: $os\n"; - exit BAD_USAGE; + pod2usage( + -msg => "Unrecognized OS: $os", + -exitval => BAD_USAGE, + -verbose => 99, + -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS" + ); } } return; } -my ( @platforms, $project, $build_type ); +my ( @platforms, $project, $build_type, $opt_help ); + +GetOptions( 'p|platform=s' => \@platforms, 'help!' => \$opt_help ) + or pod2usage( + -msg => "See '$0 --help' for more information.", + -exitval => BAD_USAGE + ); -exit BAD_USAGE unless GetOptions( 'p|platform=s' => \@platforms ); +pod2usage( -verbose => 1 ) if $opt_help; verify_platforms(@platforms); if ( @ARGV != 2 ) { - print "You must specify a project and build type.\n"; - exit BAD_USAGE; + pod2usage( + -msg => "You must specify a project and build type.", + -exitval => BAD_USAGE + ); } ( $project, $build_type ) = @ARGV; @@ -76,8 +98,10 @@ if ( $project =~ /^(citus|enterprise|rebalancer)$/ ) { $project = $1; } else { - print "Unrecognized project: $project\n"; - exit BAD_USAGE; + pod2usage( + -msg => "Unrecognized project: $project", + -exitval => BAD_USAGE + ); } my $homedir = ( getpwuid($<) )[7]; From 8663c13106d520f475d19e997a012a5c0d056b4e Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 15:05:56 -0600 Subject: [PATCH 06/10] Add GITHUB_TOKEN check, usage messages Cleaning up. --- packaging/citus_package | 122 ++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 17 deletions(-) diff --git a/packaging/citus_package b/packaging/citus_package index e56d3964..8ba10a18 100755 --- a/packaging/citus_package +++ b/packaging/citus_package @@ -73,6 +73,46 @@ sub verify_platforms { return; } +sub get_and_verify_token { + unless ( exists $ENV{GITHUB_TOKEN} ) { + pod2usage( + -msg => "You must have a GITHUB_TOKEN set.", + -exitval => BAD_USAGE, + -verbose => 99, + -sections => "ENVIRONMENT" + ); + } + + my $github_token = $ENV{GITHUB_TOKEN}; + if ( $ENV{GITHUB_TOKEN} =~ /^(\w+)$/ ) { + $github_token = $1; + } + else { + pod2usage( + -msg => "Malformed GITHUB_TOKEN: $github_token", + -exitval => BAD_USAGE, + -verbose => 99, + -sections => "ENVIRONMENT" + ); + } + + my $cmd = "curl -sf -H 'Authorization: token $github_token' " + . 'https://api.github.com/'; + my $result = `$cmd`; + my $exit_code = $? >> 8; + + if ( $exit_code == 22 ) { + pod2usage( + -msg => "Your token was rejected by GitHub.", + -exitval => BAD_USAGE, + -verbose => 99, + -sections => "ENVIRONMENT" + ); + } + + return $github_token; +} + my ( @platforms, $project, $build_type, $opt_help ); GetOptions( 'p|platform=s' => \@platforms, 'help!' => \$opt_help ) @@ -104,7 +144,8 @@ else { ); } -my $homedir = ( getpwuid($<) )[7]; +my $github_token = get_and_verify_token(); +my $homedir = ( getpwuid($<) )[7]; my $tempdir = tempdir( ".citus_package.XXXXX", DIR => $homedir, CLEANUP => 1 ); my $currentdir = getcwd(); @@ -127,13 +168,25 @@ foreach my $platform (@platforms) { qw(run --rm -v), "$outputdir:/packages", '-e', - "GITHUB_TOKEN=$ENV{GITHUB_TOKEN}", + "GITHUB_TOKEN=$github_token", "citusdata/packaging:$docker_platform-$pg", $project, $build_type ); system( 'docker', @docker_args ); + + if ( $? == -1 ) { + die "failed to execute: $!\n"; + } + elsif ( $? & 127 ) { + die "child died with signal %d, %s coredump\n", + ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without'; + } + else { + my $exit_code = $? >> 8; + die "docker run failed. see output for details.\n" if $exit_code; + } } } @@ -172,6 +225,10 @@ for a given project, which is usually C, but can differ by project. All other I values are passed directly to GitHub, which is free to interpret them how it sees fit, e.g. branch names, tags, or commit identifiers. +B uses the GitHub API to gather information about the project +it is building. As such, a valid C environment variable must be +set. See the L section for details. + =head1 OPTIONS =over 4 @@ -182,15 +239,32 @@ Platform: required. Provide more than once for multi-platform builds =back +=head1 ENVIRONMENT + +For B to do its job, the C environment variable +must be populated with a valid GitHub personal access token. It is recommended +that you add a line to your shell profile to ensure this variable is always +correctly set. + +To generate a new access token, ensure you're logged into GitHub, then navigate +to your account settings. Choose "Personal access tokens" from the sidebar, +press the "Generate new token" button and name your token (like "packaging"). +Ensure the top-level C and C boxes are checked and press the +"Generate token" button. + +B Paste it +into your e.g. C<.bash_profile> or C<.zshrc> to ensure your shells will have +access to your new token. + =head1 SUPPORTED PROJECTS =over 4 -=item B Citus (Open-Source) +=item I Citus (Open-Source) -=item B Citus Enterprise +=item I Citus Enterprise -=item B Shard Rebalancer +=item I Shard Rebalancer =back @@ -198,29 +272,29 @@ Platform: required. Provide more than once for multi-platform builds =over 4 -=item B Debian 8 "Jessie" +=item I Debian 8 "Jessie" -=item B Debian 7 "Wheezy" +=item I Debian 7 "Wheezy" -=item B Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux) +=item I Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux) -=item B Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux) +=item I Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux) -=item B Fedora 23 +=item I Fedora 23 -=item B Fedora 22 +=item I Fedora 22 -=item B
        Oracle Linux 7.0 +=item I
          Oracle Linux 7.0 -=item B
            Oracle Linux 6.0 +=item I
              Oracle Linux 6.0 -=item B Ubuntu (16.04 LTS Xenial Xerus) +=item I Ubuntu (16.04 LTS Xenial Xerus) -=item B Ubuntu 15.10 (Wily Werewolf) +=item I Ubuntu 15.10 (Wily Werewolf) -=item B Ubuntu 14.04 LTS (Trusty Tahr) +=item I Ubuntu 14.04 LTS (Trusty Tahr) -=item B Ubuntu 12.04 LTS (Precise Pangolin) +=item I Ubuntu 12.04 LTS (Precise Pangolin) =back @@ -228,6 +302,20 @@ Platform: required. Provide more than once for multi-platform builds Eventually support a different output folder. +=head1 SEE ALSO + +=over 4 + +=item L + +=item L + +=item L + +=item L + +=back + =head1 AUTHOR Jason Petersen Ljason@citusdata.comE> From e37b52ae5925c80bf4d4f2e7cb460560aae8e8c6 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 15:19:17 -0600 Subject: [PATCH 07/10] Add call to verify Docker running --- packaging/citus_package | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/packaging/citus_package b/packaging/citus_package index 8ba10a18..27aad6fc 100755 --- a/packaging/citus_package +++ b/packaging/citus_package @@ -15,7 +15,10 @@ local $ENV{'PATH'} = '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin'; delete @ENV{ 'IFS', 'CDPATH', 'ENV', 'BASH_ENV' }; -use constant BAD_USAGE => 64; ## no critic (ProhibitConstantPragma) +use constant BAD_USAGE => 64; ## no critic (ProhibitConstantPragma) +use constant BAD_INPUT => 65; ## no critic (ProhibitConstantPragma) +use constant NO_SERVICE => 69; ## no critic (ProhibitConstantPragma) +use constant BAD_CONFIG => 78; ## no critic (ProhibitConstantPragma) my %supported_platforms = ( debian => [ "jessie", "wheezy" ], @@ -54,7 +57,7 @@ sub verify_platforms { if ( none { $_ eq $release } @releases ) { pod2usage( -msg => "Unrecognized $os release: $release", - -exitval => BAD_USAGE, + -exitval => BAD_INPUT, -verbose => 99, -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS" ); @@ -63,7 +66,7 @@ sub verify_platforms { else { pod2usage( -msg => "Unrecognized OS: $os", - -exitval => BAD_USAGE, + -exitval => BAD_INPUT, -verbose => 99, -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS" ); @@ -76,8 +79,8 @@ sub verify_platforms { sub get_and_verify_token { unless ( exists $ENV{GITHUB_TOKEN} ) { pod2usage( - -msg => "You must have a GITHUB_TOKEN set.", - -exitval => BAD_USAGE, + -msg => "You must have a GITHUB_TOKEN set.", + -exitval => BAD_CONFIG, -verbose => 99, -sections => "ENVIRONMENT" ); @@ -89,8 +92,8 @@ sub get_and_verify_token { } else { pod2usage( - -msg => "Malformed GITHUB_TOKEN: $github_token", - -exitval => BAD_USAGE, + -msg => "Malformed GITHUB_TOKEN: $github_token", + -exitval => BAD_INPUT, -verbose => 99, -sections => "ENVIRONMENT" ); @@ -98,13 +101,13 @@ sub get_and_verify_token { my $cmd = "curl -sf -H 'Authorization: token $github_token' " . 'https://api.github.com/'; - my $result = `$cmd`; + my $result = `$cmd > /dev/null 2>&1`; my $exit_code = $? >> 8; if ( $exit_code == 22 ) { pod2usage( - -msg => "Your token was rejected by GitHub.", - -exitval => BAD_USAGE, + -msg => "Your token was rejected by GitHub.", + -exitval => BAD_INPUT, -verbose => 99, -sections => "ENVIRONMENT" ); @@ -113,6 +116,18 @@ sub get_and_verify_token { return $github_token; } +sub verify_docker_running { + my $result = `docker info > /dev/null 2>&1`; + my $exit_code = $? >> 8; + + unless ( $exit_code == 0 ) { + warn "Cannot connect to the Docker daemon. Is Docker running?\n"; + exit NO_SERVICE; + } + + return; +} + my ( @platforms, $project, $build_type, $opt_help ); GetOptions( 'p|platform=s' => \@platforms, 'help!' => \$opt_help ) @@ -140,10 +155,12 @@ if ( $project =~ /^(citus|enterprise|rebalancer)$/ ) { else { pod2usage( -msg => "Unrecognized project: $project", - -exitval => BAD_USAGE + -exitval => BAD_INPUT ); } +verify_docker_running(); + my $github_token = get_and_verify_token(); my $homedir = ( getpwuid($<) )[7]; my $tempdir = tempdir( ".citus_package.XXXXX", DIR => $homedir, CLEANUP => 1 ); From 890d309344f5f50cb365229596c397d2de803e84 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 15:28:51 -0600 Subject: [PATCH 08/10] Update formula Need to ensure users have Docker installed, one way or another. --- HomebrewFormula/citustools.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/HomebrewFormula/citustools.rb b/HomebrewFormula/citustools.rb index b6332230..a11e2c08 100644 --- a/HomebrewFormula/citustools.rb +++ b/HomebrewFormula/citustools.rb @@ -1,3 +1,14 @@ +class Docker < Requirement + fatal true + default_formula "docker" + + satisfy { which "docker" } + + def message + "Docker is required for this package." + end +end + class Citustools < Formula desc "Tools and config used in Citus Data projects." homepage "https://github.com/citusdata/tools" @@ -5,6 +16,7 @@ class Citustools < Formula sha256 "dc773c21989aa4d716b653ed7542d333f63f14a10d470f9a24fe12fac836b262" depends_on "uncrustify" + depends_on Docker def install system "make", "install", "prefix=#{prefix}", "sysconfdir=#{etc}" From 4334f827ef95c20ea1795e636062581ac960effe Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 15:36:22 -0600 Subject: [PATCH 09/10] Fix packaging README Oops. --- packaging/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packaging/README.md b/packaging/README.md index 02f16889..b93adfb3 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,17 +1,15 @@ # Packaging -`citus_indent` wraps [`uncrustify`][1], a popular C source code beautifier. When invoked, it immediately applies Citus C style on any git-tracked C files under the current working directory, though a `--check` flag is implemented to check style without modifying any files. +`citus_package` encapsulates complex packaging logic to ensure team members can easily build release, nightly, and custom packages for any Citus project on any supported OS. Under the hood, it's using [Docker][1] to guarantee some level of repeatability. ## Getting Started -`citus_indent` requires `uncrustify` v0.60 or greater. +`citus_package` requires `docker` v1.10 or greater. -`make install` to install the script, the Citus style configuration file, and a man page. `man citus_indent` for more details. +`make install` to install the script and a man page. `man citus_package` for more details. ## Usage -Apply the `citus-style` git attribute to any files that need the Citus C style applied. After that, just ensure you're within the project's directory hierarchy and run `citus_indent` to format all files. Add style changes with `git add -p`. +Ensure your `GITHUB_TOKEN` environment variable is properly set (see the man page if you're not sure how to do that). Make sure Docker is running, then you're off to the races! For example, build a `citus` nightly on CentOS 7, Debian Jessie and Ubuntu Xenial like so: `citus_package -p el/7 -p debian/jessie -p ubuntu/xenial citus nightly` -`citus_indent --check` is useful for scripts: it will not modify any files and simply exits with a non-zero status if any files marked with `citus-style` are non-compliant. - -[1]: http://uncrustify.sourceforge.net +[1]: https://www.docker.com From 3768b0df25018e888fdfa07a7868812c691796f8 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 13 May 2016 15:38:18 -0600 Subject: [PATCH 10/10] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb4eed22..89f610a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +### citustools v0.2.0 (May 13, 2016) ### + +* Adds wrapper to simplify generating OS packages for Citus projects + +* Some perlcritic and perltidy cleanup here and there + ### citustools v0.1.0 (February 16, 2016) ### * Initial release