[SPARK-8338] Use AMI preinstalled ganglia

ganglia/check_packages.py

@@ -0,0 +1,102 @@
+#!/bin/env python
+"""
+Script to check for exact ganglia and httpd package version installed
+If not found - install them
+If found but version doesn't match - remove and install correct versions
+"""
+
+# the following imports were copied from yum-cli/cli.py module - to make API work
+# there are some side effects that make YumBaseCli().getOptionsConfig() method work
+import sys
+sys.path.insert(0, '/usr/share/yum-cli')
+
+import os
+import os.path
+import sys
+import logging
+import time
+import errno
+
+from yum import Errors
+from yum import plugins
+from yum import logginglevels
+from yum import _
+from yum.i18n import to_unicode, utf8_width
+import yum.misc
+import cli
+from utils import suppress_keyboard_interrupt_message, show_lock_owner, exception2msg
+
+
+yb = cli.YumBaseCli()
+# point yum to older version of Amazon repo and disable updates repo to point yum to ganglia 3.3.7
+args = ["--releasever=2013.03", "--disablerepo=amzn-updates*", "--skip-broken", "list", "ganglia"]
+yb.getOptionsConfig(args)
+
+installed_packages_dict = dict([(p.name, p) for p in yb.rpmdb.returnPackages()])
+
+operations_log = []
+
+def remove_package(**kwargs):
+    """
+
+    :param kwargs: dict
+        accepted keys: name and version, pattern
+    :return:
+    """
+    yb.remove(**kwargs)
+    yb.resolveDeps()
+    yb.buildTransaction()
+    yb.processTransaction()
+
+
+def install_package(**kwargs):
+    """
+
+    :param kwargs: dict
+        accepted keys: name and version, pattern
+    :return:
+    """
+    yb.install(**kwargs)
+    yb.resolveDeps()
+    yb.buildTransaction()
+    yb.processTransaction()
+
+
+# check httpd version
+# if doesn't match uninstall and install httpd2.2
+
+if "httpd" in installed_packages_dict:
+    httpd_po = installed_packages_dict["httpd"]
+    if httpd_po:
+        if not httpd_po.version.startswith("2.2."):
+            remove_package(pattern="httpd*")
+            install_package(pattern="httpd*2.2.*")
+else:
+    install_package(pattern="httpd*2.2.*")
+
+# check ganglia version
+# if doesn't match - uninstall php* and ganglia*
+
+if "ganglia" in installed_packages_dict:
+    ganglia_po = installed_packages_dict["ganglia"]
+    if not ganglia_po.version.startswith("3.3."):
+        operations_log.append("Invalid ganglia version: %s != %s" % (ganglia_po.version, "3.3.*"))
+        print "Removing packages: php*"
+        remove_package(pattern="php*")
+        operations_log.append("Removed packages: %s" % ("php*"))
+        print "Removing packages: ganglia*"
+        remove_package(pattern="ganglia*")
+        operations_log.append("Removed packages: ganglia*")
+        print "Installing ganglia"
+        install_package(pattern="ganglia*")
+        operations_log.append("Installed package: ganglia")
+else:
+    install_package(pattern="ganglia*")
+    operations_log.append("installed ganglia packages")
+
+
+print "Report:"
+if operations_log:
+    print "\n".join(operations_log)
+else:
+    print "All ganglia related packages have been already installed"

ganglia/init.sh

@@ -8,18 +8,14 @@ rm -rf /mnt/ganglia/rrds/*
 mkdir -p /mnt/ganglia/rrds
 chown -R nobody:nobody /mnt/ganglia/rrds
 
-# Install ganglia
-# TODO: Remove this once the AMI has ganglia by default
+/root/spark-ec2/ganglia/check_packages.py
 
-GANGLIA_PACKAGES="ganglia ganglia-web ganglia-gmond ganglia-gmetad"
-
-if ! rpm --quiet -q $GANGLIA_PACKAGES; then
-  yum install -q -y $GANGLIA_PACKAGES;
-fi
-for node in $SLAVES $OTHER_MASTERS; do
-  ssh -t -t $SSH_OPTS root@$node "if ! rpm --quiet -q $GANGLIA_PACKAGES; then yum install -q -y $GANGLIA_PACKAGES; fi" & sleep 0.3
-done
-wait
+pssh --inline \
+    --host "$SLAVES $OTHER_MASTER" \
+    --user root \
+    --extra-args "-t -t $SSH_OPTS" \
+    --timeout 0 \
+    "spark-ec2/ganglia/check_packages.py"
 
 # Post-package installation : Symlink /var/lib/ganglia/rrds to /mnt/ganglia/rrds
 rmdir /var/lib/ganglia/rrds

templates/etc/httpd/conf/httpd.conf

@@ -151,7 +151,6 @@ LoadModule auth_basic_module modules/mod_auth_basic.so
 LoadModule auth_digest_module modules/mod_auth_digest.so
 LoadModule authn_file_module modules/mod_authn_file.so
 LoadModule authn_anon_module modules/mod_authn_anon.so
-LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authn_dbm_module modules/mod_authn_dbm.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule authz_user_module modules/mod_authz_user.so
@@ -193,10 +192,6 @@ LoadModule cache_module modules/mod_cache.so
 LoadModule suexec_module modules/mod_suexec.so
 LoadModule cgi_module modules/mod_cgi.so
 LoadModule version_module modules/mod_version.so
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule mpm_prefork_module modules/mod_mpm_prefork.so
-LoadModule php5_module modules/libphp-5.5.so
 
 #
 # The following modules are not loaded by default: