From 99039cd69f35a43d75f65c287b788ba1b41a6a5a Mon Sep 17 00:00:00 2001 From: Diego Gutierrez Date: Sun, 18 Mar 2018 09:17:32 -0400 Subject: [PATCH] MODIFIED: complete full go rewrite added environment variables to control the container at runtime --- .gitignore | 4 +- Dockerfile | 44 +++++++--- README.md | 32 ++++++- build/build.sh | 10 --- build/release.sh | 2 - changeLog.txt | 4 +- entrypoint.sh | 27 +++++- nginx.conf | 4 +- scan_repo.rb | 40 --------- src/main.go | 215 +++++++++++++++++++++++++++++++++++++++++++++++ supervisord.conf | 17 +--- test/test.json | 9 -- test/test.sh | 57 ------------- 13 files changed, 311 insertions(+), 154 deletions(-) delete mode 100644 build/build.sh delete mode 100644 build/release.sh delete mode 100755 scan_repo.rb create mode 100644 src/main.go delete mode 100644 test/test.json delete mode 100644 test/test.sh diff --git a/.gitignore b/.gitignore index d19812c..3c4a0dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -.idea/** \ No newline at end of file +.idea/** +logs/** +repo/** \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a73e9d1..4499e04 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,46 @@ +# build stage +FROM golang:1.8.3 as builder + +WORKDIR /go/src/github.com/dgutierrez1287/docker-yum-repo + +RUN go get -d -v github.com/Sirupsen/logrus && \ + go get -d -v github.com/rjeczalik/notify && \ + go get -d -v gopkg.in/dickeyxxx/golock.v1 && \ + go get -d -v gopkg.in/natefinch/lumberjack.v2 + +COPY src/*.go . + +RUN GOOS=linux go build -x -o repoScanner . + +# application image FROM centos:7 -MAINTAINER Diego Gutierrez +LABEL maintainer="Diego Gutierrez " -RUN yum -y install epel-release && yum clean all -RUN yum -y update && yum clean all -RUN yum -y install ruby gcc ruby-devel supervisor createrepo yum-utils nginx && yum clean all -RUN gem install rb-inotify +RUN yum -y install epel-release && \ + yum -y update && \ + yum -y install supervisor createrepo yum-utils nginx && \ + yum clean all -RUN mkdir /root/repo -RUN mkdir /root/logs +RUN mkdir /repo && \ + chmod 777 /repo && \ + mkdir -p /logs COPY nginx.conf /etc/nginx/nginx.conf COPY supervisord.conf /etc/supervisord.conf -COPY scan_repo.rb /root/scan_repo.rb +COPY --from=builder /go/src/github.com/dgutierrez1287/docker-yum-repo/repoScanner /root/ -RUN chmod 700 /root/scan_repo.rb +RUN chmod 700 /root/repoScanner EXPOSE 80 -VOLUME /root/repo /root/logs +VOLUME /repo /logs + +ENV DEBUG false +ENV LINUX_HOST true +ENV SERVE_FILES true COPY entrypoint.sh /root/entrypoint.sh -RUN chmod 700 entrypoint.sh -ENTRYPOINT ["entrypoint.sh"] +RUN chmod 700 /root/entrypoint.sh +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/README.md b/README.md index e53e128..1978738 100755 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Docker-Yum-Repo docker-yum-repo builds a yum repo server to run in a docker container. It is built off of CentOS 7 and will update the repo automatically when an rpm is added or removed -courtesy of a custom ruby script using [rb-inotify](https://github.com/nex3/rb-inotify) +courtesy of a custom repo scanner written in go using [rjeczalik/notify](https://github.com/rjeczalik/notify) ## Install @@ -24,7 +24,19 @@ docker run -d -p 8080:80 -v /opt/repo:/repo dgutierrez1287/yum-repo ### Mapping Logs There is a log volume (/logs) that can be mapped to the host machine. In that directory -nginx, supervisord and the custom rudy script (repo_scanner) will log to subdirectories. +nginx, supervisord and the custom go program (repoScanner) will log to subdirectories. + +### Environment Variables + +There are three environment variables that can be set to change the operation of the repoScanner and the entrypoint script to configure the container at runtime + +DEBUG (String, Default: 'true') - This will enable debug logging for the repo scanner program, this should only be run for debugging since the log is very chatty. + +LINUX_HOST (String, Default: 'true') - This container should be run on a linux host for production in which case this should be true, if you wanted to test this on docker for Windows or docker for Mac set this to false. I have found that Docker on a non linux platform will only send the file system notifications available for that system which is a subset of what is available for Linux. This setting will change the repo scanner to only look for the notifications available from Windows and Mac (if its false) or the notifications only available from Linux machines (if its true). + + Note: When running in non linux host mode the create repo tasks will fire more then once even if there is only one file placed in a repo directory, this is because the notifications available for non linux hosts are not a precise as the ones for Linux hosts. please see https://godoc.org/github.com/rjeczalik/notify for more detail. + +SERVE_FILES (String, Default 'true') - This will stop nginx on the container from starting, this should be used if you only want to use the repo scanner portion of the container and will serve the files some other way. This could be accomplished by not mapping the port but this will set nginx to not run thereby saving a few extra resources. ### Using the Repo Directory @@ -43,4 +55,18 @@ let me know any issues that come up with the image or make an issue on the ### Contributing You are invited to contribute any new features or fixes; and I am happy to receive pull -requests. \ No newline at end of file +requests. + +## Disclaimer + +This module is provided without warranty of any kind, the creator(s) and contributors do their best to ensure stablity but can make no warranty about the stability of this docker image in different environments. The creator(s) and contributors reccomend that you test this image and all future releases of this image in your environment before use. + +## ChangeLog + +Version: 1.0.0, 2017-03-22 + +This initial release happened months ago however since I am going to be making some heavy changes I figured I would tag it to maintain the point in time. + +Version: 2.0.0, 2018-03-18 + +This is almost a full re-write with many new features. The ruby script has been replaced by a go program, which will do an inital scan at start up and uses concurrency and file locking. I have added multi-stage build process for the container to bring down the final container size. I have also added controls for not enabling nginx (if file serving will be done another way and to save resouces) amd turning on debugging. diff --git a/build/build.sh b/build/build.sh deleted file mode 100644 index e0e8ef4..0000000 --- a/build/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -docker build -t dgutierrez1287/yum-repo . - -if [[ $? -ne 0 ]]; then - echo "ERROR: error building container" - exit 1 -fi - -exit 0 \ No newline at end of file diff --git a/build/release.sh b/build/release.sh deleted file mode 100644 index 05a7907..0000000 --- a/build/release.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash - diff --git a/changeLog.txt b/changeLog.txt index e751b1c..0d1520f 100644 --- a/changeLog.txt +++ b/changeLog.txt @@ -1,2 +1,4 @@ -Version: 1.0.0 +Version: 1.0.0, 2017-03-22 This initial release happened months ago however since I am going to be making some heavy changes I figured I would tag it to maintain the point in time. +Version: 2.0.0, 2018-03-18 +This is almost a full re-write with many new features. The ruby script has been replaced by a go program, which will do an inital scan at start up and uses concurrency and file locking. I have added multi-stage build process for the container to bring down the final container size. I have also added controls for not enabling nginx (if file serving will be done another way and to save resouces) amd turning on debugging. diff --git a/entrypoint.sh b/entrypoint.sh index 4b44fee..1d55c68 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,9 +1,28 @@ #!/bin/bash -mkdir /root/logs/repo-scanner -mkdir /root/logs/nginx -mkdir /root/logs/supervisord +mkdir /logs/repo-scanner +mkdir /logs/nginx +mkdir /logs/supervisord -chown nginx:nginx /root/logs/nginx +chown nginx:nginx /logs/nginx + +chmod -R 0755 /repo + +if [[ "${SERVE_FILES}" == "true" ]]; then + echo "Serving Files is on" +cat << EOF >> /etc/supervisord.conf +[program:nginx] +priority=10 +directory=/ +command=/usr/sbin/nginx -c /etc/nginx/nginx.conf -g "daemon off;" +user=root +autostart=true +autorestart=true +stopsignal=QUIT +redirect_stderr=true +EOF +else + echo "Serving Files is off" +fi exec /usr/bin/supervisord -n -c /etc/supervisord.conf \ No newline at end of file diff --git a/nginx.conf b/nginx.conf index 8129685..a2b43d1 100644 --- a/nginx.conf +++ b/nginx.conf @@ -13,8 +13,8 @@ http { listen 80 default_server; location /{ - autoindex on; - alias /repo/; + autoindex on; + alias /repo/; } access_log /logs/nginx/access.log; diff --git a/scan_repo.rb b/scan_repo.rb deleted file mode 100755 index 28fea2b..0000000 --- a/scan_repo.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'rb-inotify' -require 'logger' - -$logger = Logger.new('/root/logs/repo-scanner/scanner.log', 'daily') -$logger.level = Logger::INFO - -$top_dir = "/root/repo" - -## -# scan_and_update -# this will go through all the directories containing .rpm files -# and will run a createrepo that will update the yum repo metadata -## -def scan_and_update - - # get repo dir list, clean and dedup - repo_dirs = Dir["#$top_dir/**/*.rpm"] - repo_dirs.map! { |item| item = File.dirname item } - repo_dirs.uniq! - - for dir in repo_dirs - $logger.info("scanning repo #{dir}") - system "createrepo --update #{dir}" - if $?.exitstatus != 0 - $logger.error("Could not update repo #{dir}") - end - end -end - -notifier = INotify::Notifier.new -notifier.watch($top_dir, :recursive, :close_write, :move, :delete) do |event| - if event.name.match(/^.*\.rpm$/) - $logger.info("rpm change detected ... running repo scan") - scan_and_update - end -end - -## MAIN() ## -scan_and_update # run an initial repo scan at startup -notifier.run \ No newline at end of file diff --git a/src/main.go b/src/main.go new file mode 100644 index 0000000..39a713f --- /dev/null +++ b/src/main.go @@ -0,0 +1,215 @@ +package main + +import ( + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "syscall" + + "github.com/Sirupsen/logrus" + "github.com/rjeczalik/notify" + "gopkg.in/dickeyxxx/golock.v1" + "gopkg.in/natefinch/lumberjack.v2" +) + +// Constants +const ( + // RepoDir the parent repo directory + RepoDir = "/repo" + // LogDir the parent log directory for the application + LogDir = "/logs/repo-scanner" + // LockFileName the name of the lockfile + LockFileName = "repoUpdate.lock" +) + +// Global Variables +var ( + // Var to hold the logger instance + log = logrus.New() + // Var to hold the compiled regex for finding an RPM + rpmRegex, _ = regexp.Compile("^.*\\.rpm$") +) + +// Types +type rpmPaths []string + +// init() +// Init function to set up and configure logger +func init() { + + // Check debug env variable and set log level accordingly + if strings.ToLower(os.Getenv("DEBUG")) == "true" { + log.Level = logrus.DebugLevel + } else { + log.Level = logrus.InfoLevel + } + + // Set log output to file and log rotation + log.Out = &lumberjack.Logger{ + Filename: LogDir + "/scanner.log", + MaxSize: 500, + MaxBackups: 3, + MaxAge: 15, + } +} + +// checkErrorAndLog(err error) +// This will check if an error is not nil and will log out the error as fatal. +// This will take in the error as a parameter +// This will not return anything. +func checkErrorAndLog(e error) { + if e != nil { + log.Fatal(e.Error()) + } +} + +// updateRepo(path *string) +// This will run the update for the repo +// This will take in a pointer to the path +// This will return nothing +func updateRepo(path *string) { + + lockfile := *path + "/" + LockFileName + log.Debugf("Trying to create lockfile %s", lockfile) + golock.Lock(lockfile) + + cmd := "createrepo" + cmdArgs := []string{"--update", *path} + + log.Debugf("Running command: %s %s", cmd, strings.Join(cmdArgs, " ")) + + if err := exec.Command(cmd, cmdArgs...).Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + status := exitErr.Sys().(syscall.WaitStatus) + if status != 0 { + log.Errorf("Could not update repo %s", *path) + } + } else { + checkErrorAndLog(err) + } + } else { + log.Debugf("Successfully updated repo %s", *path) + } + + log.Debug("Unlocking directory") + golock.Unlock(lockfile) +} + +// findRpms(path string, info os.FileInfo, err error) +// This is used by filepath.walk on each file visited it will find directories that contain +// rpms and then add them to the list of rpmPaths +// This will take in rpmPaths as a reciever and parameters sent by filepath.walk +// This will return only an error if it can't go somewhere +func (paths *rpmPaths) findRpms(path string, info os.FileInfo, err error) error { + if err != nil { + checkErrorAndLog(err) + return nil + } + // If the location is a directory check for RPMs + if info.IsDir() { + + log.Debugf("Checking directory %s", path) + + // Get a list of files in the directory and loop + files, _ := ioutil.ReadDir(path) + for _, file := range files { + log.Debugf("Checking file %s", file.Name) + + // If the file is an RPM add the directory to the list and break the loop + if rpmRegex.MatchString(file.Name()) { + log.Debugf("Adding %s to rpm directories", path) + *paths = append(*paths, path) + break + } + } + } + return nil +} + +// initialScanAndUpdate() +// This will walk the repo directory and find all directories with rpms and run a repo update +// This will take in nothing +// This will return nothing +func initialScanAndUpdate() { + + log.Info("Running startup update of RPM directories") + + var paths rpmPaths + + // recursively walk the top repo directory to search for RPMs + err := filepath.Walk(RepoDir, paths.findRpms) + checkErrorAndLog(err) + + log.Infof("%d directories found that contain RPMs, running update", len(paths)) + + // Create a channel that is buffered by the number of paths found + ch := make(chan string, len(paths)) + paths.toChannel(ch) + close(ch) + + for rpmPath := range ch { + log.Debugf("Creating go routine to update %s", rpmPath) + go updateRepo(&rpmPath) + } +} + +// (paths *rpmPaths) toChannel(ch chan string) +// This will convert a list of all the paths that contain RPMs into a channel for processing +// This will take in a channel and rpmPaths as a receiver +// This will return nothing +func (paths *rpmPaths) toChannel(ch chan string) { + + log.Debug("Converting RPM directory paths to channel for processing") + + for _, path := range *paths { + log.Debugf("Adding path %s to channel", path) + ch <- path + } +} + +func main() { + + log.Info("Repo scanner starting ...") + + // Run the inital scan and update of all repos + initialScanAndUpdate() + + // Make a buffered channel for file events + log.Debug("Making event channel") + ch := make(chan notify.EventInfo, 100) + + // Start a recursive watcher + // Use different notify types based on docker host type + // If the host is is linux then use the linux specific notify types + if strings.ToLower(os.Getenv("LINUX_HOST")) == "true" { + log.Debug("Linux Docker Host") + err := notify.Watch(RepoDir+"/...", ch, notify.InCloseWrite, notify.InMovedTo, notify.InMovedFrom, notify.InDelete) + checkErrorAndLog(err) + + // if the host is not linux then use the generic notify types + } else { + log.Debug("Non Linux Docker Host") + err := notify.Watch(RepoDir+"/...", ch, notify.Write, notify.Create, notify.Remove, notify.Rename) + checkErrorAndLog(err) + } + + // Forever loop to process file events from the channel + for { + // Block until there is an event + event := <-ch + + log.Debugf("Event %s on %s", event.Event().String(), event.Path()) + + // if the event was an RPM file + if rpmRegex.MatchString(event.Path()) { + + // Get the directory and start update + rpmDir := filepath.Dir(event.Path()) + log.Infof("RPM change detected in %s", rpmDir) + go updateRepo(&rpmDir) + } + } +} diff --git a/supervisord.conf b/supervisord.conf index 64c1d2a..e31dc43 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -4,7 +4,7 @@ pidfile = /run/supervisord.pid logfile = /logs/supervisord/supervisord.log # Set loglevel=debug, only then all logs from child services are printed out # to container logs (and thus available via `docker logs [container]` -loglevel = debug +loglevel = info # These two (unix_http_server, rpcinterface) are needed for supervisorctl to work [inet_http_server] @@ -23,22 +23,13 @@ password = password [include] files = /etc/supervisor.d/*.conf -[program:nginx] -priority=10 +[program:repoScanner] +priority=15 directory=/ -command=/usr/sbin/nginx -c /etc/nginx/nginx.conf -g "daemon off;" +command=/root/repoScanner user=root autostart=true autorestart=true stopsignal=QUIT redirect_stderr=true -[program:reposcan] -priority=15 -directory=/repo -command=ruby /root/scan_repo.rb -user=root -autostart=true -autorestart=true -stopsignal=QUIT -redirect_stderr=true \ No newline at end of file diff --git a/test/test.json b/test/test.json deleted file mode 100644 index 7ebc762..0000000 --- a/test/test.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "dependencyCommands": [ - "yum -y install wget" - ], - "testCases": [ - "triggerRepoScan", - "repoUpdateCheck" - ] -} \ No newline at end of file diff --git a/test/test.sh b/test/test.sh deleted file mode 100644 index f19c2e9..0000000 --- a/test/test.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -## constants -imageName="dgutierrez1287/yum-repo" - -## command line args - - -## -# installTestDeps() -# args: containerID -# This will install dependencies needed -# to test that aren't installed on the container -## -function installTestDeps() { - local _containerID=$1 - - -} - -## -# findUnusedPort() -# This will return the number of an unused port -## -function findUnusedPort() { - - local _lowerPort=$(cat /proc/sys/net/ipv4/ip_local_port_range | awk '{print $1}') - local _upperPort=$(cat /proc/sys/net/ipv4/ip_local_port_range | awk '{print $2}') - local _port - - while :; do - for (( _port = _lowerPort; port <= _upperPort ; _port++ )); do - local _test - _test=$(netstat -ant | grep $_port) - if [[ -z $_test ]]; then - break 2 - fi - done - done - echo $_port -} - -## -# startTestContainer() -# This will start the test container -## -function startTestContainer() { - local _port=$1 - - -} - -## MAIN() ## - -testingPort=$(findUnusedPort) - -