Applied changes from mcuadros#137

Author: rdelcorro

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.1-alpine AS builder
+FROM golang:1.18.1-alpine3.15 AS builder
 
 RUN apk --no-cache add gcc musl-dev
 
@@ -7,7 +7,7 @@ COPY . ${GOPATH}/src/github.com/mcuadros/ofelia
 
 RUN go build -o /go/bin/ofelia .
 
-FROM alpine:3.14.2
+FROM alpine:3.16.0
 
 # this label is required to identify container with ofelia running
 LABEL ofelia.service=true

README.md

@@ -65,8 +65,6 @@ In order to use this type of configurations, ofelia need access to docker socket
 ```sh
 docker run -it --rm \
     -v /var/run/docker.sock:/var/run/docker.sock:ro \
-    --label ofelia.job-local.my-test-job.schedule="@every 5s" \
-    --label ofelia.job-local.my-test-job.command="date" \
         mcuadros/ofelia:latest daemon --docker
 ```
 
@@ -86,9 +84,8 @@ docker run -it --rm \
         nginx
 ```
 
-Now if we start `ofelia` container with the command provided above, it will pickup 2 jobs:
+Now if we start `ofelia` container with the command provided above, it will execute the task:
 
-- Local - `date`
 - Exec  - `uname -a`
 
 Or with docker-compose:
@@ -103,9 +100,6 @@ services:
     command: daemon --docker
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
-    labels:
-      ofelia.job-local.my-test-job.schedule: "@every 5s"
-      ofelia.job-local.my-test-job.command: "date"
 
   nginx:
     image: nginx
@@ -115,6 +109,42 @@ services:
       ofelia.job-exec.datecron.command: "uname -a"
 ```
 
+#### Dynamic docker configuration
+
+You can start ofelia in its own container or on the host itself, and it will magically pick up any container that starts, stops or is modified on the fly.
+In order to achieve this, you simply have to use docker containers with the labels described above and let ofelia take care of the rest. 
+
+#### Hybrid configuration (INI files + Docker)
+
+You can specify part of the configuration on the INI files, such as globals for the middlewares or even declare tasks in there but also merge them with docker.
+The docker labels will be parsed, added and removed on the fly but also, the file config can be used. 
+
+**Use the INI file to:**
+
+- Configure the slack or other middleware integration
+- Configure any global setting
+- Create a job-run so it executes on a new container each time
+
+```ini
+[global]
+slack-webhook = https://myhook.com/auth
+
+[job-run "job-executed-on-new-container"]
+schedule = @hourly
+image = ubuntu:latest
+command = touch /tmp/example
+```
+
+**Use docker to:**
+
+```sh
+docker run -it --rm \
+    --label ofelia.enabled=true \
+    --label ofelia.job-exec.test-exec-job.schedule="@every 5s" \
+    --label ofelia.job-exec.test-exec-job.command="uname -a" \
+        nginx
+```
+
 ### Logging
 **Ofelia** comes with three different logging drivers that can be configured in the `[global]` section:
 - `mail` to send mails

cli/config.go

@@ -1,12 +1,8 @@
 package cli
 
 import (
-	"os"
-
-	docker "github.com/fsouza/go-dockerclient"
 	"github.com/mcuadros/ofelia/core"
 	"github.com/mcuadros/ofelia/middlewares"
-	logging "github.com/op/go-logging"
 
 	defaults "github.com/mcuadros/go-defaults"
 	gcfg "gopkg.in/gcfg.v1"
@@ -20,132 +16,223 @@ const (
 	jobLocal      = "job-local"
 )
 
-var IsDockerEnv bool
-
 // Config contains the configuration
 type Config struct {
 	Global struct {
 		middlewares.SlackConfig `mapstructure:",squash"`
 		middlewares.SaveConfig  `mapstructure:",squash"`
 		middlewares.MailConfig  `mapstructure:",squash"`
 	}
-	ExecJobs    map[string]*ExecJobConfig    `gcfg:"job-exec" mapstructure:"job-exec,squash"`
-	RunJobs     map[string]*RunJobConfig     `gcfg:"job-run" mapstructure:"job-run,squash"`
-	ServiceJobs map[string]*RunServiceConfig `gcfg:"job-service-run" mapstructure:"job-service-run,squash"`
-	LocalJobs   map[string]*LocalJobConfig   `gcfg:"job-local" mapstructure:"job-local,squash"`
+	ExecJobs      map[string]*ExecJobConfig    `gcfg:"job-exec" mapstructure:"job-exec,squash"`
+	RunJobs       map[string]*RunJobConfig     `gcfg:"job-run" mapstructure:"job-run,squash"`
+	ServiceJobs   map[string]*RunServiceConfig `gcfg:"job-service-run" mapstructure:"job-service-run,squash"`
+	LocalJobs     map[string]*LocalJobConfig   `gcfg:"job-local" mapstructure:"job-local,squash"`
+	sh            *core.Scheduler
+	dockerHandler *DockerHandler
+	logger        core.Logger
 }
 
-// BuildFromDockerLabels builds a scheduler using the config from a docker labels
-func BuildFromDockerLabels() (*core.Scheduler, error) {
-	c := &Config{}
-
-	d, err := c.buildDockerClient()
-	if err != nil {
-		return nil, err
-	}
-
-	labels, err := getLabels(d)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := c.buildFromDockerLabels(labels); err != nil {
-		return nil, err
+func NewConfig(logger core.Logger) *Config {
+	c := &Config{
+		ExecJobs:    make(map[string]*ExecJobConfig),
+		RunJobs:     make(map[string]*RunJobConfig),
+		ServiceJobs: make(map[string]*RunServiceConfig),
+		LocalJobs:   make(map[string]*LocalJobConfig),
+		logger:      logger,
 	}
 
-	return c.build()
+	defaults.SetDefaults(c)
+	return c
 }
 
 // BuildFromFile builds a scheduler using the config from a file
-func BuildFromFile(filename string) (*core.Scheduler, error) {
-	c := &Config{}
-	if err := gcfg.ReadFileInto(c, filename); err != nil {
-		return nil, err
-	}
-
-	return c.build()
+func BuildFromFile(filename string, logger core.Logger) (*Config, error) {
+	c := NewConfig(logger)
+	err := gcfg.ReadFileInto(c, filename)
+	return c, err
 }
 
 // BuildFromString builds a scheduler using the config from a string
-func BuildFromString(config string) (*core.Scheduler, error) {
-	c := &Config{}
+func BuildFromString(config string, logger core.Logger) (*Config, error) {
+	c := NewConfig(logger)
 	if err := gcfg.ReadStringInto(c, config); err != nil {
 		return nil, err
 	}
-
-	return c.build()
+	return c, nil
 }
 
-func (c *Config) build() (*core.Scheduler, error) {
-	defaults.SetDefaults(c)
+// Call this only once at app init
+func (c *Config) InitializeApp() error {
+	c.sh = core.NewScheduler(c.logger)
+	c.buildSchedulerMiddlewares(c.sh)
 
-	d, err := c.buildDockerClient()
+	var err error
+	c.dockerHandler, err = NewDockerHandler(c, c.logger)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	sh := core.NewScheduler(c.buildLogger())
-	c.buildSchedulerMiddlewares(sh)
+	// In order to support non dynamic job types such as Local or Run using labels
+	// lets parse the labels and merge the job lists
+	dockerLabels, err := c.dockerHandler.GetDockerLabels()
+	var parsedLabelConfig Config
+	parsedLabelConfig.buildFromDockerLabels(dockerLabels)
+	for name, j := range parsedLabelConfig.RunJobs {
+		c.RunJobs[name] = j
+	}
+	for name, j := range parsedLabelConfig.LocalJobs {
+		c.LocalJobs[name] = j
+	}
+	for name, j := range parsedLabelConfig.ServiceJobs {
+		c.ServiceJobs[name] = j
+	}
 
 	for name, j := range c.ExecJobs {
 		defaults.SetDefaults(j)
-
-		j.Client = d
+		j.Client = c.dockerHandler.GetInternalDockerClient()
 		j.Name = name
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
 	for name, j := range c.RunJobs {
 		defaults.SetDefaults(j)
-
-		j.Client = d
+		j.Client = c.dockerHandler.GetInternalDockerClient()
 		j.Name = name
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
 	for name, j := range c.LocalJobs {
 		defaults.SetDefaults(j)
-
 		j.Name = name
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
 	for name, j := range c.ServiceJobs {
 		defaults.SetDefaults(j)
 		j.Name = name
-		j.Client = d
+		j.Client = c.dockerHandler.GetInternalDockerClient()
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
-	return sh, nil
+	return nil
 }
 
-func (c *Config) buildDockerClient() (*docker.Client, error) {
-	d, err := docker.NewClientFromEnv()
-	if err != nil {
-		return nil, err
+func (c *Config) buildSchedulerMiddlewares(sh *core.Scheduler) {
+	sh.Use(middlewares.NewSlack(&c.Global.SlackConfig))
+	sh.Use(middlewares.NewSave(&c.Global.SaveConfig))
+	sh.Use(middlewares.NewMail(&c.Global.MailConfig))
+}
+
+func (c *Config) dockerLabelsUpdate(labels map[string]map[string]string) {
+	// Get the current labels
+	var parsedLabelConfig Config
+	parsedLabelConfig.buildFromDockerLabels(labels)
+
+	// Calculate the delta execJobs
+	for name, j := range c.ExecJobs {
+		found := false
+		for newJobsName, newJob := range parsedLabelConfig.ExecJobs {
+			// Check if the schedule has changed
+			if name == newJobsName {
+				found = true
+				// There is a slight race condition were a job can be canceled / restarted with different params
+				// so, lets take care of it by simply restarting
+				// For the hash to work properly, we must fill the fields before calling it
+				defaults.SetDefaults(newJob)
+				newJob.Client = c.dockerHandler.GetInternalDockerClient()
+				newJob.Name = newJobsName
+				if newJob.Hash() != j.Hash() {
+					// Remove from the scheduler
+					c.sh.RemoveJob(j)
+					// Add the job back to the scheduler
+					newJob.buildMiddlewares()
+					c.sh.AddJob(newJob)
+					// Update the job config
+					c.ExecJobs[name] = newJob
+				}
+				break
+			}
+		}
+		if !found {
+			// Remove the job
+			c.sh.RemoveJob(j)
+			delete(c.ExecJobs, name)
+		}
 	}
 
-	return d, nil
-}
+	// Check for aditions
+	for newJobsName, newJob := range parsedLabelConfig.ExecJobs {
+		found := false
+		for name := range c.ExecJobs {
+			if name == newJobsName {
+				found = true
+				break
+			}
+		}
+		if !found {
+			defaults.SetDefaults(newJob)
+			newJob.Client = c.dockerHandler.GetInternalDockerClient()
+			newJob.Name = newJobsName
+			newJob.buildMiddlewares()
+			c.sh.AddJob(newJob)
+			c.ExecJobs[newJobsName] = newJob
+		}
+	}
 
-func (c *Config) buildLogger() core.Logger {
-	stdout := logging.NewLogBackend(os.Stdout, "", 0)
-	// Set the backends to be used.
-	logging.SetBackend(stdout)
-	logging.SetFormatter(logging.MustStringFormatter(logFormat))
+	for name, j := range c.RunJobs {
+		found := false
+		for newJobsName, newJob := range parsedLabelConfig.RunJobs {
+			// Check if the schedule has changed
+			if name == newJobsName {
+				found = true
+				// There is a slight race condition were a job can be canceled / restarted with different params
+				// so, lets take care of it by simply restarting
+				// For the hash to work properly, we must fill the fields before calling it
+				defaults.SetDefaults(newJob)
+				newJob.Client = c.dockerHandler.GetInternalDockerClient()
+				newJob.Name = newJobsName
+				if newJob.Hash() != j.Hash() {
+					// Remove from the scheduler
+					c.sh.RemoveJob(j)
+					// Add the job back to the scheduler
+					newJob.buildMiddlewares()
+					c.sh.AddJob(newJob)
+					// Update the job config
+					c.RunJobs[name] = newJob
+				}
+				break
+			}
+		}
+		if !found {
+			// Remove the job
+			c.sh.RemoveJob(j)
+			delete(c.RunJobs, name)
+		}
+	}
 
-	return logging.MustGetLogger("ofelia")
-}
+	// Check for aditions
+	for newJobsName, newJob := range parsedLabelConfig.RunJobs {
+		found := false
+		for name := range c.RunJobs {
+			if name == newJobsName {
+				found = true
+				break
+			}
+		}
+		if !found {
+			defaults.SetDefaults(newJob)
+			newJob.Client = c.dockerHandler.GetInternalDockerClient()
+			newJob.Name = newJobsName
+			newJob.buildMiddlewares()
+			c.sh.AddJob(newJob)
+			c.RunJobs[newJobsName] = newJob
+		}
+	}
 
-func (c *Config) buildSchedulerMiddlewares(sh *core.Scheduler) {
-	sh.Use(middlewares.NewSlack(&c.Global.SlackConfig))
-	sh.Use(middlewares.NewSave(&c.Global.SaveConfig))
-	sh.Use(middlewares.NewMail(&c.Global.MailConfig))
 }
 
 // ExecJobConfig contains all configuration params needed to build a ExecJob