implement crashLoopBackOff for failed processes

README.md

@@ -119,6 +119,18 @@ job "foo" {
 }
 ```
 
+Set `maxAttempts` to `-1` will restart the process "forever".
+
+```hcl
+job "foo" {
+  command = "/usr/local/bin/foo"
+  args = ["bar"]
+  maxAttempts = -1
+  canFail = false
+  workingDirectory = "/some/path"
+}
+```
+
 You can append a custom environment to the process by setting `env`:
 
 ```hcl

pkg/proc/crashloopbackoff.go

@@ -0,0 +1,17 @@
+package proc
+
+import (
+	"time"
+)
+
+// calculates the next backOff based on the current backOff
+func calculateNextBackOff(currBackOff, maxBackoff time.Duration) time.Duration {
+	if currBackOff.Seconds() <= 1 {
+		return 2 * time.Second
+	}
+	next := time.Duration(2*currBackOff.Seconds()) * time.Second
+	if next.Seconds() > maxBackoff.Seconds() {
+		return maxBackoff
+	}
+	return next
+}

pkg/proc/job_common.go

@@ -14,6 +14,13 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+const (
+	// longest duration between two restarts
+	maxBackOff = 300 * time.Second
+	// retries before next backOff interval is used
+	maxBackOffRetries = 5
+)
+
 func (job *CommonJob) Init() {
 	job.restart = false
 	job.stop = false
@@ -48,10 +55,17 @@ func (job *CommonJob) Run(ctx context.Context, _ chan<- error) error {
 	attempts := 0
 	maxAttempts := job.Config.MaxAttempts
 
-	if maxAttempts <= 0 {
+	backOff := 1 * time.Second
+	backOffRetries := 0
+
+	if maxAttempts == 0 {
 		maxAttempts = 3
 	}
 
+	if maxAttempts < 0 {
+		maxAttempts = -1
+	}
+
 	p := make(chan *os.Process)
 	defer close(p)
 
@@ -83,9 +97,22 @@ func (job *CommonJob) Run(ctx context.Context, _ chan<- error) error {
 		}
 
 		attempts++
-		if attempts < maxAttempts {
+		if maxAttempts == -1 || attempts < maxAttempts {
+			currBackOff := backOff
+			backOffRetries++
+			if backOffRetries > maxBackOffRetries {
+				backOff = calculateNextBackOff(currBackOff, maxBackOff)
+				backOffRetries = 0
+			}
+
 			job.phase.Set(JobPhaseReasonCrashLooping)
-			l.WithField("job.maxAttempts", maxAttempts).WithField("job.usedAttempts", attempts).Info("remaining attempts")
+			l.
+				WithField("job.maxAttempts", maxAttempts).
+				WithField("job.usedAttempts", attempts).
+				WithField("job.nextRestartIn", currBackOff.String()).
+				Info("remaining attempts")
+
+			time.Sleep(currBackOff)
 			continue
 		}