fetch job-run logs on container failure as well

fixes #141

Author: taraspos

core/common.go

@@ -4,11 +4,11 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
-	"github.com/armon/circbuf"
 	"reflect"
 	"strings"
 	"time"
 
+	"github.com/armon/circbuf"
 	docker "github.com/fsouza/go-dockerclient"
 )
 
@@ -21,8 +21,11 @@ var (
 	ErrLocalImageNotFound = errors.New("couldn't find image on the host")
 )
 
-// maximum size of a stdout/stderr stream to be kept in memory and optional stored/sent via mail
-const maxStreamSize = 10 * 1024 * 1024
+const (
+	// maximum size of a stdout/stderr stream to be kept in memory and optional stored/sent via mail
+	maxStreamSize = 10 * 1024 * 1024
+	logPrefix     = "[Job %q (%s)] %s"
+)
 
 type Job interface {
 	GetName() string
@@ -111,19 +114,23 @@ func (c *Context) Stop(err error) {
 }
 
 func (c *Context) Log(msg string) {
-	format := "[Job %q (%s)] %s"
 	args := []interface{}{c.Job.GetName(), c.Execution.ID, msg}
 
 	switch {
 	case c.Execution.Failed:
-		c.Logger.Errorf(format, args...)
+		c.Logger.Errorf(logPrefix, args...)
 	case c.Execution.Skipped:
-		c.Logger.Warningf(format, args...)
+		c.Logger.Warningf(logPrefix, args...)
 	default:
-		c.Logger.Noticef(format, args...)
+		c.Logger.Noticef(logPrefix, args...)
 	}
 }
 
+func (c *Context) Warn(msg string) {
+	args := []interface{}{c.Job.GetName(), c.Execution.ID, msg}
+	c.Logger.Warningf(logPrefix, args...)
+}
+
 // Execution contains all the information relative to a Job execution.
 type Execution struct {
 	ID        string

core/runjob.go

@@ -102,26 +102,32 @@ func (j *RunJob) Run(ctx *Context) error {
 		return err
 	}
 
-	if err := j.watchContainer(container.ID); err != nil {
+	err = j.watchContainer(container.ID)
+	if err == ErrUnexpected {
 		return err
 	}
 
-	if err := j.Client.Logs(docker.LogsOptions{
+	if logsErr := j.Client.Logs(docker.LogsOptions{
 		Container:    container.ID,
 		OutputStream: ctx.Execution.OutputStream,
 		ErrorStream:  ctx.Execution.ErrorStream,
 		Stdout:       true,
 		Stderr:       true,
 		Since:        startTime.Unix(),
-		RawTerminal:  true,
-	}); err != nil {
-		return err
+		RawTerminal:  j.TTY,
+	}); logsErr != nil {
+		ctx.Warn("failed to fetch container logs: " + logsErr.Error())
 	}
 
 	if j.Container == "" {
-		return j.deleteContainer(container.ID)
+		defer func() {
+			if delErr := j.deleteContainer(container.ID); delErr != nil {
+				ctx.Warn("failed to delete container: " + delErr.Error())
+			}
+		}()
 	}
-	return nil
+
+	return err
 }
 
 func (j *RunJob) searchLocalImage() error {

core/scheduler.go

@@ -107,10 +107,12 @@ func (w *jobWrapper) stop(ctx *Context, err error) {
 		errText = ctx.Execution.Error.Error()
 	}
 
-	output := ctx.Execution.OutputStream.Bytes()
+	if ctx.Execution.OutputStream.TotalWritten() > 0 {
+		ctx.Log("StdOut: " + ctx.Execution.OutputStream.String())
+	}
 
-	if len(output) > 0 {
-		ctx.Log("Output: " + string(output))
+	if ctx.Execution.ErrorStream.TotalWritten() > 0 {
+		ctx.Log("StdErr: " + ctx.Execution.ErrorStream.String())
 	}
 
 	msg := fmt.Sprintf(

go.sum

@@ -28,6 +28,7 @@ github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH
 github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -63,7 +64,9 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
@@ -92,6 +95,7 @@ github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2 h1:SPoLlS9qUUnXcIY4pvA4C
 github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2/go.mod h1:TjQg8pa4iejrUrjiz0MCtMV38jdMNW4doKSiBrEvCQQ=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -185,6 +189,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

middlewares/save.go

@@ -1,11 +1,9 @@
 package middlewares
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
-	"os"
+	"io/ioutil"
 	"path/filepath"
 
 	"github.com/mcuadros/ofelia/core"
@@ -60,12 +58,12 @@ func (m *Save) saveToDisk(ctx *core.Context) error {
 	))
 
 	e := ctx.Execution
-	err := m.saveReaderToDisk(bytes.NewReader(e.ErrorStream.Bytes()), fmt.Sprintf("%s.stderr.log", root))
+	err := m.writeFile(e.ErrorStream.Bytes(), fmt.Sprintf("%s.stderr.log", root))
 	if err != nil {
 		return err
 	}
 
-	err = m.saveReaderToDisk(bytes.NewReader(e.OutputStream.Bytes()), fmt.Sprintf("%s.stdout.log", root))
+	err = m.writeFile(e.OutputStream.Bytes(), fmt.Sprintf("%s.stdout.log", root))
 	if err != nil {
 		return err
 	}
@@ -84,19 +82,9 @@ func (m *Save) saveContextToDisk(ctx *core.Context, filename string) error {
 		"Execution": ctx.Execution,
 	}, "", "  ")
 
-	return m.saveReaderToDisk(bytes.NewBuffer(js), filename)
+	return m.writeFile(js, filename)
 }
 
-func (m *Save) saveReaderToDisk(r io.Reader, filename string) error {
-	f, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-
-	defer f.Close()
-	if _, err := io.Copy(f, r); err != nil {
-		return err
-	}
-
-	return nil
+func (m *Save) writeFile(data []byte, filename string) error {
+	return ioutil.WriteFile(filename, data, 0644)
 }