Change synclog format

Author: larrabee

cli/cli.go

@@ -77,6 +77,7 @@ type args struct {
 	Workers           uint   `arg:"-w" help:"Workers count"`
 	Debug             bool   `arg:"-d" help:"Show debug logging"`
 	SyncLog           bool   `arg:"--sync-log" help:"Show sync log"`
+	SyncLogFormat     string `arg:"--sync-log-format" help:"Format of sync log. Possible values: json"`
 	ShowProgress      bool   `arg:"--sync-progress,-p" help:"Show sync progress"`
 	OnFail            string `arg:"--on-fail,-f" help:"Action on failed. Possible values: fatal, skip, skipmissing (DEPRECATED, use --error-handling instead)"`
 	ErrorHandlingMask uint8  `arg:"--error-handling" help:"Controls error handling. Sum of the values: 1 for ignoring NotFound errors, 2 for ignoring PermissionDenied errors OR 255 to ignore all errors"`
@@ -111,6 +112,7 @@ func GetCliArgs() (cli argsParsed, err error) {
 	rawCli.ListBuffer = 1000
 	rawCli.RateLimitObjPerSec = 0
 	rawCli.ErrorHandlingMask = 0
+	rawCli.SyncLogFormat = ""
 
 	p := arg.MustParse(&rawCli)
 	cli.args = rawCli
@@ -138,6 +140,12 @@ func GetCliArgs() (cli argsParsed, err error) {
 		p.Fail("--on-fail must be one of \"fatal, skip, skipmissing\"")
 	}
 
+	switch cli.args.SyncLogFormat {
+	case "json":
+	default:
+		p.Fail("--sync-log-format must be one of \"json\"")
+	}
+
 	if rate, ok := parseBandwith(cli.args.RateLimitBandwidth); ok {
 		cli.RateLimitBandwidth = rate
 	} else {

cli/main.go

@@ -49,6 +49,9 @@ func init() {
 	if cli.Debug {
 		log.SetLevel(logrus.DebugLevel)
 	}
+	if cli.SyncLogFormat == "json" {
+		log.SetFormatter(&logrus.JSONFormatter{})
+	}
 	pipeline.Log = log
 	storage.Log = log
 }

cli/print.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/larrabee/s3sync/pipeline"
+	"github.com/sirupsen/logrus"
 	"time"
 )
 
@@ -26,20 +27,40 @@ func printLiveStats(ctx context.Context, syncGroup *pipeline.Group) {
 func printFinalStats(syncGroup *pipeline.Group, status syncStatus) {
 	dur := time.Since(syncGroup.StartTime).Seconds()
 	for _, val := range syncGroup.GetStepsInfo() {
-		log.Infof("%d %s: Input: %d; Output: %d (%.f obj/sec); Errors: %d\n", val.Num, val.Name, val.Stats.Input, val.Stats.Output, float64(val.Stats.Output)/dur, val.Stats.Error)
+		log.WithFields(logrus.Fields{
+			"stepNum":    val.Num,
+			"stepName": val.Name,
+			"InputObj": val.Stats.Input,
+			"OutputObj": val.Stats.Output,
+			"ErrorObj": val.Stats.Error,
+			"InputObjSpeed": float64(val.Stats.Input)/dur,
+			"OutputObjSpeed": float64(val.Stats.Output)/dur,
+		}).Info("Pipeline step finished")
 	}
-	log.Infof("Duration: %s", time.Since(syncGroup.StartTime).String())
+	log.WithFields(logrus.Fields{
+		"durationSec":    time.Since(syncGroup.StartTime).Seconds(),
+	}).Infof("Duration: %s", time.Since(syncGroup.StartTime).String())
 
 	switch status {
 	case syncStatusOk:
-		log.Infof("Sync Done")
+		log.WithFields(logrus.Fields{
+			"status":    status,
+		}).Infof("Sync Done")
 	case syncStatusFailed:
-		log.Error("Sync Failed")
+		log.WithFields(logrus.Fields{
+			"status":    status,
+		}).Error("Sync Failed")
 	case syncStatusAborted:
-		log.Warnf("Sync Aborted")
+		log.WithFields(logrus.Fields{
+			"status":    status,
+		}).Warnf("Sync Aborted")
 	case syncStatusConfError:
-		log.Errorf("Sync Configuration error")
+		log.WithFields(logrus.Fields{
+			"status":    status,
+		}).Errorf("Sync Configuration error")
 	default:
-		log.Warnf("Sync Unknown status")
+		log.WithFields(logrus.Fields{
+			"status":    status,
+		}).Warnf("Sync Unknown status")
 	}
 }

pipeline/collection/misc.go

@@ -26,7 +26,11 @@ var Logger pipeline.StepFn = func(group *pipeline.Group, stepNum int, input <-ch
 	}
 	for obj := range input {
 		if ok {
-			cfg.Infof("Key: %s", *obj.Key)
+			cfg.WithFields(logrus.Fields{
+				"key":    *obj.Key,
+				"size": len(*obj.Content),
+				"Content-Type": *obj.ContentType,
+			}).Infof("Sync file")
 			output <- obj
 		}
 	}