also append resource summary to apply output (#1708)

* also append resource summary to apply output

Author: motatoes

backend/controllers/projects.go

@@ -319,7 +319,7 @@ func RunHistoryForProject(c *gin.Context) {
 type SetJobStatusRequest struct {
 	Status          string                                  `json:"status"`
 	Timestamp       time.Time                               `json:"timestamp"`
-	JobSummary      *terraform_utils.PlanSummary            `json:"job_summary"`
+	JobSummary      *terraform_utils.TerraformSummary       `json:"job_summary"`
 	Footprint       *terraform_utils.TerraformPlanFootprint `json:"job_plan_footprint"`
 	PrCommentUrl    string                                  `json:"pr_comment_url"`
 	TerraformOutput string                                  `json:"terraform_output""`

cli/pkg/digger/digger.go

@@ -135,16 +135,13 @@ func RunJobs(jobs []orchestrator.Job, prService ci.PullRequestService, orgServic
 		projectNameForBackendReporting := currentJob.ProjectName
 		// TODO: handle the apply result summary as well to report it to backend. Possibly reporting changed resources as well
 		// Some kind of generic terraform operation summary might need to be introduced
-		var planResult *execution.DiggerExecutorPlanResult = nil
-		if exectorResults[0].PlanResult != nil {
-			planResult = exectorResults[0].PlanResult
-		}
+		summary := exectorResults[0].GetTerraformSummary()
 		terraformOutput := ""
 		if reportTerraformOutput {
 			terraformOutput = exectorResults[0].TerraformOutput
 		}
 		prNumber := *currentJob.PullRequestNumber
-		batchResult, err := backendApi.ReportProjectJobStatus(repoNameForBackendReporting, projectNameForBackendReporting, jobId, "succeeded", time.Now(), planResult, jobPrCommentUrl, terraformOutput)
+		batchResult, err := backendApi.ReportProjectJobStatus(repoNameForBackendReporting, projectNameForBackendReporting, jobId, "succeeded", time.Now(), &summary, "", jobPrCommentUrl, terraformOutput)
 		if err != nil {
 			log.Printf("error reporting Job status: %v.\n", err)
 			return false, false, fmt.Errorf("error while running command: %v", err)
@@ -328,6 +325,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 				return nil, msg, fmt.Errorf(msg)
 			}
 			result := execution.DiggerExecutorResult{
+				OperationType:   execution.DiggerOparationTypePlan,
 				TerraformOutput: plan,
 				PlanResult: &execution.DiggerExecutorPlanResult{
 					PlanSummary:   *planSummary,
@@ -404,7 +402,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 
 			// Running apply
 
-			applyPerformed, output, err := diggerExecutor.Apply()
+			applySummary, applyPerformed, output, err := diggerExecutor.Apply()
 			if err != nil {
 				//TODO reuse executor error handling
 				log.Printf("Failed to Run digger apply command. %v", err)
@@ -424,8 +422,11 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 				appliesPerProject[job.ProjectName] = true
 			}
 			result := execution.DiggerExecutorResult{
+				OperationType:   execution.DiggerOparationTypeApply,
 				TerraformOutput: output,
-				ApplyResult:     &execution.DiggerExecutorApplyResult{},
+				ApplyResult: &execution.DiggerExecutorApplyResult{
+					ApplySummary: *applySummary,
+				},
 			}
 			return &result, output, nil
 		}
@@ -662,7 +663,7 @@ func RunJob(
 			if err != nil {
 				log.Printf("Failed to send usage report. %v", err)
 			}
-			_, output, err := diggerExecutor.Apply()
+			_, _, output, err := diggerExecutor.Apply()
 			if err != nil {
 				msg := fmt.Sprintf("Failed to Run digger apply command. %v", err)
 				log.Printf(msg)

cli/pkg/spec/spec.go

@@ -17,7 +17,7 @@ import (
 
 func reportError(spec spec.Spec, backendApi backend2.Api, message string, err error) {
 	log.Printf(message)
-	_, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "")
+	_, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "", "")
 	if reportingError != nil {
 		usage.ReportErrorAndExit(spec.VCS.RepoOwner, fmt.Sprintf("Failed to run commands. %v", err), 5)
 	}
@@ -131,7 +131,7 @@ func RunSpec(
 	jobs := []scheduler.Job{job}
 
 	fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
-	_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "")
+	_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "", "")
 	if err != nil {
 		message := fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err)
 		reportError(spec, backendApi, message, err)
@@ -152,7 +152,7 @@ func RunSpec(
 	reportTerraformOutput := spec.Reporter.ReportTerraformOutput
 	allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, true, reportTerraformOutput, commentId, currentDir)
 	if !allAppliesSuccess || err != nil {
-		serializedBatch, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "")
+		serializedBatch, reportingError := backendApi.ReportProjectJobStatus(spec.VCS.RepoName, spec.Job.ProjectName, spec.JobId, "failed", time.Now(), nil, "", "", "")
 		if reportingError != nil {
 			message := fmt.Sprintf("Failed run commands. %s", err)
 			reportError(spec, backendApi, message, err)

libs/backendapi/backend.go

@@ -1,15 +1,15 @@
 package backendapi
 
 import (
-	"github.com/diggerhq/digger/libs/execution"
 	"github.com/diggerhq/digger/libs/scheduler"
+	"github.com/diggerhq/digger/libs/terraform_utils"
 	"time"
 )
 
 type Api interface {
 	ReportProject(repo string, projectName string, configuration string) error
 	ReportProjectRun(repo string, projectName string, startedAt time.Time, endedAt time.Time, status string, command string, output string) error
-	ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error)
+	ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error)
 	UploadJobArtefact(zipLocation string) (*int, *string, error)
 	DownloadJobArtefact(downloadTo string) (*string, error)
 }

libs/backendapi/diggerapi.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/diggerhq/digger/libs/execution"
 	"github.com/diggerhq/digger/libs/scheduler"
 	"github.com/diggerhq/digger/libs/terraform_utils"
 	"io"
@@ -30,7 +29,7 @@ func (n NoopApi) ReportProjectRun(namespace string, projectName string, startedA
 	return nil
 }
 
-func (n NoopApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
+func (n NoopApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
 	return nil, nil
 }
 
@@ -130,26 +129,26 @@ func (d DiggerApi) ReportProjectRun(namespace string, projectName string, starte
 	return nil
 }
 
-func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, planResult *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
+func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
 	u, err := url.Parse(d.DiggerHost)
 	if err != nil {
 		log.Fatalf("Not able to parse digger cloud url: %v", err)
 	}
 
 	var planSummaryJson interface{}
-	var planFootprint *terraform_utils.TerraformPlanFootprint
-	if planResult == nil {
+	var planFootprint *terraform_utils.TerraformPlanFootprint = &terraform_utils.TerraformPlanFootprint{}
+	if summary == nil {
 		log.Printf("Warning: nil passed to plan result, sending empty")
 		planSummaryJson = nil
 		planFootprint = nil
 	} else {
-		planJson := planResult.TerraformJson
-		planSummary := planResult.PlanSummary
+		planSummary := summary
 		planSummaryJson = planSummary.ToJson()
-		planFootprint, err = terraform_utils.GetPlanFootprint(planJson)
-		if err != nil {
-			log.Printf("Error, could not get footprint from json plan: %v", err)
-			return nil, fmt.Errorf("error, could not get footprint from json plan: %v", err)
+		if planJson != "" {
+			planFootprint, err = terraform_utils.GetPlanFootprint(planJson)
+			if err != nil {
+				log.Printf("Error, could not get footprint from json plan: %v", err)
+			}
 		}
 	}
 

libs/backendapi/mocks.go

@@ -1,8 +1,8 @@
 package backendapi
 
 import (
-	"github.com/diggerhq/digger/libs/execution"
 	"github.com/diggerhq/digger/libs/scheduler"
+	"github.com/diggerhq/digger/libs/terraform_utils"
 	"time"
 )
 
@@ -17,7 +17,7 @@ func (t MockBackendApi) ReportProjectRun(repo string, projectName string, starte
 	return nil
 }
 
-func (t MockBackendApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
+func (t MockBackendApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *terraform_utils.TerraformSummary, planJson string, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
 	return nil, nil
 }
 

libs/comment_utils/summary/updater.go

@@ -24,24 +24,15 @@ func (b BasicCommentUpdater) UpdateComment(jobs []scheduler.SerializedJob, prNum
 	}
 	firstJobSpec := jobSpecs[0]
 	jobType := firstJobSpec.JobType
-	isPlan := jobType == string(scheduler.DiggerCommandPlan)
 	jobTypeTitle := cases.Title(language.AmericanEnglish).String(string(jobType))
 	message := ""
-	if isPlan {
-		message = message + fmt.Sprintf("| Project | Status | %v | + | ~ | - |\n", jobTypeTitle)
-		message = message + fmt.Sprintf("|---------|--------|------|---|---|---|\n")
-	} else {
-		message = message + fmt.Sprintf("| Project | Status | %v |\n", jobTypeTitle)
-		message = message + fmt.Sprintf("|---------|--------|-------|\n")
-	}
+	message = message + fmt.Sprintf("| Project | Status | %v | + | ~ | - |\n", jobTypeTitle)
+	message = message + fmt.Sprintf("|---------|--------|------|---|---|---|\n")
+
 	for i, job := range jobs {
 		jobSpec := jobSpecs[i]
 		prCommentUrl := job.PRCommentUrl
-		if isPlan {
-			message = message + fmt.Sprintf("|%v **%v** |<a href='%v'>%v</a> | <a href='%v'>%v</a> | %v | %v | %v|\n", job.Status.ToEmoji(), jobSpec.ProjectName, *job.WorkflowRunUrl, job.Status.ToString(), prCommentUrl, jobTypeTitle, job.ResourcesCreated, job.ResourcesUpdated, job.ResourcesDeleted)
-		} else {
-			message = message + fmt.Sprintf("|%v **%v** |<a href='%v'>%v</a> | <a href='%v'>%v</a> |\n", job.Status.ToEmoji(), jobSpec.ProjectName, *job.WorkflowRunUrl, job.Status.ToString(), prCommentUrl, jobTypeTitle)
-		}
+		message = message + fmt.Sprintf("|%v **%v** |<a href='%v'>%v</a> | <a href='%v'>%v</a> | %v | %v | %v|\n", job.Status.ToEmoji(), jobSpec.ProjectName, *job.WorkflowRunUrl, job.Status.ToString(), prCommentUrl, jobTypeTitle, job.ResourcesCreated, job.ResourcesUpdated, job.ResourcesDeleted)
 	}
 
 	prService.EditComment(prNumber, prCommentId, message)

libs/execution/execution.go

@@ -20,8 +20,8 @@ import (
 )
 
 type Executor interface {
-	Plan() (*terraform_utils.PlanSummary, bool, bool, string, string, error)
-	Apply() (bool, string, error)
+	Plan() (*terraform_utils.TerraformSummary, bool, bool, string, string, error)
+	Apply() (*terraform_utils.TerraformSummary, bool, string, error)
 	Destroy() (bool, error)
 }
 
@@ -30,7 +30,7 @@ type LockingExecutorWrapper struct {
 	Executor    Executor
 }
 
-func (l LockingExecutorWrapper) Plan() (*terraform_utils.PlanSummary, bool, bool, string, string, error) {
+func (l LockingExecutorWrapper) Plan() (*terraform_utils.TerraformSummary, bool, bool, string, string, error) {
 	plan := ""
 	locked, err := l.ProjectLock.Lock()
 	if err != nil {
@@ -44,17 +44,17 @@ func (l LockingExecutorWrapper) Plan() (*terraform_utils.PlanSummary, bool, bool
 	}
 }
 
-func (l LockingExecutorWrapper) Apply() (bool, string, error) {
+func (l LockingExecutorWrapper) Apply() (*terraform_utils.TerraformSummary, bool, string, error) {
 	locked, err := l.ProjectLock.Lock()
 	if err != nil {
 		msg := fmt.Sprintf("digger apply, error locking project: %v", err)
-		return false, msg, fmt.Errorf(msg)
+		return nil, false, msg, fmt.Errorf(msg)
 	}
 	log.Printf("Lock result: %t\n", locked)
 	if locked {
 		return l.Executor.Apply()
 	} else {
-		return false, "couldn't lock ", nil
+		return nil, false, "couldn't lock ", nil
 	}
 }
 
@@ -103,20 +103,37 @@ type DiggerExecutor struct {
 	PlanPathProvider  PlanPathProvider
 }
 
+type DiggerOperationType string
+
+var DiggerOparationTypePlan DiggerOperationType = "plan"
+var DiggerOparationTypeApply DiggerOperationType = "apply"
+
 type DiggerExecutorResult struct {
+	OperationType   DiggerOperationType
 	TerraformOutput string
 	PlanResult      *DiggerExecutorPlanResult
 	ApplyResult     *DiggerExecutorApplyResult
 }
 
 type DiggerExecutorApplyResult struct {
+	ApplySummary terraform_utils.TerraformSummary
 }
 
 type DiggerExecutorPlanResult struct {
-	PlanSummary   terraform_utils.PlanSummary
+	PlanSummary   terraform_utils.TerraformSummary
 	TerraformJson string
 }
 
+func (d DiggerExecutorResult) GetTerraformSummary() terraform_utils.TerraformSummary {
+	var summary terraform_utils.TerraformSummary
+	if d.OperationType == DiggerOparationTypePlan && d.PlanResult != nil {
+		summary = d.PlanResult.PlanSummary
+	} else if d.OperationType == DiggerOparationTypeApply && d.ApplyResult != nil {
+		summary = d.ApplyResult.ApplySummary
+	}
+	return summary
+}
+
 type PlanPathProvider interface {
 	LocalPlanFilePath() string
 	StoredPlanFilePath() string
@@ -180,10 +197,10 @@ func (d DiggerExecutor) RetrievePlanJson() (string, error) {
 	}
 }
 
-func (d DiggerExecutor) Plan() (*terraform_utils.PlanSummary, bool, bool, string, string, error) {
+func (d DiggerExecutor) Plan() (*terraform_utils.TerraformSummary, bool, bool, string, string, error) {
 	plan := ""
 	terraformPlanOutput := ""
-	planSummary := &terraform_utils.PlanSummary{}
+	planSummary := &terraform_utils.TerraformSummary{}
 	isEmptyPlan := true
 	var planSteps []scheduler.Step
 
@@ -217,7 +234,7 @@ func (d DiggerExecutor) Plan() (*terraform_utils.PlanSummary, bool, bool, string
 			showArgs := []string{"-no-color", "-json", d.PlanPathProvider.LocalPlanFilePath()}
 			terraformPlanOutput, _, _ = d.TerraformExecutor.Show(showArgs, d.CommandEnvVars)
 
-			isEmptyPlan, planSummary, err = terraform_utils.GetPlanSummary(terraformPlanOutput)
+			isEmptyPlan, planSummary, err = terraform_utils.GetSummaryFromPlanJson(terraformPlanOutput)
 			if err != nil {
 				return nil, false, false, "", "", fmt.Errorf("error checking for empty plan: %v", err)
 			}
@@ -284,14 +301,15 @@ func reportError(r reporting.Reporter, stderr string) {
 	}
 }
 
-func (d DiggerExecutor) Apply() (bool, string, error) {
+func (d DiggerExecutor) Apply() (*terraform_utils.TerraformSummary, bool, string, error) {
 	var applyOutput string
 	var plansFilename *string
+	summary := terraform_utils.TerraformSummary{}
 	if d.PlanStorage != nil {
 		var err error
 		plansFilename, err = d.PlanStorage.RetrievePlan(d.PlanPathProvider.LocalPlanFilePath(), d.PlanPathProvider.ArtifactName(), d.PlanPathProvider.StoredPlanFilePath())
 		if err != nil {
-			return false, "", fmt.Errorf("error retrieving plan: %v", err)
+			return nil, false, "", fmt.Errorf("error retrieving plan: %v", err)
 		}
 	}
 
@@ -315,18 +333,24 @@ func (d DiggerExecutor) Apply() (bool, string, error) {
 			stdout, stderr, err := d.TerraformExecutor.Init(step.ExtraArgs, d.StateEnvVars)
 			if err != nil {
 				reportTerraformError(d.Reporter, stderr)
-				return false, stdout, fmt.Errorf("error running init: %v", err)
+				return nil, false, stdout, fmt.Errorf("error running init: %v", err)
 			}
 		}
 		if step.Action == "apply" {
 			applyArgs := []string{"-lock-timeout=3m"}
 			applyArgs = append(applyArgs, step.ExtraArgs...)
 			stdout, stderr, err := d.TerraformExecutor.Apply(applyArgs, plansFilename, d.CommandEnvVars)
 			applyOutput = cleanupTerraformApply(true, err, stdout, stderr)
+
 			reportTerraformApplyOutput(d.Reporter, d.projectId(), applyOutput)
 			if err != nil {
 				reportApplyError(d.Reporter, err)
-				return false, stdout, fmt.Errorf("error executing apply: %v", err)
+				return nil, false, stdout, fmt.Errorf("error executing apply: %v", err)
+			}
+
+			summary, err = terraform_utils.GetSummaryFromTerraformApplyOutput(stdout)
+			if err != nil {
+				log.Printf("Warning: get summary from apply output failed: %v", err)
 			}
 		}
 		if step.Action == "run" {
@@ -338,12 +362,12 @@ func (d DiggerExecutor) Apply() (bool, string, error) {
 			log.Printf("Running %v for **%v**\n", step.Value, d.ProjectNamespace+"#"+d.ProjectName)
 			_, stderr, err := d.CommandRunner.Run(d.ProjectPath, step.Shell, commands, d.RunEnvVars)
 			if err != nil {
-				return false, stderr, fmt.Errorf("error running command: %v", err)
+				return nil, false, stderr, fmt.Errorf("error running command: %v", err)
 			}
 		}
 	}
 	reportAdditionalOutput(d.Reporter, d.projectId())
-	return true, applyOutput, nil
+	return &summary, true, applyOutput, nil
 }
 
 func reportApplyError(r reporting.Reporter, err error) {