Merge pull request #1507 from diggerhq/feat/render-source-grouping

Update comments in group by source render mode

Author: ZIJ

backend/controllers/github.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"github.com/diggerhq/digger/backend/segment"
 	"github.com/diggerhq/digger/backend/services"
-	comment_updater "github.com/diggerhq/digger/libs/comment_utils/summary"
+	comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
 	"github.com/google/uuid"
 	"log"

backend/controllers/projects.go

@@ -8,6 +8,7 @@ import (
 	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/backend/services"
 	"github.com/diggerhq/digger/backend/utils"
+	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	"github.com/diggerhq/digger/libs/digger_config"
 	"github.com/diggerhq/digger/libs/orchestrator"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
@@ -490,7 +491,8 @@ func SetJobStatusForProject(c *gin.Context) {
 
 	}
 
-	log.Printf("!!!Batch to json struct: %v", res)
+	UpdateCommentsForBatchGroup(&utils.DiggerGithubRealClientProvider{}, batch, res.Jobs)
+
 	c.JSON(http.StatusOK, res)
 }
 
@@ -570,6 +572,53 @@ func CreateRunForProject(c *gin.Context) {
 	c.JSON(http.StatusOK, run.MapToJsonStruct())
 }
 
+func UpdateCommentsForBatchGroup(gh utils.GithubClientProvider, batch *models.DiggerBatch, serializedJobs []orchestrator_scheduler.SerializedJob) error {
+	diggerYmlString := batch.DiggerConfig
+	diggerConfigYml, err := digger_config.LoadDiggerConfigYamlFromString(diggerYmlString)
+	if err != nil {
+		log.Printf("Error loading digger config from batch: %v", err)
+		return fmt.Errorf("error loading digger config from batch: %v", err)
+	}
+
+	if diggerConfigYml.CommentRenderMode != nil &&
+		*diggerConfigYml.CommentRenderMode != digger_config.CommentRenderModeGroupByModule {
+		log.Printf("render mode is not group_by_module, skipping")
+		return nil
+	}
+
+	ghService, _, err := utils.GetGithubService(
+		gh,
+		batch.GithubInstallationId,
+		batch.RepoFullName,
+		batch.RepoOwner,
+		batch.RepoName,
+	)
+
+	var sourceDetails []reporting.SourceDetails
+	err = json.Unmarshal(batch.SourceDetails, &sourceDetails)
+	if err != nil {
+		log.Printf("failed to unmarshall sourceDetails: %v", err)
+		return fmt.Errorf("failed to unmarshall sourceDetails: %v", err)
+	}
+
+	// project_name => terraform output
+	projectToTerraformOutput := make(map[string]string)
+	// TODO: add projectName as a field of Job
+	for _, serialJob := range serializedJobs {
+		job, err := models.DB.GetDiggerJob(serialJob.DiggerJobId)
+		if err != nil {
+			return fmt.Errorf("Could not get digger job: %v", err)
+		}
+		projectToTerraformOutput[serialJob.ProjectName] = job.TerraformOutput
+	}
+
+	for _, detail := range sourceDetails {
+		reporter := reporting.SourceGroupingReporter{serializedJobs, batch.PrNumber, ghService}
+		reporter.UpdateComment(sourceDetails, detail.SourceLocation, projectToTerraformOutput)
+	}
+	return nil
+}
+
 func AutomergePRforBatchIfEnabled(gh utils.GithubClientProvider, batch *models.DiggerBatch) error {
 	diggerYmlString := batch.DiggerConfig
 	diggerConfigYml, err := digger_config.LoadDiggerConfigYamlFromString(diggerYmlString)

backend/models/scheduler.go

@@ -85,7 +85,7 @@ type GithubDiggerJobLink struct {
 	Status              DiggerJobLinkStatus
 }
 
-func (j *DiggerJob) MapToJsonStruct() (interface{}, error) {
+func (j *DiggerJob) MapToJsonStruct() (orchestrator_scheduler.SerializedJob, error) {
 	var job orchestrator.JobJson
 	err := json.Unmarshal(j.SerializedJobSpec, &job)
 	if err != nil {
@@ -104,7 +104,7 @@ func (j *DiggerJob) MapToJsonStruct() (interface{}, error) {
 		ResourcesDeleted: j.DiggerJobSummary.ResourcesDeleted,
 	}, nil
 }
-func (b *DiggerBatch) MapToJsonStruct() (interface{}, error) {
+func (b *DiggerBatch) MapToJsonStruct() (orchestrator_scheduler.SerializedBatch, error) {
 	res := orchestrator_scheduler.SerializedBatch{
 		ID:           b.ID.String(),
 		PrNumber:     b.PrNumber,
@@ -119,14 +119,14 @@ func (b *DiggerBatch) MapToJsonStruct() (interface{}, error) {
 	serializedJobs := make([]orchestrator_scheduler.SerializedJob, 0)
 	jobs, err := DB.GetDiggerJobsForBatch(b.ID)
 	if err != nil {
-		return nil, fmt.Errorf("could not unmarshall digger batch: %v", err)
+		return res, fmt.Errorf("could not unmarshall digger batch: %v", err)
 	}
 	for _, job := range jobs {
 		jobJson, err := job.MapToJsonStruct()
 		if err != nil {
-			return nil, fmt.Errorf("error mapping job to struct (ID: %v); %v", job.ID, err)
+			return res, fmt.Errorf("error mapping job to struct (ID: %v); %v", job.ID, err)
 		}
-		serializedJobs = append(serializedJobs, jobJson.(orchestrator_scheduler.SerializedJob))
+		serializedJobs = append(serializedJobs, jobJson)
 	}
 	res.Jobs = serializedJobs
 	return res, nil

cli/pkg/core/execution/execution.go

@@ -2,6 +2,7 @@ package execution
 
 import (
 	"fmt"
+	"github.com/diggerhq/digger/libs/comment_utils/utils"
 	"github.com/diggerhq/digger/libs/terraform_utils"
 	"github.com/samber/lo"
 	"log"
@@ -15,7 +16,6 @@ import (
 	"github.com/diggerhq/digger/cli/pkg/core/runners"
 	"github.com/diggerhq/digger/cli/pkg/core/storage"
 	"github.com/diggerhq/digger/cli/pkg/core/terraform"
-	"github.com/diggerhq/digger/cli/pkg/core/utils"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	configuration "github.com/diggerhq/digger/libs/digger_config"
 	"github.com/diggerhq/digger/libs/orchestrator"

cli/pkg/digger/digger.go

@@ -3,6 +3,7 @@ package digger
 import (
 	"errors"
 	"fmt"
+	coreutils "github.com/diggerhq/digger/libs/comment_utils/utils"
 	"log"
 	"os"
 	"path"
@@ -19,7 +20,6 @@ import (
 	"github.com/diggerhq/digger/cli/pkg/core/runners"
 	"github.com/diggerhq/digger/cli/pkg/core/storage"
 	"github.com/diggerhq/digger/cli/pkg/core/terraform"
-	coreutils "github.com/diggerhq/digger/cli/pkg/core/utils"
 	"github.com/diggerhq/digger/cli/pkg/locking"
 	"github.com/diggerhq/digger/cli/pkg/usage"
 	utils "github.com/diggerhq/digger/cli/pkg/utils"

cli/pkg/digger/digger_test.go

@@ -281,7 +281,7 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) {
 
 	commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage, planPathProvider)
 
-	assert.Equal(t, []string{"RetrievePlan plan", "Init ", "Apply -lock-timeout=3m", "PublishComment 1 <details open=\"false\"><summary>Apply output</summary>\n\n```terraform\n\n```\n</details>", "Run   echo"}, commandStrings)
+	assert.Equal(t, []string{"RetrievePlan plan", "Init ", "Apply -lock-timeout=3m", "PublishComment 1 <details ><summary>Apply output</summary>\n\n```terraform\n\n```\n</details>", "Run   echo"}, commandStrings)
 }
 
 func TestCorrectCommandExecutionWhenDestroying(t *testing.T) {

cli/pkg/locking/locking.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/diggerhq/digger/libs/comment_utils/utils"
 	"log"
 	"os"
 	"strconv"
@@ -13,7 +14,6 @@ import (
 	"github.com/diggerhq/digger/cli/pkg/aws/envprovider"
 	"github.com/diggerhq/digger/cli/pkg/azure"
 	"github.com/diggerhq/digger/cli/pkg/core/locking"
-	"github.com/diggerhq/digger/cli/pkg/core/utils"
 	"github.com/diggerhq/digger/cli/pkg/gcp"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	"github.com/diggerhq/digger/libs/orchestrator"

libs/comment_utils/reporting/reporting.go

@@ -2,7 +2,7 @@ package reporting
 
 import (
 	"fmt"
-	"github.com/diggerhq/digger/cli/pkg/core/utils"
+	"github.com/diggerhq/digger/libs/comment_utils/utils"
 	"github.com/diggerhq/digger/libs/orchestrator"
 	"log"
 	"strings"

libs/comment_utils/reporting/source_grouping.go

@@ -3,12 +3,14 @@ package reporting
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/diggerhq/digger/libs/comment_utils/utils"
 	"github.com/diggerhq/digger/libs/digger_config"
 	"github.com/diggerhq/digger/libs/orchestrator"
 	"github.com/diggerhq/digger/libs/orchestrator/scheduler"
 	"github.com/diggerhq/digger/libs/terraform_utils"
 	"github.com/samber/lo"
 	"log"
+	"strconv"
 )
 
 type ProjectNameSourceDetail struct {
@@ -25,24 +27,29 @@ type SourceGroupingReporter struct {
 	PrService orchestrator.PullRequestService
 }
 
-func (r SourceGroupingReporter) Report(report string, reportFormatting func(report string) string) (string, string, error) {
+func (r SourceGroupingReporter) UpdateComment(sourceDetails []SourceDetails, location string, terraformOutputs map[string]string) error {
 	jobSpecs, err := scheduler.GetJobSpecs(r.Jobs)
 	if err != nil {
-		return "", "", fmt.Errorf("could not get job specs: %v", err)
+		return fmt.Errorf("could not get job specs: %v", err)
 	}
 
-	//impactedSources := jobSpecs[0].ImpactedSources
-	// TODO: populate from batch field
-	var impactedSources map[string]digger_config.ProjectToSourceMapping
+	sourceDetaiItem, found := lo.Find(sourceDetails, func(item SourceDetails) bool {
+		return item.SourceLocation == location
+	})
+
+	if !found {
+		log.Printf("location not found in sourcedetails list")
+		return fmt.Errorf("location not found in sourcedetails list")
+	}
 
 	projectNameToJobMap, err := scheduler.JobsToProjectMap(r.Jobs)
 	if err != nil {
-		return "", "", fmt.Errorf("could not convert jobs to map: %v", err)
+		return fmt.Errorf("could not convert jobs to map: %v", err)
 	}
 
 	projectNameToJobSpecMap, err := orchestrator.JobsSpecsToProjectMap(jobSpecs)
 	if err != nil {
-		return "", "", fmt.Errorf("could not convert jobs to map: %v", err)
+		return fmt.Errorf("could not convert jobs to map: %v", err)
 	}
 
 	projectNameToFootPrintMap := make(map[string]terraform_utils.TerraformPlanFootprint)
@@ -52,51 +59,50 @@ func (r SourceGroupingReporter) Report(report string, reportFormatting func(repo
 			err := json.Unmarshal(job.PlanFootprint, &footprint)
 			if err != nil {
 				log.Printf("could not unmarshal footprint: %v", err)
-				return "", "", fmt.Errorf("could not unmarshal footprint: %v", err)
+				return fmt.Errorf("could not unmarshal footprint: %v", err)
 			}
 		} else {
 			footprint = terraform_utils.TerraformPlanFootprint{}
 		}
 		projectNameToFootPrintMap[job.ProjectName] = footprint
 	}
 
-	groupsToProjectMap := ImpactedSourcesMapToGroupMapping(impactedSources, projectNameToJobMap, projectNameToJobSpecMap, projectNameToFootPrintMap)
-
-	message := ":construction_worker: Jobs status:\n\n"
-	for sourceLocation, projectSourceDetailList := range groupsToProjectMap {
-		footprints := lo.Map(projectSourceDetailList, func(detail ProjectNameSourceDetail, i int) terraform_utils.TerraformPlanFootprint {
-			return detail.PlanFootPrint
-		})
-		allSimilarInGroup, err := terraform_utils.SimilarityCheck(footprints)
-		if err != nil {
-			return "", "", fmt.Errorf("error performing similar check: %v", err)
+	footprints := lo.FilterMap(sourceDetaiItem.Projects, func(project string, i int) (terraform_utils.TerraformPlanFootprint, bool) {
+		if projectNameToJobMap[project].Status == scheduler.DiggerJobSucceeded {
+			return projectNameToFootPrintMap[project], true
 		}
+		return terraform_utils.TerraformPlanFootprint{}, false
+	})
+	allSimilarInGroup, err := terraform_utils.SimilarityCheck(footprints)
+	if err != nil {
+		return fmt.Errorf("error performing similar check: %v", err)
+	}
 
-		message = message + fmt.Sprintf("# Group: %v (similar: %v)", sourceLocation, allSimilarInGroup)
-		for _, projectSourceDetail := range projectSourceDetailList {
-			job := projectSourceDetail.Job
-			jobSpec := projectSourceDetail.JobSpec
-			isPlan := jobSpec.IsPlan()
-			message = message + fmt.Sprintf("<!-- PROJECTHOLDER %v -->\n", job.ProjectName)
-			message = message + fmt.Sprintf("%v **%v** <a href='%v'>%v</a>%v\n", job.Status.ToEmoji(), job.ProjectName, *job.WorkflowRunUrl, job.Status.ToString(), job.ResourcesSummaryString(isPlan))
-			message = message + fmt.Sprintf("<!-- PROJECTHOLDEREND %v -->\n", job.ProjectName)
+	message := ""
+	message = message + fmt.Sprintf("# Group: %v (similar: %v)\n", location, allSimilarInGroup)
+	for i, project := range sourceDetaiItem.Projects {
+		job := projectNameToJobMap[project]
+		if job.Status != scheduler.DiggerJobSucceeded {
+			continue
 		}
-
+		jobSpec := projectNameToJobSpecMap[project]
+		isPlan := jobSpec.JobType == orchestrator.DiggerCommandPlan
+		expanded := i == 0 || !allSimilarInGroup
+		var commenter func(terraformOutput string) string
+		if isPlan {
+			commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Plan for %v", project), expanded)
+		} else {
+			commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Apply for %v", project), false)
+		}
+		message = message + commenter(terraformOutputs[project]) + "\n"
 	}
 
-	r.PrService.PublishComment(r.PrNumber, message)
-	return "", "", nil
-}
-
-func (reporter SourceGroupingReporter) Flush() (string, string, error) {
-	return "", "", nil
-}
-
-func (reporter SourceGroupingReporter) SupportsMarkdown() bool {
-	return false
-}
-
-func (reporter SourceGroupingReporter) Suppress() error {
+	CommentId, err := strconv.ParseInt(sourceDetaiItem.CommentId, 10, 64)
+	if err != nil {
+		log.Printf("Could not convert commentId to int64: %v", err)
+		return fmt.Errorf("could not convert commentId to int64: %v", err)
+	}
+	r.PrService.EditComment(r.PrNumber, CommentId, message)
 	return nil
 }
 

libs/comment_utils/summary/utils.go renamed to libs/comment_utils/reporting/utils.go

@@ -1,8 +1,7 @@
-package comment_updater
+package reporting
 
 import (
 	"fmt"
-	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
 	dg_github "github.com/diggerhq/digger/libs/orchestrator/github"
 	"log"
@@ -26,10 +25,10 @@ func PostInitialSourceComments(ghService *dg_github.GithubService, prNumber int,
 		}
 	}
 	for location, projects := range locations {
-		reporter := reporting.CiReporter{
+		reporter := CiReporter{
 			PrNumber:       prNumber,
 			CiService:      ghService,
-			ReportStrategy: reporting.CommentPerRunStrategy{fmt.Sprintf("Report for location: %v", location), time.Now()},
+			ReportStrategy: CommentPerRunStrategy{fmt.Sprintf("Report for location: %v", location), time.Now()},
 		}
 		commentId, _, err := reporter.Report("Comment Reporter", func(report string) string { return "" })
 		if err != nil {

cli/pkg/core/utils/comments.go renamed to libs/comment_utils/utils/comments.go

@@ -3,14 +3,20 @@ package utils
 import "fmt"
 
 func GetTerraformOutputAsCollapsibleComment(summary string, open bool) func(string) string {
+	var openTag string
+	if open {
+		openTag = "open=\"true\""
+	} else {
+		openTag = ""
+	}
 
 	return func(comment string) string {
-		return fmt.Sprintf(`<details open="%v"><summary>`+summary+`</summary>
+		return fmt.Sprintf(`<details %v><summary>`+summary+`</summary>
 
 `+"```terraform"+`
 `+comment+`
 `+"```"+`
-</details>`, open)
+</details>`, openTag)
 	}
 }