From f9089bfe2725b3b135e0225cf8995d55ed1e62ea Mon Sep 17 00:00:00 2001 From: motatoes Date: Sat, 14 Dec 2024 18:49:03 +0000 Subject: [PATCH 1/7] wip generation --- backend/controllers/github.go | 37 +++++++++++++++++++ backend/utils/ai.go | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 backend/utils/ai.go diff --git a/backend/controllers/github.go b/backend/controllers/github.go index d6ec08e0..170adc53 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -744,6 +744,43 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu return fmt.Errorf("error getting digger config") } + // terraform code generator + if os.Getenv("DIGGER_GENERATION_ENABLED") == "1" { + if strings.HasPrefix(*payload.Comment.Body, "digger generate") { + projectName := ci.ParseProjectName(*payload.Comment.Body) + if projectName == "" { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: generate requires argument -p : %v", err)) + log.Printf("missing project in command: %v", *payload.Comment.Body) + return fmt.Errorf("generate requires argument -p : %v", err) + } + + project := config.GetProject(projectName) + if project == nil { + commentReporterManager.UpdateComment(fmt.Sprintf("could not find project %v in digger.yml", projectName)) + log.Printf("could not find project %v in digger.yml", projectName) + return fmt.Errorf("could not find project %v in digger.yml", projectName) + } + + generationEndpoint := os.Getenv("DIGGER_GENERATION_ENDPOINT") + if generationEndpoint == "" { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify")) + log.Printf("server does not have generation endpoint configured, please verify") + return fmt.Errorf("server does not have generation endpoint configured, please verify") + } + webhookSecret := os.Getenv("DIGGER_GENERATION_WEBHOOK_SECRET") + appCode := "mycode" + terraformCode, err := utils.GenerateTerraformCode(appCode, generationEndpoint, webhookSecret) + if err != nil { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify")) + log.Printf("server does not have generation endpoint configured, please verify") + return fmt.Errorf("server does not have generation endpoint configured, please verify") + } + // comment terraform code to project dir + //project.Dir + log.Printf(terraformCode) + } + } + commentIdStr := strconv.FormatInt(userCommentId, 10) err = ghService.CreateCommentReaction(commentIdStr, string(dg_github.GithubCommentEyesReaction)) if err != nil { diff --git a/backend/utils/ai.go b/backend/utils/ai.go new file mode 100644 index 00000000..38fee498 --- /dev/null +++ b/backend/utils/ai.go @@ -0,0 +1,69 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/diggerhq/digger/libs/digger_config" + "io" + "net/http" +) + +func GenerateTerraformCode(appCode string, generationEndpoint string, webhookSecret string) (string, error) { + + payload := map[string]string{ + "code": appCode, + } + + // Convert payload to JSON + jsonData, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("Error marshalling JSON: %v\n", err) + } + + // Create request + req, err := http.NewRequest("POST", generationEndpoint, bytes.NewBuffer(jsonData)) + if err != nil { + return "", fmt.Errorf("Error creating request: %v\n", err) + } + + // Set headers + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Webhook-Secret", webhookSecret) // Replace with your webhook secret + + // Make the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("Error making request: %v\n", err) + } + defer resp.Body.Close() + + // Read response + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("Error reading response: %v\n", err) + } + + // Print response + if resp.StatusCode == 400 { + return "", fmt.Errorf("unable to generate terraform code from the code available, is it valid application code") + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("unexpected error occured while generating code") + } + + type GeneratorResponse struct { + Result string `json:"result"` + } + + var response GeneratorResponse + err = json.Unmarshal(body, &resp) + if err != nil { + return "", fmt.Errorf("unable to parse generator response: %v", err) + } + + return response.Result, nil + +} From 0c142a43c141e0894bc86e9fe01587dd3d678abb Mon Sep 17 00:00:00 2001 From: motatoes Date: Sat, 14 Dec 2024 19:03:16 +0000 Subject: [PATCH 2/7] fix build --- backend/utils/ai.go | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/utils/ai.go b/backend/utils/ai.go index 38fee498..7f8c99c3 100644 --- a/backend/utils/ai.go +++ b/backend/utils/ai.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/diggerhq/digger/libs/digger_config" "io" "net/http" ) From 767e8c683b928c916f55b5a2aa5df02fb8f0d70c Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 16 Dec 2024 10:01:23 +0000 Subject: [PATCH 3/7] commit new terraform --- backend/controllers/github.go | 145 ++++++++++++++++++++++++++++++---- backend/utils/ai.go | 3 +- 2 files changed, 132 insertions(+), 16 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 170adc53..cc67f221 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -6,6 +6,18 @@ import ( "encoding/json" "errors" "fmt" + "log" + "math/rand" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "github.com/davecgh/go-spew/spew" "github.com/diggerhq/digger/backend/ci_backends" config2 "github.com/diggerhq/digger/backend/config" @@ -19,16 +31,6 @@ import ( orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/google/uuid" "gorm.io/gorm" - "log" - "math/rand" - "net/http" - "net/url" - "os" - "path" - "reflect" - "slices" - "strconv" - "strings" "github.com/diggerhq/digger/backend/middleware" "github.com/diggerhq/digger/backend/models" @@ -761,6 +763,8 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu return fmt.Errorf("could not find project %v in digger.yml", projectName) } + commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded project")) + generationEndpoint := os.Getenv("DIGGER_GENERATION_ENDPOINT") if generationEndpoint == "" { commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify")) @@ -768,16 +772,127 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu return fmt.Errorf("server does not have generation endpoint configured, please verify") } webhookSecret := os.Getenv("DIGGER_GENERATION_WEBHOOK_SECRET") - appCode := "mycode" + + // Get all code content from the repository at a specific commit + getCodeFromCommit := func(ghService *dg_github.GithubService, repoOwner, repoName string, commitSha *string, projectDir string) (string, error) { + // Get the commit's changes compared to default branch + comparison, _, err := ghService.Client.Repositories.CompareCommits( + context.Background(), + repoOwner, + repoName, + defaultBranch, + *commitSha, + nil, + ) + if err != nil { + return "", fmt.Errorf("error comparing commits: %v", err) + } + + var appCode strings.Builder + for _, file := range comparison.Files { + if file.Patch == nil { + continue // Skip files without patches + } + log.Printf("file patch: %v", *file.Patch) + if *file.Additions > 0 { + lines := strings.Split(*file.Patch, "\n") + for _, line := range lines { + if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") { + appCode.WriteString(strings.TrimPrefix(line, "+")) + appCode.WriteString("\n") + } + } + } + appCode.WriteString("\n") + } + + if appCode.Len() == 0 { + return "", fmt.Errorf("no changes found in commit: %s", *commitSha) + } + + return appCode.String(), nil + } + + appCode, err := getCodeFromCommit(ghService, repoOwner, repoName, commitSha, project.Dir) + if err != nil { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get code content: %v", err)) + log.Printf("Error getting code content: %v", err) + return fmt.Errorf("error getting code content: %v", err) + } + + commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded code from commit")) + + log.Printf("the app code is: %v", appCode) + + commentReporterManager.UpdateComment(fmt.Sprintf("Generating terraform...")) terraformCode, err := utils.GenerateTerraformCode(appCode, generationEndpoint, webhookSecret) if err != nil { - commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify")) - log.Printf("server does not have generation endpoint configured, please verify") - return fmt.Errorf("server does not have generation endpoint configured, please verify") + commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not generate terraform code: %v", err)) + log.Printf("could not generate terraform code: %v", err) + return fmt.Errorf("could not generate terraform code: %v", err) } + + commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Generated terraform")) + // comment terraform code to project dir //project.Dir - log.Printf(terraformCode) + log.Printf("terraform code is %v", terraformCode) + + baseTree, _, err := ghService.Client.Git.GetTree(context.Background(), repoOwner, repoName, *commitSha, false) + if err != nil { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get base tree: %v", err)) + log.Printf("Error getting base tree: %v", err) + return fmt.Errorf("error getting base tree: %v", err) + } + + // Create a new tree with the new file + treeEntries := []*github.TreeEntry{ + { + Path: github.String(filepath.Join(project.Dir, fmt.Sprintf("generated_%v.tf", issueNumber))), + Mode: github.String("100644"), + Type: github.String("blob"), + Content: github.String(terraformCode), + }, + } + + newTree, _, err := ghService.Client.Git.CreateTree(context.Background(), repoOwner, repoName, *baseTree.SHA, treeEntries) + if err != nil { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to create new tree: %v", err)) + log.Printf("Error creating new tree: %v", err) + return fmt.Errorf("error creating new tree: %v", err) + } + + // Create the commit + commitMsg := fmt.Sprintf("Add generated Terraform code for %v", projectName) + commit := &github.Commit{ + Message: &commitMsg, + Tree: newTree, + Parents: []*github.Commit{{SHA: commitSha}}, + } + + newCommit, _, err := ghService.Client.Git.CreateCommit(context.Background(), repoOwner, repoName, commit, nil) + if err != nil { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to commit Terraform file: %v", err)) + log.Printf("Error committing Terraform file: %v", err) + return fmt.Errorf("error committing Terraform file: %v", err) + } + + // Update the reference to point to the new commit + ref := &github.Reference{ + Ref: github.String(fmt.Sprintf("refs/heads/%s", *branch)), + Object: &github.GitObject{ + SHA: newCommit.SHA, + }, + } + _, _, err = ghService.Client.Git.UpdateRef(context.Background(), repoOwner, repoName, ref, false) + if err != nil { + commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to update branch reference: %v", err)) + log.Printf("Error updating branch reference: %v", err) + return fmt.Errorf("error updating branch reference: %v", err) + } + + commentReporterManager.UpdateComment(":white_check_mark: Successfully generated and committed Terraform code") + return nil } } diff --git a/backend/utils/ai.go b/backend/utils/ai.go index 7f8c99c3..1c6f5239 100644 --- a/backend/utils/ai.go +++ b/backend/utils/ai.go @@ -55,10 +55,11 @@ func GenerateTerraformCode(appCode string, generationEndpoint string, webhookSec type GeneratorResponse struct { Result string `json:"result"` + Status string `json:"status"` } var response GeneratorResponse - err = json.Unmarshal(body, &resp) + err = json.Unmarshal(body, &response) if err != nil { return "", fmt.Errorf("unable to parse generator response: %v", err) } From eab45d56f53f16e63fdd6b2392a708b1ce82b18d Mon Sep 17 00:00:00 2001 From: Mohamed Habib Date: Tue, 17 Dec 2024 12:35:17 +0000 Subject: [PATCH 4/7] Update backend/controllers/github.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- backend/controllers/github.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index cc67f221..8315b1ea 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -775,6 +775,8 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu // Get all code content from the repository at a specific commit getCodeFromCommit := func(ghService *dg_github.GithubService, repoOwner, repoName string, commitSha *string, projectDir string) (string, error) { + const MaxPatchSize = 1024 * 1024 // 1MB limit + // Get the commit's changes compared to default branch comparison, _, err := ghService.Client.Repositories.CompareCommits( context.Background(), @@ -790,10 +792,13 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu var appCode strings.Builder for _, file := range comparison.Files { + if file.Size > MaxPatchSize { + return "", fmt.Errorf("file %s exceeds maximum allowed size", *file.Filename) + } if file.Patch == nil { continue // Skip files without patches } - log.Printf("file patch: %v", *file.Patch) + log.Printf("Processing patch for file: %s", *file.Filename) if *file.Additions > 0 { lines := strings.Split(*file.Patch, "\n") for _, line := range lines { @@ -807,7 +812,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu } if appCode.Len() == 0 { - return "", fmt.Errorf("no changes found in commit: %s", *commitSha) + return "", fmt.Errorf("no code changes found in commit %s. Please ensure the PR contains added or modified code", *commitSha) } return appCode.String(), nil From 56c1d74533a525a790dc5a67517dab6c5a46fb7f Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 17 Dec 2024 12:42:28 +0000 Subject: [PATCH 5/7] fix conflict --- backend/controllers/github.go | 36 +++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index cc67f221..f3a974e6 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -22,26 +22,26 @@ import ( "github.com/diggerhq/digger/backend/ci_backends" config2 "github.com/diggerhq/digger/backend/config" "github.com/diggerhq/digger/backend/locking" + "github.com/diggerhq/digger/backend/middleware" + "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/segment" "github.com/diggerhq/digger/backend/services" + "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/libs/ci" "github.com/diggerhq/digger/libs/ci/generic" + dg_github "github.com/diggerhq/digger/libs/ci/github" comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" dg_locking "github.com/diggerhq/digger/libs/locking" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" - "github.com/google/uuid" - "gorm.io/gorm" - - "github.com/diggerhq/digger/backend/middleware" - "github.com/diggerhq/digger/backend/models" - "github.com/diggerhq/digger/backend/utils" - dg_github "github.com/diggerhq/digger/libs/ci/github" - dg_configuration "github.com/diggerhq/digger/libs/digger_config" "github.com/dominikbraun/graph" "github.com/gin-gonic/gin" "github.com/google/go-github/v61/github" + "github.com/google/uuid" "github.com/samber/lo" "golang.org/x/oauth2" + "gorm.io/gorm" + "runtime/debug" ) type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error @@ -311,6 +311,16 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent, appI } func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error { + defer func() { + if r := recover(); r != nil { + log.Printf("Recovered from panic in handlePullRequestEvent handler: %v", r) + log.Printf("\n=== PANIC RECOVERED ===\n") + log.Printf("Error: %v\n", r) + log.Printf("Stack Trace:\n%s", string(debug.Stack())) + log.Printf("=== END PANIC ===\n") + } + }() + installationId := *payload.Installation.ID repoName := *payload.Repo.Name repoOwner := *payload.Repo.Owner.Login @@ -686,6 +696,16 @@ func getBatchType(jobs []orchestrator_scheduler.Job) orchestrator_scheduler.Digg } func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64, postCommentHooks []IssueCommentHook) error { + defer func() { + if r := recover(); r != nil { + log.Printf("Recovered from panic in handleIssueCommentEvent handler: %v", r) + log.Printf("\n=== PANIC RECOVERED ===\n") + log.Printf("Error: %v\n", r) + log.Printf("Stack Trace:\n%s", string(debug.Stack())) + log.Printf("=== END PANIC ===\n") + } + }() + installationId := *payload.Installation.ID repoName := *payload.Repo.Name repoOwner := *payload.Repo.Owner.Login From 900148b1baa23524c9fea4c23dc3906e3e4c79e3 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 17 Dec 2024 12:47:35 +0000 Subject: [PATCH 6/7] dont check for max size for now --- backend/controllers/github.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 21b81d6a..61e449d5 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -812,9 +812,6 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu var appCode strings.Builder for _, file := range comparison.Files { - if file.Size > MaxPatchSize { - return "", fmt.Errorf("file %s exceeds maximum allowed size", *file.Filename) - } if file.Patch == nil { continue // Skip files without patches } From 5fd36ee0b722359f8f67e4e8999fa5ccaaab228f Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 17 Dec 2024 13:00:54 +0000 Subject: [PATCH 7/7] fix imports --- backend/controllers/github.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 61e449d5..0b6f8a73 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -6,18 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "log" - "math/rand" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "reflect" - "slices" - "strconv" - "strings" - "github.com/davecgh/go-spew/spew" "github.com/diggerhq/digger/backend/ci_backends" config2 "github.com/diggerhq/digger/backend/config" @@ -41,7 +29,18 @@ import ( "github.com/samber/lo" "golang.org/x/oauth2" "gorm.io/gorm" + "log" + "math/rand" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "reflect" "runtime/debug" + "slices" + "strconv" + "strings" ) type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error