Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

wip generation of terraform code from application code #1855

Merged
merged 8 commits into from
Dec 17, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 162 additions & 10 deletions backend/controllers/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,18 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"

"github.com/davecgh/go-spew/spew"
"github.com/diggerhq/digger/backend/ci_backends"
config2 "github.com/diggerhq/digger/backend/config"
Expand All @@ -19,16 +31,6 @@ import (
orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
"github.com/google/uuid"
"gorm.io/gorm"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"reflect"
"slices"
"strconv"
"strings"

"github.com/diggerhq/digger/backend/middleware"
"github.com/diggerhq/digger/backend/models"
Expand Down Expand Up @@ -744,6 +746,156 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
return fmt.Errorf("error getting digger config")
}

// terraform code generator
if os.Getenv("DIGGER_GENERATION_ENABLED") == "1" {
if strings.HasPrefix(*payload.Comment.Body, "digger generate") {
projectName := ci.ParseProjectName(*payload.Comment.Body)
if projectName == "" {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: generate requires argument -p <project_name>: %v", err))
log.Printf("missing project in command: %v", *payload.Comment.Body)
return fmt.Errorf("generate requires argument -p <project_name>: %v", err)
}
Comment on lines +768 to +776
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

⚠️ Potential issue

Fix error handling and improve feature flag management.

The code has the following issues:

  1. The error message in line 752 references an undefined err variable
  2. The feature flag should be defined as a constant or in configuration for better maintainability

Apply this diff to fix the error handling:

-				commentReporterManager.UpdateComment(fmt.Sprintf(":x: generate requires argument -p <project_name>: %v", err))
+				commentReporterManager.UpdateComment(":x: generate requires argument -p <project_name>")

Consider moving the feature flag to a constant or configuration:

+const (
+    FeatureFlagGeneration = "DIGGER_GENERATION_ENABLED"
+)

-	if os.Getenv("DIGGER_GENERATION_ENABLED") == "1" {
+	if os.Getenv(FeatureFlagGeneration) == "1" {

Committable suggestion skipped: line range outside the PR's diff.


project := config.GetProject(projectName)
if project == nil {
commentReporterManager.UpdateComment(fmt.Sprintf("could not find project %v in digger.yml", projectName))
log.Printf("could not find project %v in digger.yml", projectName)
return fmt.Errorf("could not find project %v in digger.yml", projectName)
}

commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded project"))

generationEndpoint := os.Getenv("DIGGER_GENERATION_ENDPOINT")
if generationEndpoint == "" {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify"))
log.Printf("server does not have generation endpoint configured, please verify")
return fmt.Errorf("server does not have generation endpoint configured, please verify")
}
webhookSecret := os.Getenv("DIGGER_GENERATION_WEBHOOK_SECRET")

// Get all code content from the repository at a specific commit
getCodeFromCommit := func(ghService *dg_github.GithubService, repoOwner, repoName string, commitSha *string, projectDir string) (string, error) {
// Get the commit's changes compared to default branch
comparison, _, err := ghService.Client.Repositories.CompareCommits(
context.Background(),
repoOwner,
repoName,
defaultBranch,
*commitSha,
nil,
)
if err != nil {
return "", fmt.Errorf("error comparing commits: %v", err)
}

var appCode strings.Builder
for _, file := range comparison.Files {
if file.Patch == nil {
continue // Skip files without patches
}
log.Printf("file patch: %v", *file.Patch)
if *file.Additions > 0 {
lines := strings.Split(*file.Patch, "\n")
for _, line := range lines {
if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") {
appCode.WriteString(strings.TrimPrefix(line, "+"))
appCode.WriteString("\n")
}
}
}
appCode.WriteString("\n")
}

if appCode.Len() == 0 {
return "", fmt.Errorf("no changes found in commit: %s", *commitSha)
}

return appCode.String(), nil
}

appCode, err := getCodeFromCommit(ghService, repoOwner, repoName, commitSha, project.Dir)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get code content: %v", err))
log.Printf("Error getting code content: %v", err)
return fmt.Errorf("error getting code content: %v", err)
}

commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded code from commit"))

log.Printf("the app code is: %v", appCode)

Comment on lines +846 to +847
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Remove or redact sensitive logging.

Logging the entire application code could expose sensitive information and consume excessive log space.

Apply this diff to improve the logging:

-			log.Printf("the app code is: %v", appCode)
+			log.Printf("Successfully extracted application code, size: %d bytes", len(appCode))
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
log.Printf("the app code is: %v", appCode)
log.Printf("Successfully extracted application code, size: %d bytes", len(appCode))

commentReporterManager.UpdateComment(fmt.Sprintf("Generating terraform..."))
terraformCode, err := utils.GenerateTerraformCode(appCode, generationEndpoint, webhookSecret)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not generate terraform code: %v", err))
log.Printf("could not generate terraform code: %v", err)
return fmt.Errorf("could not generate terraform code: %v", err)
}

commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Generated terraform"))

// comment terraform code to project dir
//project.Dir
log.Printf("terraform code is %v", terraformCode)

baseTree, _, err := ghService.Client.Git.GetTree(context.Background(), repoOwner, repoName, *commitSha, false)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get base tree: %v", err))
log.Printf("Error getting base tree: %v", err)
return fmt.Errorf("error getting base tree: %v", err)
}

// Create a new tree with the new file
treeEntries := []*github.TreeEntry{
{
Path: github.String(filepath.Join(project.Dir, fmt.Sprintf("generated_%v.tf", issueNumber))),
Mode: github.String("100644"),
Type: github.String("blob"),
Content: github.String(terraformCode),
},
Comment on lines +872 to +876
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Improve file path handling and content type safety.

The code directly joins paths and doesn't validate the generated terraform code format.

Apply this diff to improve the implementation:

 				{
-					Path:    github.String(filepath.Join(project.Dir, fmt.Sprintf("generated_%v.tf", issueNumber))),
+					Path:    github.String(filepath.ToSlash(filepath.Join(project.Dir, fmt.Sprintf("generated_%d.tf", issueNumber)))),
 					Mode:    github.String("100644"),
 					Type:    github.String("blob"),
-					Content: github.String(terraformCode),
+					Content: github.String(strings.TrimSpace(terraformCode)),
 				},
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
Path: github.String(filepath.Join(project.Dir, fmt.Sprintf("generated_%v.tf", issueNumber))),
Mode: github.String("100644"),
Type: github.String("blob"),
Content: github.String(terraformCode),
},
Path: github.String(filepath.ToSlash(filepath.Join(project.Dir, fmt.Sprintf("generated_%d.tf", issueNumber)))),
Mode: github.String("100644"),
Type: github.String("blob"),
Content: github.String(strings.TrimSpace(terraformCode)),
},

}

newTree, _, err := ghService.Client.Git.CreateTree(context.Background(), repoOwner, repoName, *baseTree.SHA, treeEntries)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to create new tree: %v", err))
log.Printf("Error creating new tree: %v", err)
return fmt.Errorf("error creating new tree: %v", err)
}

// Create the commit
commitMsg := fmt.Sprintf("Add generated Terraform code for %v", projectName)
commit := &github.Commit{
Message: &commitMsg,
Tree: newTree,
Parents: []*github.Commit{{SHA: commitSha}},
}

newCommit, _, err := ghService.Client.Git.CreateCommit(context.Background(), repoOwner, repoName, commit, nil)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to commit Terraform file: %v", err))
log.Printf("Error committing Terraform file: %v", err)
return fmt.Errorf("error committing Terraform file: %v", err)
}

// Update the reference to point to the new commit
ref := &github.Reference{
Ref: github.String(fmt.Sprintf("refs/heads/%s", *branch)),
Object: &github.GitObject{
SHA: newCommit.SHA,
},
}
_, _, err = ghService.Client.Git.UpdateRef(context.Background(), repoOwner, repoName, ref, false)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to update branch reference: %v", err))
log.Printf("Error updating branch reference: %v", err)
return fmt.Errorf("error updating branch reference: %v", err)
}

commentReporterManager.UpdateComment(":white_check_mark: Successfully generated and committed Terraform code")
return nil
}
}

commentIdStr := strconv.FormatInt(userCommentId, 10)
err = ghService.CreateCommentReaction(commentIdStr, string(dg_github.GithubCommentEyesReaction))
if err != nil {
Expand Down
69 changes: 69 additions & 0 deletions backend/utils/ai.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package utils

import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
)

func GenerateTerraformCode(appCode string, generationEndpoint string, webhookSecret string) (string, error) {

payload := map[string]string{
"code": appCode,
}

// Convert payload to JSON
jsonData, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("Error marshalling JSON: %v\n", err)
}

// Create request
req, err := http.NewRequest("POST", generationEndpoint, bytes.NewBuffer(jsonData))
if err != nil {
return "", fmt.Errorf("Error creating request: %v\n", err)
}

// Set headers
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Webhook-Secret", webhookSecret) // Replace with your webhook secret

// Make the request
client := &http.Client{}
resp, err := client.Do(req)
Comment on lines +34 to +35
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Add timeout to HTTP client

The HTTP client should have a reasonable timeout to prevent hanging on slow responses.

-    client := &http.Client{}
+    client := &http.Client{
+        Timeout: 30 * time.Second,
+    }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
client := &http.Client{}
resp, err := client.Do(req)
client := &http.Client{
Timeout: 30 * time.Second,
}
resp, err := client.Do(req)

if err != nil {
return "", fmt.Errorf("Error making request: %v\n", err)
}
defer resp.Body.Close()

// Read response
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("Error reading response: %v\n", err)
}

// Print response
if resp.StatusCode == 400 {
return "", fmt.Errorf("unable to generate terraform code from the code available, is it valid application code")
}

if resp.StatusCode != 200 {
return "", fmt.Errorf("unexpected error occured while generating code")
}

type GeneratorResponse struct {
Result string `json:"result"`
Status string `json:"status"`
}

var response GeneratorResponse
err = json.Unmarshal(body, &response)
if err != nil {
return "", fmt.Errorf("unable to parse generator response: %v", err)
}

return response.Result, nil

}
Loading