Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat bitbucket support #1890

Merged
merged 6 commits into from
Mar 2, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions backend/bootstrap/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ package bootstrap
import (
"embed"
"fmt"
"github.com/diggerhq/digger/backend/config"
"github.com/diggerhq/digger/backend/segment"
pprof_gin "github.com/gin-contrib/pprof"
"html/template"
"io/fs"
"log"
Expand All @@ -15,6 +12,10 @@ import (
"runtime"
"runtime/pprof"

"github.com/diggerhq/digger/backend/config"
"github.com/diggerhq/digger/backend/segment"
pprof_gin "github.com/gin-contrib/pprof"

"time"

"github.com/diggerhq/digger/backend/controllers"
Expand Down Expand Up @@ -216,14 +217,20 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController

if enableApi := os.Getenv("DIGGER_ENABLE_API_ENDPOINTS"); enableApi == "true" {
apiGroup := r.Group("/api")
apiGroup.Use(middleware.HeadersApiAuth())
apiGroup.Use(middleware.InternalApiAuth(), middleware.HeadersApiAuth())

reposApiGroup := apiGroup.Group("/repos")
reposApiGroup.GET("/", controllers.ListReposApi)
reposApiGroup.GET("/:repo_id/jobs", controllers.GetJobsForRepoApi)

githubApiGroup := apiGroup.Group("/github")
githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi)

vcsApiGroup := apiGroup.Group("/connections")
vcsApiGroup.GET("/:id", controllers.GetVCSConnection)
vcsApiGroup.GET("/", controllers.ListVCSConnectionsApi)
vcsApiGroup.POST("/", controllers.CreateVCSConnectionApi)
vcsApiGroup.DELETE("/:id", controllers.DeleteVCSConnection)
}

return r
Expand Down
1 change: 0 additions & 1 deletion backend/controllers/bitbucket.go

This file was deleted.

2 changes: 1 addition & 1 deletion backend/controllers/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {

// update the cache here, do it async for immediate response
go func() {
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
diggerYmlStr = string(diggerYmlBytes)
config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
Expand Down
199 changes: 199 additions & 0 deletions backend/controllers/connections.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
package controllers

import (
"errors"
"github.com/samber/lo"
"log"
"net/http"
"os"

"github.com/diggerhq/digger/backend/utils"
"gorm.io/gorm"

"github.com/diggerhq/digger/backend/middleware"
"github.com/diggerhq/digger/backend/models"
"github.com/gin-gonic/gin"
)

func ListVCSConnectionsApi(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"})
return
}

var connections []models.VCSConnection
err = models.DB.GormDB.Where("organisation_id = ?", org.ID).Find(&connections).Error
if err != nil {
log.Printf("could not fetch VCS connections: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not fetch VCS connections"})
return
}

connectionsSlim := lo.Map(connections, func(c models.VCSConnection, i int) gin.H {
return gin.H{
"connection_id": c.ID,
"vcs": "bitbucket",
"connection_name": c.Name,
}
})
c.JSON(http.StatusOK, gin.H{
"result": connectionsSlim,
})
}

func CreateVCSConnectionApi(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"})
return
}

type CreateVCSConnectionRequest struct {
VCS string `json:"type" binding:"required"`
Name string `json:"connection_name"`
BitbucketAccessToken string `json:"bitbucket_access_token"`
BitbucketWebhookSecret string `json:"bitbucket_webhook_secret"`
}

var request CreateVCSConnectionRequest
if err := c.BindJSON(&request); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"})
return
}

if request.VCS != "bitbucket" {
log.Printf("VCS type not supported: %v", request.VCS)
c.JSON(http.StatusBadRequest, gin.H{"error": "VCS type not supported"})
return
}

secret := os.Getenv("DIGGER_ENCRYPTION_SECRET")
if secret == "" {
log.Printf("ERROR: no encryption secret specified")
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"})
return
}

bitbucketAccessTokenEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketAccessToken)
if err != nil {
log.Printf("could not encrypt access token: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"})
return
}

bitbucketWebhookSecretEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketWebhookSecret)
if err != nil {
log.Printf("could not encrypt webhook secret: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt webhook secret"})
return
}

connection, err := models.DB.CreateVCSConnection(
request.Name,
0,
"",
"",
"",
"",
"",
"",
"",
bitbucketAccessTokenEncrypted,
bitbucketWebhookSecretEncrypted,
org.ID,
)
if err != nil {
log.Printf("")
}

Comment on lines +117 to +119
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Proper error handling for connection creation failures
If an error occurs while creating the VCS connection, log.Printf("") is empty and the code does not return a response. This silently fails and leaves the client hanging. Ensure you return a relevant error or message to the client.

if err != nil {
-    log.Printf("")
+    log.Printf("Failed to create VCS connection: %v", err)
+    c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create VCS connection"})
+    return
}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
log.Printf("")
}
if err != nil {
log.Printf("Failed to create VCS connection: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create VCS connection"})
return
}

c.JSON(http.StatusCreated, gin.H{
"connection": connection.ID,
})
}

func GetVCSConnection(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)
connectionId := c.Param("id")

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
} else {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId)
}
return
}

var connection models.VCSConnection
err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find connection: "+connectionId)
} else {
log.Printf("could not fetch connection: %v err: %v", connectionId, err)
c.String(http.StatusInternalServerError, "Could not fetch connection")
}
return
}

c.JSON(http.StatusOK, gin.H{
"connection_name": connection.Name,
"connection_id": connection.ID,
})
}

func DeleteVCSConnection(c *gin.Context) {
organisationId := c.GetString(middleware.ORGANISATION_ID_KEY)
organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY)
connectionId := c.Param("id")

var org models.Organisation
err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find organisation: "+organisationId)
} else {
log.Printf("could not fetch organisation: %v err: %v", organisationId, err)
c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId)
}
return
}

var connection models.VCSConnection
err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.String(http.StatusNotFound, "Could not find connection: "+connectionId)
} else {
log.Printf("could not fetch connection: %v err: %v", connectionId, err)
c.String(http.StatusInternalServerError, "Could not fetch connection")
}
return
}

err = models.DB.GormDB.Delete(&connection).Error
if err != nil {
log.Printf("could not delete connection: %v err: %v", connectionId, err)
c.String(http.StatusInternalServerError, "Could not delete connection")
return
}

c.JSON(http.StatusOK, gin.H{
"status": "success",
})
}
6 changes: 3 additions & 3 deletions backend/controllers/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
aiSummaryCommentId = aiSummaryComment.Id
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)
batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, nil)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down Expand Up @@ -595,7 +595,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
var diggerYmlStr string
var dependencyGraph graph.Graph[string, dg_configuration.Project]

err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir)
if err != nil {
log.Printf("could not load digger config: %v", err)
Expand Down Expand Up @@ -930,7 +930,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
aiSummaryCommentId = aiSummaryComment.Id
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)
batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, nil)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down
10 changes: 5 additions & 5 deletions backend/controllers/github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {

// migrate tables
err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{},
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.VCSConnection{}, &models.GithubAppInstallationLink{},
&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.JobToken{})
if err != nil {
log.Fatal(err)
Expand Down Expand Up @@ -731,7 +731,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
graph, err := configuration.CreateProjectDependencyGraph(projects)
assert.NoError(t, err)

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 1, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
Expand Down Expand Up @@ -760,7 +760,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
projectMap["dev"] = project1
projectMap["prod"] = project2

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 2, len(result))

Expand Down Expand Up @@ -793,7 +793,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
projectMap["dev"] = project1
projectMap["prod"] = project2

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 2, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
Expand Down Expand Up @@ -838,7 +838,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
projectMap["555"] = project5
projectMap["666"] = project6

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil)
assert.NoError(t, err)
assert.Equal(t, 6, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)
Expand Down
4 changes: 2 additions & 2 deletions backend/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ replace github.com/ugorji/go => github.com/ugorji/go v1.2.12
require (
ariga.io/atlas-provider-gorm v0.5.0
github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/dchest/uniuri v1.2.0
github.com/diggerhq/digger/libs v0.4.15
github.com/dominikbraun/graph v0.23.0
Expand All @@ -20,6 +21,7 @@ require (
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-github/v61 v61.0.0
github.com/google/uuid v1.6.0
github.com/ktrysmt/go-bitbucket v0.9.81
github.com/migueleliasweb/go-github-mock v0.0.23
github.com/robfig/cron v1.2.0
github.com/samber/lo v1.39.0
Expand All @@ -29,7 +31,6 @@ require (
github.com/stretchr/testify v1.9.0
github.com/xanzy/go-gitlab v0.106.0
golang.org/x/oauth2 v0.24.0
gorm.io/datatypes v1.2.4
gorm.io/driver/postgres v1.5.7
gorm.io/driver/sqlite v1.5.5
gorm.io/gorm v1.25.11
Expand Down Expand Up @@ -119,7 +120,6 @@ require (
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/creack/pty v1.1.17 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/dineshba/tf-summarize v0.3.10 // indirect
github.com/envoyproxy/go-control-plane v0.13.0 // indirect
Expand Down
Loading
Loading