From dcc50055bd73e0b37a40dc71d59024417ff1de20 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 27 Feb 2025 21:12:32 +0300 Subject: [PATCH 1/6] bb support --- backend/bootstrap/main.go | 15 +- backend/controllers/bitbucket.go | 1 - backend/controllers/cache.go | 2 +- backend/controllers/connections.go | 199 +++++++++++ backend/controllers/github.go | 2 +- backend/controllers/github_test.go | 2 +- backend/go.mod | 4 +- backend/go.sum | 22 +- backend/migrations/20250226185150.sql | 2 + backend/migrations/atlas.sum | 3 +- backend/models/github.go | 32 +- backend/models/scheduler_test.go | 2 +- backend/models/storage.go | 39 ++- backend/models/storage_test.go | 2 +- backend/{models => queries}/queries.go | 2 +- backend/tasks/runs_test.go | 2 +- backend/utils/bitbucket.go | 79 +++++ backend/utils/crypt.go | 44 ++- backend/utils/github.go | 4 +- backend/utils/github_test.go | 8 +- backend/utils/gitlab.go | 4 +- backend/utils/gitshell.go | 25 +- cli/cmd/digger/root.go | 2 +- ee/backend/controllers/bitbucket.go | 348 ++++++++++++++++++++ ee/backend/controllers/bitbucket_utils.go | 208 ++++++++++++ ee/backend/controllers/github.go | 6 +- ee/backend/controllers/gitlab.go | 7 +- ee/backend/main.go | 2 + ee/backend/providers/github/providers.go | 4 +- ee/cli/cmd/digger/root.go | 2 +- ee/cli/pkg/policy/policy.go | 2 +- ee/cli/pkg/utils/github.go | 4 +- ee/drift/tasks/github.go | 2 +- ee/drift/utils/github.go | 2 +- go.work.sum | 11 +- {cli/pkg => libs/ci}/bitbucket/bitbucket.go | 43 +-- libs/ci/bitbucket/bitbucket_service.go | 193 +++++++++++ next/controllers/github.go | 2 +- next/services/config.go | 2 +- 39 files changed, 1203 insertions(+), 132 deletions(-) delete mode 100644 backend/controllers/bitbucket.go create mode 100644 backend/controllers/connections.go create mode 100644 backend/migrations/20250226185150.sql rename backend/{models => queries}/queries.go (97%) create mode 100644 backend/utils/bitbucket.go create mode 100644 ee/backend/controllers/bitbucket.go create mode 100644 ee/backend/controllers/bitbucket_utils.go rename {cli/pkg => libs/ci}/bitbucket/bitbucket.go (93%) create mode 100644 libs/ci/bitbucket/bitbucket_service.go diff --git a/backend/bootstrap/main.go b/backend/bootstrap/main.go index ec2a664d7..a68afcbf4 100644 --- a/backend/bootstrap/main.go +++ b/backend/bootstrap/main.go @@ -3,9 +3,6 @@ package bootstrap import ( "embed" "fmt" - "github.com/diggerhq/digger/backend/config" - "github.com/diggerhq/digger/backend/segment" - pprof_gin "github.com/gin-contrib/pprof" "html/template" "io/fs" "log" @@ -15,6 +12,10 @@ import ( "runtime" "runtime/pprof" + "github.com/diggerhq/digger/backend/config" + "github.com/diggerhq/digger/backend/segment" + pprof_gin "github.com/gin-contrib/pprof" + "time" "github.com/diggerhq/digger/backend/controllers" @@ -216,7 +217,7 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController if enableApi := os.Getenv("DIGGER_ENABLE_API_ENDPOINTS"); enableApi == "true" { apiGroup := r.Group("/api") - apiGroup.Use(middleware.HeadersApiAuth()) + apiGroup.Use(middleware.InternalApiAuth(), middleware.HeadersApiAuth()) reposApiGroup := apiGroup.Group("/repos") reposApiGroup.GET("/", controllers.ListReposApi) @@ -224,6 +225,12 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController githubApiGroup := apiGroup.Group("/github") githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi) + + vcsApiGroup := apiGroup.Group("/connections") + vcsApiGroup.GET("/:id", controllers.GetVCSConnection) + vcsApiGroup.GET("/", controllers.ListVCSConnectionsApi) + vcsApiGroup.POST("/", controllers.CreateVCSConnectionApi) + vcsApiGroup.DELETE("/:id", controllers.DeleteVCSConnection) } return r diff --git a/backend/controllers/bitbucket.go b/backend/controllers/bitbucket.go deleted file mode 100644 index 2d3293679..000000000 --- a/backend/controllers/bitbucket.go +++ /dev/null @@ -1 +0,0 @@ -package controllers diff --git a/backend/controllers/cache.go b/backend/controllers/cache.go index 6f29ca2a2..b72fec23a 100644 --- a/backend/controllers/cache.go +++ b/backend/controllers/cache.go @@ -76,7 +76,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) { return err } return nil - }) + }, "") if err != nil { log.Printf("could not load digger config :%v", err) diff --git a/backend/controllers/connections.go b/backend/controllers/connections.go new file mode 100644 index 000000000..a21c90554 --- /dev/null +++ b/backend/controllers/connections.go @@ -0,0 +1,199 @@ +package controllers + +import ( + "errors" + "github.com/samber/lo" + "log" + "net/http" + "os" + + "github.com/diggerhq/digger/backend/utils" + "gorm.io/gorm" + + "github.com/diggerhq/digger/backend/middleware" + "github.com/diggerhq/digger/backend/models" + "github.com/gin-gonic/gin" +) + +func ListVCSConnectionsApi(c *gin.Context) { + organisationId := c.GetString(middleware.ORGANISATION_ID_KEY) + organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY) + + var org models.Organisation + err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error + if err != nil { + log.Printf("could not fetch organisation: %v err: %v", organisationId, err) + c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"}) + return + } + + var connections []models.VCSConnection + err = models.DB.GormDB.Where("organisation_id = ?", org.ID).Find(&connections).Error + if err != nil { + log.Printf("could not fetch VCS connections: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not fetch VCS connections"}) + return + } + + connectionsSlim := lo.Map(connections, func(c models.VCSConnection, i int) gin.H { + return gin.H{ + "connection_id": c.ID, + "vcs": "bitbucket", + "connection_name": c.Name, + } + }) + c.JSON(http.StatusOK, gin.H{ + "result": connectionsSlim, + }) +} + +func CreateVCSConnectionApi(c *gin.Context) { + organisationId := c.GetString(middleware.ORGANISATION_ID_KEY) + organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY) + + var org models.Organisation + err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error + if err != nil { + log.Printf("could not fetch organisation: %v err: %v", organisationId, err) + c.JSON(http.StatusNotFound, gin.H{"error": "Could not fetch organisation"}) + return + } + + type CreateVCSConnectionRequest struct { + VCS string `json:"type" binding:"required"` + Name string `json:"connection_name"` + BitbucketAccessToken string `json:"bitbucket_access_token"` + BitbucketWebhookSecret string `json:"bitbucket_webhook_secret"` + } + + var request CreateVCSConnectionRequest + if err := c.BindJSON(&request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request body"}) + return + } + + if request.VCS != "bitbucket" { + log.Printf("VCS type not supported: %v", request.VCS) + c.JSON(http.StatusBadRequest, gin.H{"error": "VCS type not supported"}) + return + } + + secret := os.Getenv("DIGGER_ENCRYPTION_SECRET") + if secret == "" { + log.Printf("ERROR: no encryption secret specified") + c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"}) + return + } + + bitbucketAccessTokenEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketAccessToken) + if err != nil { + log.Printf("could not encrypt access token: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt access token"}) + return + } + + bitbucketWebhookSecretEncrypted, err := utils.AESEncrypt([]byte(secret), request.BitbucketWebhookSecret) + if err != nil { + log.Printf("could not encrypt webhook secret: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not encrypt webhook secret"}) + return + } + + connection, err := models.DB.CreateVCSConnection( + request.Name, + 0, + "", + "", + "", + "", + "", + "", + "", + bitbucketAccessTokenEncrypted, + bitbucketWebhookSecretEncrypted, + org.ID, + ) + if err != nil { + log.Printf("") + } + + c.JSON(http.StatusCreated, gin.H{ + "connection": connection.ID, + }) +} + +func GetVCSConnection(c *gin.Context) { + organisationId := c.GetString(middleware.ORGANISATION_ID_KEY) + organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY) + connectionId := c.Param("id") + + var org models.Organisation + err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.String(http.StatusNotFound, "Could not find organisation: "+organisationId) + } else { + log.Printf("could not fetch organisation: %v err: %v", organisationId, err) + c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId) + } + return + } + + var connection models.VCSConnection + err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.String(http.StatusNotFound, "Could not find connection: "+connectionId) + } else { + log.Printf("could not fetch connection: %v err: %v", connectionId, err) + c.String(http.StatusInternalServerError, "Could not fetch connection") + } + return + } + + c.JSON(http.StatusOK, gin.H{ + "connection_name": connection.Name, + "connection_id": connection.ID, + }) +} + +func DeleteVCSConnection(c *gin.Context) { + organisationId := c.GetString(middleware.ORGANISATION_ID_KEY) + organisationSource := c.GetString(middleware.ORGANISATION_SOURCE_KEY) + connectionId := c.Param("id") + + var org models.Organisation + err := models.DB.GormDB.Where("external_id = ? AND external_source = ?", organisationId, organisationSource).First(&org).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.String(http.StatusNotFound, "Could not find organisation: "+organisationId) + } else { + log.Printf("could not fetch organisation: %v err: %v", organisationId, err) + c.String(http.StatusNotFound, "Could not fetch organisation: "+organisationId) + } + return + } + + var connection models.VCSConnection + err = models.DB.GormDB.Where("id = ? AND organisation_id = ?", connectionId, org.ID).First(&connection).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.String(http.StatusNotFound, "Could not find connection: "+connectionId) + } else { + log.Printf("could not fetch connection: %v err: %v", connectionId, err) + c.String(http.StatusInternalServerError, "Could not fetch connection") + } + return + } + + err = models.DB.GormDB.Delete(&connection).Error + if err != nil { + log.Printf("could not delete connection: %v err: %v", connectionId, err) + c.String(http.StatusInternalServerError, "Could not delete connection") + return + } + + c.JSON(http.StatusOK, gin.H{ + "status": "success", + }) +} diff --git a/backend/controllers/github.go b/backend/controllers/github.go index a91297bb9..4f04b9bb9 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -607,7 +607,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 return err } return nil - }) + }, "") if err != nil { log.Printf("Error cloning and loading config: %v", err) return "", nil, nil, nil, fmt.Errorf("error cloning and loading config %v", err) diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index 39b57cfe7..2ddf74216 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -589,7 +589,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) { // migrate tables err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{}, - &models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{}, + &models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.VCSConnection{}, &models.GithubAppInstallationLink{}, &models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.JobToken{}) if err != nil { log.Fatal(err) diff --git a/backend/go.mod b/backend/go.mod index 5f5a6aa44..369dbc959 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -10,6 +10,7 @@ replace github.com/ugorji/go => github.com/ugorji/go v1.2.12 require ( ariga.io/atlas-provider-gorm v0.5.0 github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dchest/uniuri v1.2.0 github.com/diggerhq/digger/libs v0.4.15 github.com/dominikbraun/graph v0.23.0 @@ -20,6 +21,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/go-github/v61 v61.0.0 github.com/google/uuid v1.6.0 + github.com/ktrysmt/go-bitbucket v0.9.81 github.com/migueleliasweb/go-github-mock v0.0.23 github.com/robfig/cron v1.2.0 github.com/samber/lo v1.39.0 @@ -29,7 +31,6 @@ require ( github.com/stretchr/testify v1.9.0 github.com/xanzy/go-gitlab v0.106.0 golang.org/x/oauth2 v0.24.0 - gorm.io/datatypes v1.2.4 gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.5 gorm.io/gorm v1.25.11 @@ -119,7 +120,6 @@ require ( github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/creack/pty v1.1.17 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dineshba/tf-summarize v0.3.10 // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 1b1227e03..76fb19cf0 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -78,6 +78,7 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -1024,6 +1025,8 @@ github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaI github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -1042,6 +1045,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -1049,6 +1053,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ktrysmt/go-bitbucket v0.9.81 h1:PQxJsFcGdblDOv5PhFA03uNgXMiJfpLo03oYIUdQ2h0= +github.com/ktrysmt/go-bitbucket v0.9.81/go.mod h1:eWIy5+e1l2eDf9xxwCEmK7oPvNKR91vwYocJWIUQISQ= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= @@ -1204,6 +1210,7 @@ github.com/pkg/browser v0.0.0-20201207095918-0426ae3fba23/go.mod h1:N6UoU20jOqgg github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1251,6 +1258,8 @@ github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfm github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -1487,6 +1496,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1598,6 +1609,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1625,6 +1638,7 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1735,6 +1749,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1747,6 +1763,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1765,6 +1783,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2109,8 +2129,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/datatypes v1.2.4 h1:uZmGAcK/QZ0uyfCuVg0VQY1ZmV9h1fuG0tMwKByO1z4= -gorm.io/datatypes v1.2.4/go.mod h1:f4BsLcFAX67szSv8svwLRjklArSHAvHLeE3pXAS5DZI= gorm.io/driver/mysql v1.4.0/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8= gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= diff --git a/backend/migrations/20250226185150.sql b/backend/migrations/20250226185150.sql new file mode 100644 index 000000000..5ff7be9dd --- /dev/null +++ b/backend/migrations/20250226185150.sql @@ -0,0 +1,2 @@ +-- Modify "github_app_connections" table +ALTER TABLE "public"."github_app_connections" ADD COLUMN "bitbucket_access_token_encrypted" text NULL, ADD COLUMN "bitbucket_webhook_secret_encrypted" text NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index 3e6ffeb0a..2ba62ac92 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:5Ds30GVrcb/6GVoexZ0yD9I3TD4q/Udz6UfOgUkrpks= +h1:uESS+Cvtzp9ftW1x7GX98EeuQMvEHX6PZgBN0UNOzMM= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -42,3 +42,4 @@ h1:5Ds30GVrcb/6GVoexZ0yD9I3TD4q/Udz6UfOgUkrpks= 20250220173439.sql h1:Dho4Gw361D8kuL5pB56hUj5hZM1NPTbMYVBjBW54DkE= 20250221044813.sql h1:PPcVcoMaMY5DXyoptQYeBOEwofrzIfyKcWRd3S12I2U= 20250224152926.sql h1:EjoFpfeoCpk/SjSo2i7sajKCR3t7YCn+1ZgGJrT0L9Y= +20250226185150.sql h1:K7e/3Zy2wSTqKa3iYpIb02GTAniYSXHObTIqOV9aOhM= diff --git a/backend/models/github.go b/backend/models/github.go index 1cd7828b6..a3080b4f4 100644 --- a/backend/models/github.go +++ b/backend/models/github.go @@ -4,19 +4,27 @@ import ( "gorm.io/gorm" ) -type GithubAppConnection struct { +type VCSConnection struct { gorm.Model - GithubId int64 // app id - ClientID string - ClientSecretEncrypted string - WebhookSecretEncrypted string - PrivateKeyEncrypted string - PrivateKeyBase64Encrypted string - Org string - Name string - GithubAppUrl string - OrganisationID uint - Organisation Organisation + GithubId int64 // app id + ClientID string + ClientSecretEncrypted string + WebhookSecretEncrypted string + PrivateKeyEncrypted string + PrivateKeyBase64Encrypted string + Org string + Name string + GithubAppUrl string + BitbucketAccessTokenEncrypted string + BitbucketWebhookSecretEncrypted string + OrganisationID uint + Organisation Organisation +} + +// TODO: Create migration to rename this table to vcs_connections +// for some reason atlas wants to destroy and recreate and I did not have time to look into it +func (VCSConnection) TableName() string { + return "github_app_connections" // Keep the original table name } type GithubAppInstallStatus int diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 581e2964e..9b00e98ff 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -38,7 +38,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) { // migrate tables err = gdb.AutoMigrate(&Policy{}, &Organisation{}, &Repo{}, &Project{}, &Token{}, - &User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubAppConnection{}, &GithubAppInstallationLink{}, + &User{}, &ProjectRun{}, &GithubAppInstallation{}, &VCSConnection{}, &GithubAppInstallationLink{}, &GithubDiggerJobLink{}, &DiggerJob{}, &DiggerJobParentLink{}) if err != nil { log.Fatal(err) diff --git a/backend/models/storage.go b/backend/models/storage.go index 24f093106..bc6cacde7 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/dchest/uniuri" + "github.com/diggerhq/digger/backend/queries" configuration "github.com/diggerhq/digger/libs/digger_config" scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/gin-gonic/gin" @@ -437,18 +438,20 @@ func (db *Database) GetGithubAppInstallationLink(installationId int64) (*GithubA return &link, nil } -func (db *Database) CreateGithubAppConnection(name string, githubId int64, ClientID string, ClientSecretEncrypted string, WebhookSecretEncrypted string, PrivateKeyEncrypted string, PrivateKeyBase64Encrypted string, Org string, url string, orgId uint) (*GithubAppConnection, error) { - app := GithubAppConnection{ - Name: name, - GithubId: githubId, - ClientID: ClientID, - ClientSecretEncrypted: ClientSecretEncrypted, - WebhookSecretEncrypted: WebhookSecretEncrypted, - PrivateKeyEncrypted: PrivateKeyEncrypted, - PrivateKeyBase64Encrypted: PrivateKeyBase64Encrypted, - Org: Org, - GithubAppUrl: url, - OrganisationID: orgId, +func (db *Database) CreateVCSConnection(name string, githubId int64, ClientID string, ClientSecretEncrypted string, WebhookSecretEncrypted string, PrivateKeyEncrypted string, PrivateKeyBase64Encrypted string, Org string, url string, bitbucketAccessTokenEnc string, bitbucketWebhookSecretEnc string, orgId uint) (*VCSConnection, error) { + app := VCSConnection{ + Name: name, + GithubId: githubId, + ClientID: ClientID, + ClientSecretEncrypted: ClientSecretEncrypted, + WebhookSecretEncrypted: WebhookSecretEncrypted, + PrivateKeyEncrypted: PrivateKeyEncrypted, + PrivateKeyBase64Encrypted: PrivateKeyBase64Encrypted, + Org: Org, + GithubAppUrl: url, + BitbucketWebhookSecretEncrypted: bitbucketWebhookSecretEnc, + BitbucketAccessTokenEncrypted: bitbucketAccessTokenEnc, + OrganisationID: orgId, } result := db.GormDB.Save(&app) if result.Error != nil { @@ -458,8 +461,8 @@ func (db *Database) CreateGithubAppConnection(name string, githubId int64, Clien return &app, nil } -func (db *Database) GetGithubAppConnectionById(id string) (*GithubAppConnection, error) { - app := GithubAppConnection{} +func (db *Database) GetVCSConnectionById(id string) (*VCSConnection, error) { + app := VCSConnection{} result := db.GormDB.Where("id = ?", id).Find(&app) if result.Error != nil { log.Printf("Failed to find GitHub App for id: %v, error: %v\n", id, result.Error) @@ -469,8 +472,8 @@ func (db *Database) GetGithubAppConnectionById(id string) (*GithubAppConnection, } // GetGithubApp return GithubApp by Id -func (db *Database) GetGithubAppConnection(gitHubAppId any) (*GithubAppConnection, error) { - app := GithubAppConnection{} +func (db *Database) GetVCSConnection(gitHubAppId any) (*VCSConnection, error) { + app := VCSConnection{} result := db.GormDB.Where("github_id = ?", gitHubAppId).Find(&app) if result.Error != nil { log.Printf("Failed to find GitHub App for id: %v, error: %v\n", gitHubAppId, result.Error) @@ -924,8 +927,8 @@ func (db *Database) GetDiggerJobsForBatch(batchId uuid.UUID) ([]DiggerJob, error return jobs, nil } -func (db *Database) GetJobsByRepoName(orgId uint, repoFullName string) ([]JobQueryResult, error) { - var results []JobQueryResult +func (db *Database) GetJobsByRepoName(orgId uint, repoFullName string) ([]queries.JobQueryResult, error) { + var results []queries.JobQueryResult query := ` SELECT diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index d9d06ba72..509ed2343 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -36,7 +36,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *Database, *Organisation) { // migrate tables err = gdb.AutoMigrate(&Policy{}, &Organisation{}, &Repo{}, &Project{}, &Token{}, - &User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubAppConnection{}, &GithubAppInstallationLink{}, + &User{}, &ProjectRun{}, &GithubAppInstallation{}, &VCSConnection{}, &GithubAppInstallationLink{}, &GithubDiggerJobLink{}, &DiggerJob{}, &DiggerJobParentLink{}) if err != nil { log.Fatal(err) diff --git a/backend/models/queries.go b/backend/queries/queries.go similarity index 97% rename from backend/models/queries.go rename to backend/queries/queries.go index 35919857a..3f5627b85 100644 --- a/backend/models/queries.go +++ b/backend/queries/queries.go @@ -1,4 +1,4 @@ -package models +package queries import "time" diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 9e42f6cee..f9c36bc4c 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -50,7 +50,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) { // migrate tables err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{}, - &models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{}, + &models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.VCSConnection{}, &models.GithubAppInstallationLink{}, &models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.DiggerRun{}, &models.DiggerRunQueueItem{}) if err != nil { log.Fatal(err) diff --git a/backend/utils/bitbucket.go b/backend/utils/bitbucket.go new file mode 100644 index 000000000..ab8b751bd --- /dev/null +++ b/backend/utils/bitbucket.go @@ -0,0 +1,79 @@ +package utils + +import ( + "fmt" + orchestrator_bitbucket "github.com/diggerhq/digger/libs/ci/bitbucket" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" + "github.com/dominikbraun/graph" + "github.com/ktrysmt/go-bitbucket" + "log" + "net/http" + "os" + "path" +) + +type BitbucketProvider interface { + NewClient(token string) (*bitbucket.Client, error) +} + +type BitbucketClientProvider struct{} + +func (b BitbucketClientProvider) NewClient(token string) (*bitbucket.Client, error) { + client := bitbucket.NewOAuthbearerToken(token) + return client, nil +} + +func GetBitbucketService(bb BitbucketProvider, token string, repoOwner string, repoName string, prNumber int) (*orchestrator_bitbucket.BitbucketAPI, error) { + //token := os.Getenv("DIGGER_BITBUCKET_ACCESS_TOKEN") + + //client, err := bb.NewClient(token) + //if err != nil { + // return nil, fmt.Errorf("could not get bitbucket client: %v", err) + //} + //context := orchestrator_bitbucket.BitbucketContext{ + // RepositoryName: repoName, + // RepositoryFullName: repoFullName, + // PullRequestID: &prNumber, + //} + service := orchestrator_bitbucket.BitbucketAPI{ + AuthToken: token, + RepoWorkspace: repoOwner, + RepoName: repoName, + HttpClient: http.Client{}, + } + return &service, nil +} + +func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, prNumber int) (string, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) { + service, err := GetBitbucketService(bb, token, repoOwner, repoName, prNumber) + if err != nil { + return "", nil, nil, fmt.Errorf("could not get bitbucket service: %v", err) + } + var config *dg_configuration.DiggerConfig + var diggerYmlStr string + var dependencyGraph graph.Graph[string, dg_configuration.Project] + + changedFiles, err := service.GetChangedFiles(prNumber) + if err != nil { + log.Printf("Error getting changed files: %v", err) + return "", nil, nil, fmt.Errorf("error getting changed files") + } + + err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { + diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) + diggerYmlStr = string(diggerYmlBytes) + config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles) + if err != nil { + log.Printf("Error loading digger config: %v", err) + return err + } + return nil + }) + if err != nil { + log.Printf("Error cloning and loading config: %v", err) + return "", nil, nil, fmt.Errorf("error cloning and loading config") + } + + log.Printf("Digger config loaded successfully\n") + return diggerYmlStr, config, dependencyGraph, nil +} diff --git a/backend/utils/crypt.go b/backend/utils/crypt.go index 900b016ce..4c265d41a 100644 --- a/backend/utils/crypt.go +++ b/backend/utils/crypt.go @@ -76,22 +76,24 @@ func AESDecrypt(key []byte, encodedCiphertext string) (string, error) { } // represents a decrypted record -type DecryptedGithubAppConnection struct { - GithubId int64 - ClientID string - ClientSecret string - WebhookSecret string - PrivateKey string - PrivateKeyBase64 string - Org string - Name string - GithubAppUrl string - OrganisationID uint +type DecryptedVCSConnection struct { + GithubId int64 + ClientID string + ClientSecret string + WebhookSecret string + PrivateKey string + PrivateKeyBase64 string + Org string + Name string + GithubAppUrl string + OrganisationID uint + BitbucketAccessToken string + BitbucketWebhookSecret string } -func DecryptConnection(g *models.GithubAppConnection, key []byte) (*DecryptedGithubAppConnection, error) { +func DecryptConnection(g *models.VCSConnection, key []byte) (*DecryptedVCSConnection, error) { // Create decrypted version - decrypted := &DecryptedGithubAppConnection{ + decrypted := &DecryptedVCSConnection{ GithubId: g.GithubId, ClientID: g.ClientID, Org: g.Org, @@ -136,5 +138,21 @@ func DecryptConnection(g *models.GithubAppConnection, key []byte) (*DecryptedGit decrypted.PrivateKeyBase64 = privateKeyBase64 } + if g.BitbucketAccessTokenEncrypted != "" { + bitbucketAccessToken, err := AESDecrypt(key, g.BitbucketAccessTokenEncrypted) + if err != nil { + return nil, fmt.Errorf("failed to decrypt private key base64: %w", err) + } + decrypted.BitbucketAccessToken = bitbucketAccessToken + } + + if g.BitbucketWebhookSecretEncrypted != "" { + bitbucketWebhookSecret, err := AESDecrypt(key, g.BitbucketWebhookSecretEncrypted) + if err != nil { + return nil, fmt.Errorf("failed to decrypt private key base64: %w", err) + } + decrypted.BitbucketWebhookSecret = bitbucketWebhookSecret + } + return decrypted, nil } diff --git a/backend/utils/github.go b/backend/utils/github.go index 1e0a34880..df6f05016 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -28,9 +28,9 @@ func createTempDir() string { type action func(string) error -func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, action action) error { +func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error { dir := createTempDir() - git := NewGitShellWithTokenAuth(dir, token) + git := NewGitShellWithTokenAuth(dir, token, tokenUsername) err := git.Clone(repoUrl, branch) if err != nil { return err diff --git a/backend/utils/github_test.go b/backend/utils/github_test.go index d24939c81..17b0f26d9 100644 --- a/backend/utils/github_test.go +++ b/backend/utils/github_test.go @@ -14,14 +14,14 @@ func init() { func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) { f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", f) + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", f, "") assert.NotNil(t, err) } func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, f) + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, f, "") assert.Nil(t, err) } @@ -32,13 +32,13 @@ func TestGithubCloneWithPrivateRepoAndValidTokenThrowsNoError(t *testing.T) { return } f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, f) + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, f, "") assert.Nil(t, err) } func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, f) + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, f, "") assert.NotNil(t, err) } diff --git a/backend/utils/gitlab.go b/backend/utils/gitlab.go index 809a6e0eb..5514b7942 100644 --- a/backend/utils/gitlab.go +++ b/backend/utils/gitlab.go @@ -46,7 +46,7 @@ func GetGitlabService(gh GitlabProvider, projectId int, repoName string, repoFul return &service, nil } -func GetDiggerConfigForBranch(gh GitlabProvider, projectId int, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, prNumber int, discussionId string) (string, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) { +func GetDiggerConfigForBranchGitlab(gh GitlabProvider, projectId int, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, prNumber int, discussionId string) (string, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) { token := os.Getenv("DIGGER_GITLAB_ACCESS_TOKEN") service, err := GetGitlabService(gh, projectId, repoName, repoFullName, prNumber, discussionId) @@ -62,7 +62,7 @@ func GetDiggerConfigForBranch(gh GitlabProvider, projectId int, repoFullName str log.Printf("Error getting changed files: %v", err) return "", nil, nil, fmt.Errorf("error getting changed files") } - err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, func(dir string) error { + err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles) diff --git a/backend/utils/gitshell.go b/backend/utils/gitshell.go index 1327591a5..6e3ca6477 100644 --- a/backend/utils/gitshell.go +++ b/backend/utils/gitshell.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "log" "net/url" "os" "os/exec" @@ -12,9 +13,10 @@ import ( ) type GitAuth struct { - Username string - Password string // Can be either password or access token - Token string // x-access-token + Username string + Password string // Can be either password or access token + TokenUsername string // if set will replace x-access-token (needed for bitbucket which uses x-token-auth) + Token string // x-access-token } type GitShell struct { @@ -41,11 +43,12 @@ func NewGitShell(workDir string, auth *GitAuth) *GitShell { } } -func NewGitShellWithTokenAuth(workDir string, token string) *GitShell { +func NewGitShellWithTokenAuth(workDir string, token string, tokenUsername string) *GitShell { auth := GitAuth{ - Username: "x-access-token", - Password: "", - Token: token, + Username: "x-access-token", + Password: "", + TokenUsername: tokenUsername, + Token: token, } return NewGitShell(workDir, &auth) } @@ -64,7 +67,11 @@ func (g *GitShell) formatAuthURL(repoURL string) (string, error) { // Handle different auth types if g.auth.Token != "" { // X-Access-Token authentication - parsedURL.User = url.UserPassword("x-access-token", g.auth.Token) + tokenUsername := g.auth.TokenUsername + if tokenUsername == "" { + tokenUsername = "x-access-token" + } + parsedURL.User = url.UserPassword(tokenUsername, g.auth.Token) } else if g.auth.Username != "" { // Username/password or personal access token parsedURL.User = url.UserPassword(g.auth.Username, g.auth.Password) @@ -121,6 +128,8 @@ func (g *GitShell) Clone(repoURL, branch string) error { if branch != "" { args = append(args, "-b", branch) } + + log.Printf("auth url: %v", authURL) args = append(args, "--depth", "1") args = append(args, "--single-branch", authURL, g.workDir) diff --git a/cli/cmd/digger/root.go b/cli/cmd/digger/root.go index 8b6481a71..fdc6626d6 100644 --- a/cli/cmd/digger/root.go +++ b/cli/cmd/digger/root.go @@ -2,10 +2,10 @@ package main import ( "fmt" - "github.com/diggerhq/digger/cli/pkg/bitbucket" "github.com/diggerhq/digger/cli/pkg/utils" "github.com/diggerhq/digger/libs/backendapi" "github.com/diggerhq/digger/libs/ci" + "github.com/diggerhq/digger/libs/ci/bitbucket" orchestrator_github "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/comment_utils/reporting" locking2 "github.com/diggerhq/digger/libs/locking" diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go new file mode 100644 index 000000000..fcc2f381b --- /dev/null +++ b/ee/backend/controllers/bitbucket.go @@ -0,0 +1,348 @@ +package controllers + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/diggerhq/digger/backend/ci_backends" + "github.com/diggerhq/digger/backend/controllers" + "github.com/diggerhq/digger/backend/locking" + "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/segment" + "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/ci/generic" + dg_github "github.com/diggerhq/digger/libs/ci/github" + comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting" + dg_configuration "github.com/diggerhq/digger/libs/digger_config" + dg_locking "github.com/diggerhq/digger/libs/locking" + "github.com/diggerhq/digger/libs/scheduler" + "github.com/gin-gonic/gin" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" +) + +type BBWebhookPayload map[string]interface{} + +// TriggerPipeline triggers a CI pipeline with the specified type +func TriggerPipeline(pipelineType string) error { + // Implement pipeline triggering logic + return nil +} + +// verifySignature verifies the X-Hub-Signature header +func verifySignature(c *gin.Context, body []byte, webhookSecret string) bool { + // Get the signature from the header + signature := c.GetHeader("X-Hub-Signature") + if signature == "" { + return false + } + + // Remove the "sha256=" prefix if present + if len(signature) > 7 && signature[0:7] == "sha256=" { + signature = signature[7:] + } + + // Create a new HMAC + + mac := hmac.New(sha256.New, []byte(webhookSecret)) + mac.Write(body) + expectedMAC := mac.Sum(nil) + expectedSignature := hex.EncodeToString(expectedMAC) + + return hmac.Equal([]byte(signature), []byte(expectedSignature)) +} + +// WebhookHandler processes incoming Bitbucket webhook events +func (ee DiggerEEController) BitbucketWebhookHandler(c *gin.Context) { + eventKey := c.GetHeader("X-Event-Key") + if eventKey == "" { + log.Printf("unknown event") + return + } + + connectionId := c.GetHeader("DIGGER_CONNECTION_ID") + connectionEncrypted, err := models.DB.GetVCSConnectionById(connectionId) + if err != nil { + log.Printf("failed to fetch connection: %v", err) + c.String(http.StatusInternalServerError, "error while processing connection") + return + } + + secret := os.Getenv("DIGGER_ENCRYPTION_SECRET") + if secret == "" { + log.Printf("ERROR: no encryption secret specified, please specify DIGGER_ENCRYPTION_SECRET as 32 bytes base64 string") + c.String(http.StatusInternalServerError, "secret not specified") + return + } + connectionDecrypted, err := utils.DecryptConnection(connectionEncrypted, []byte(secret)) + if err != nil { + log.Printf("ERROR: could not perform decryption: %v", err) + c.String(http.StatusInternalServerError, "unexpected error while fetching connection") + return + } + bitbucketAccessToken := connectionDecrypted.BitbucketAccessToken + + orgId := connectionDecrypted.OrganisationID + bitbucketWebhookSecret := connectionDecrypted.BitbucketWebhookSecret + + if bitbucketWebhookSecret == "" { + log.Printf("ERROR: no encryption secret specified, please specify DIGGER_ENCRYPTION_SECRET as 32 bytes base64 string") + c.String(http.StatusInternalServerError, "unexpected error while fetching connection") + return + } + + var pullRequestCommentCreated = BitbucketCommentCreatedEvent{} + var repoPush = BitbucketPushEvent{} + var pullRequestCreated = BitbucketPullRequestCreatedEvent{} + + bodyBytes, err := io.ReadAll(c.Request.Body) + if err != nil { + c.String(http.StatusBadRequest, "Error reading request body", err) + return + } + + verifySignature(c, bodyBytes, bitbucketWebhookSecret) + + switch eventKey { + case "pullrequest:comment_created": + err := json.Unmarshal(bodyBytes, &pullRequestCommentCreated) + if err != nil { + log.Printf("error parsing pullrequest:comment_created event: %v", err) + log.Printf("error parsing pullrequest:comment_created event: %v", err) + } + handleIssueCommentEventBB(ee.BitbucketProvider, &pullRequestCommentCreated, ee.CiBackendProvider, orgId, bitbucketAccessToken) + return + case "pullrequest:created": + err := json.Unmarshal(bodyBytes, &pullRequestCreated) + if err != nil { + log.Printf("error parsing pullrequest:created event: %v", err) + } + log.Printf("pullrequest:created") + return + case "repo:push": + err := json.Unmarshal(bodyBytes, &repoPush) + if err != nil { + log.Printf("error parsing repo:push event: %v", err) + } + log.Printf("repo:push") + return + default: + log.Printf("unknown event key: %s", eventKey) + return + } +} + +func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payload *BitbucketCommentCreatedEvent, ciBackendProvider ci_backends.CiBackendProvider, organisationId uint, bbAccessToken string) error { + repoFullName := payload.Repository.FullName + repoOwner := payload.Repository.Owner.Username + repoName := payload.Repository.Name + cloneURL := payload.Repository.Links.HTML.Href + issueNumber := payload.PullRequest.ID + // TODO: fetch right draft status + isDraft := false + commentId := payload.Comment.ID + commentBody := payload.Comment.Content.Raw + branch := payload.PullRequest.Source.Branch.Name + commitSha := payload.PullRequest.Source.Commit.Hash + defaultBranch := payload.PullRequest.Source.Branch.Name + actor := payload.Actor.Nickname + //discussionId := payload.Comment.ID + + log.Printf("clone url %v", cloneURL) + log.Printf("access token :%v", bbAccessToken) + log.Printf("repoowner %v reponame %v", repoOwner, repoName) + + if !strings.HasPrefix(commentBody, "digger") { + log.Printf("comment is not a Digger command, ignoring") + return nil + } + + bbService, bberr := utils.GetBitbucketService(bitbucketProvider, bbAccessToken, repoOwner, repoName, issueNumber) + if bberr != nil { + log.Printf("GetGithubService error: %v", bberr) + return fmt.Errorf("error getting ghService to post error comment") + } + + diggerYmlStr, config, projectsGraph, err := utils.GetDiggerConfigForBitbucketBranch(bitbucketProvider, bbAccessToken, repoFullName, repoOwner, repoName, cloneURL, branch, issueNumber) + if err != nil { + log.Printf("getDiggerConfigForPR error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Could not load digger config, error: %v", err)) + return fmt.Errorf("error getting digger config") + } + + err = bbService.CreateCommentReaction(strconv.Itoa(commentId), string(dg_github.GithubCommentEyesReaction)) + if err != nil { + log.Printf("CreateCommentReaction error: %v", err) + } + + if !config.AllowDraftPRs && isDraft { + log.Printf("AllowDraftPRs is disabled, skipping PR: %v", issueNumber) + return nil + } + + commentReporter, err := utils.InitCommentReporter(bbService, issueNumber, ":construction_worker: Digger starting....") + if err != nil { + log.Printf("Error initializing comment reporter: %v", err) + return fmt.Errorf("error initializing comment reporter") + } + + diggerCommand, err := scheduler.GetCommandFromComment(commentBody) + if err != nil { + log.Printf("unknown digger command in comment: %v", commentBody) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Could not recognise comment, error: %v", err)) + return fmt.Errorf("unknown digger command in comment %v", err) + } + + prBranchName, _, err := bbService.GetBranchName(issueNumber) + if err != nil { + log.Printf("GetBranchName error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: GetBranchName error: %v", err)) + return fmt.Errorf("error while fetching branch name") + } + + impactedProjects, impactedProjectsSourceMapping, requestedProject, _, err := generic.ProcessIssueCommentEvent(issueNumber, commentBody, config, projectsGraph, bbService) + if err != nil { + log.Printf("Error processing event: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Error processing event: %v", err)) + return fmt.Errorf("error processing event") + } + log.Printf("GitHub IssueComment event processed successfully\n") + + // perform unlocking in backend + if config.PrLocks { + for _, project := range impactedProjects { + prLock := dg_locking.PullRequestLock{ + InternalLock: locking.BackendDBLock{ + OrgId: organisationId, + }, + CIService: bbService, + Reporter: comment_updater.NoopReporter{}, + ProjectName: project.Name, + ProjectNamespace: repoFullName, + PrNumber: issueNumber, + } + err = dg_locking.PerformLockingActionFromCommand(prLock, *diggerCommand) + if err != nil { + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Failed perform lock action on project: %v %v", project.Name, err)) + return fmt.Errorf("failed perform lock action on project: %v %v", project.Name, err) + } + } + } + + // if commands are locking or unlocking we don't need to trigger any jobs + if *diggerCommand == scheduler.DiggerCommandUnlock || + *diggerCommand == scheduler.DiggerCommandLock { + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":white_check_mark: Command %v completed successfully", *diggerCommand)) + return nil + } + + jobs, _, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, requestedProject, config.Workflows, prBranchName, defaultBranch) + if err != nil { + log.Printf("Error converting event to jobs: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Error converting event to jobs: %v", err)) + return fmt.Errorf("error converting event to jobs") + } + log.Printf("GitHub IssueComment event converted to Jobs successfully\n") + + err = utils.ReportInitialJobsStatus(commentReporter, jobs) + if err != nil { + log.Printf("Failed to comment initial status for jobs: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Failed to comment initial status for jobs: %v", err)) + return fmt.Errorf("failed to comment initial status for jobs") + } + + if len(jobs) == 0 { + log.Printf("no projects impacated, succeeding") + // This one is for aggregate reporting + err = utils.SetPRStatusForJobs(bbService, issueNumber, jobs) + return nil + } + + err = utils.SetPRStatusForJobs(bbService, issueNumber, jobs) + if err != nil { + log.Printf("error setting status for PR: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: error setting status for PR: %v", err)) + fmt.Errorf("error setting status for PR: %v", err) + } + + impactedProjectsMap := make(map[string]dg_configuration.Project) + for _, p := range impactedProjects { + impactedProjectsMap[p.Name] = p + } + + impactedProjectsJobMap := make(map[string]scheduler.Job) + for _, j := range jobs { + impactedProjectsJobMap[j.ProjectName] = j + } + + commentId64, err := strconv.ParseInt(commentReporter.CommentId, 10, 64) + if err != nil { + log.Printf("ParseInt err: %v", err) + return fmt.Errorf("parseint error: %v", err) + } + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false) + if err != nil { + log.Printf("ConvertJobsToDiggerJobs error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) + return fmt.Errorf("error convertingjobs") + } + + if config.CommentRenderMode == dg_configuration.CommentRenderModeGroupByModule && + (*diggerCommand == scheduler.DiggerCommandPlan || *diggerCommand == scheduler.DiggerCommandApply) { + + sourceDetails, err := comment_updater.PostInitialSourceComments(bbService, issueNumber, impactedProjectsSourceMapping) + if err != nil { + log.Printf("PostInitialSourceComments error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: PostInitialSourceComments error: %v", err)) + return fmt.Errorf("error posting initial comments") + } + batch, err := models.DB.GetDiggerBatch(batchId) + if err != nil { + log.Printf("GetDiggerBatch error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: PostInitialSourceComments error: %v", err)) + return fmt.Errorf("error getting digger batch") + } + + batch.SourceDetails, err = json.Marshal(sourceDetails) + if err != nil { + log.Printf("sourceDetails, json Marshal error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: json Marshal error: %v", err)) + return fmt.Errorf("error marshalling sourceDetails") + } + err = models.DB.UpdateDiggerBatch(batch) + if err != nil { + log.Printf("UpdateDiggerBatch error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: UpdateDiggerBatch error: %v", err)) + return fmt.Errorf("error updating digger batch") + } + } + + segment.Track(strconv.Itoa(int(organisationId)), "backend_trigger_job") + + ciBackend, err := ciBackendProvider.GetCiBackend( + ci_backends.CiBackendOptions{ + RepoName: repoName, + RepoOwner: repoOwner, + RepoFullName: repoFullName, + // BB data + }, + ) + if err != nil { + log.Printf("GetCiBackend error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: GetCiBackend error: %v", err)) + return fmt.Errorf("error fetching ci backed %v", err) + } + err = controllers.TriggerDiggerJobs(ciBackend, repoFullName, repoOwner, repoName, batchId, issueNumber, bbService, nil) + if err != nil { + log.Printf("TriggerDiggerJobs error: %v", err) + utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err)) + return fmt.Errorf("error triggering Digger Jobs") + } + return nil +} diff --git a/ee/backend/controllers/bitbucket_utils.go b/ee/backend/controllers/bitbucket_utils.go new file mode 100644 index 000000000..c6934022e --- /dev/null +++ b/ee/backend/controllers/bitbucket_utils.go @@ -0,0 +1,208 @@ +package controllers + +import "time" + +// Common structures used across different event types + +type Actor struct { + AccountID string `json:"account_id"` + DisplayName string `json:"display_name"` + Nickname string `json:"nickname,omitempty"` + UUID string `json:"uuid"` + Type string `json:"type"` + Links Links `json:"links"` +} + +type Links struct { + Self LinkItem `json:"self"` + HTML LinkItem `json:"html"` + Avatar LinkItem `json:"avatar,omitempty"` + Branches *LinkItem `json:"branches,omitempty"` + Commits *LinkItem `json:"commits,omitempty"` + Clone []CloneLink `json:"clone,omitempty"` +} + +type LinkItem struct { + Href string `json:"href"` +} + +type CloneLink struct { + Href string `json:"href"` + Name string `json:"name"` +} + +type Repository struct { + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + IsPrivate bool `json:"is_private"` + Owner Owner `json:"owner"` + Website string `json:"website,omitempty"` + SCM string `json:"scm"` + Description string `json:"description,omitempty"` + Links Links `json:"links"` + Project Project `json:"project,omitempty"` + ForkPolicy string `json:"fork_policy,omitempty"` + CreatedOn time.Time `json:"created_on,omitempty"` + UpdatedOn time.Time `json:"updated_on,omitempty"` + Size int `json:"size,omitempty"` + Language string `json:"language,omitempty"` + HasIssues bool `json:"has_issues,omitempty"` + HasWiki bool `json:"has_wiki,omitempty"` + MainBranch *Branch `json:"mainbranch,omitempty"` +} + +type Owner struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + AccountID string `json:"account_id,omitempty"` + UUID string `json:"uuid"` + Type string `json:"type"` + Links Links `json:"links"` +} + +type Project struct { + Key string `json:"key"` + UUID string `json:"uuid"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Links Links `json:"links"` + Type string `json:"type"` + IsPrivate bool `json:"is_private,omitempty"` + CreatedOn time.Time `json:"created_on,omitempty"` + UpdatedOn time.Time `json:"updated_on,omitempty"` +} + +type Branch struct { + Name string `json:"name"` + Type string `json:"type"` +} + +// -- BitbucketPushEvent (repo:push) -- + +type BitbucketPushEvent struct { + Actor Actor `json:"actor"` + Repository Repository `json:"repository"` + Push Push `json:"push"` +} + +type Push struct { + Changes []Change `json:"changes"` +} + +type Change struct { + Forced bool `json:"forced"` + Old Reference `json:"old"` + New Reference `json:"new"` + Created bool `json:"created"` + Closed bool `json:"closed"` + Commits []Commit `json:"commits"` + Truncated bool `json:"truncated"` +} + +type Reference struct { + Name string `json:"name"` + Type string `json:"type"` + Target CommitInfo `json:"target,omitempty"` +} + +type CommitInfo struct { + Hash string `json:"hash"` + Author Author `json:"author,omitempty"` + Message string `json:"message"` + Date time.Time `json:"date"` + Parents []Parent `json:"parents,omitempty"` + Type string `json:"type"` + Links Links `json:"links"` +} + +type Author struct { + Raw string `json:"raw"` + User Actor `json:"user,omitempty"` +} + +type Parent struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links Links `json:"links"` +} + +type Commit struct { + Hash string `json:"hash"` + Type string `json:"type"` + Message string `json:"message"` + Author Author `json:"author"` + Links Links `json:"links"` + Date time.Time `json:"date"` + Parents []Parent `json:"parents,omitempty"` +} + +// -- BitbucketPullRequestCreatedEvent (pullrequest:created) -- + +type BitbucketPullRequestCreatedEvent struct { + Actor Actor `json:"actor"` + Repository Repository `json:"repository"` + PullRequest PullRequest `json:"pullrequest"` +} + +type PullRequest struct { + ID int `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + State string `json:"state"` + Author Actor `json:"author"` + Source PREndpoint `json:"source"` + Destination PREndpoint `json:"destination"` + MergeCommit *CommitInfo `json:"merge_commit,omitempty"` + ClosedBy *Actor `json:"closed_by,omitempty"` + CreatedOn time.Time `json:"created_on"` + UpdatedOn time.Time `json:"updated_on"` + CommentCount int `json:"comment_count"` + TaskCount int `json:"task_count"` + CloseSourceBranch bool `json:"close_source_branch"` + Type string `json:"type"` + Links Links `json:"links"` + Summary Content `json:"summary"` + Reviewers []Actor `json:"reviewers"` + Participants []Actor `json:"participants"` +} + +type PREndpoint struct { + Branch Branch `json:"branch"` + Commit CommitInfo `json:"commit"` + Repository Repository `json:"repository"` +} + +type Content struct { + Raw string `json:"raw"` + HTML string `json:"html"` + Markup string `json:"markup"` +} + +// -- BitbucketCommentCreatedEvent (repo:commit_comment_created and pullrequest:comment_created) -- + +type BitbucketCommentCreatedEvent struct { + Actor Actor `json:"actor"` + Repository Repository `json:"repository"` + Comment Comment `json:"comment"` + Commit *CommitInfo `json:"commit,omitempty"` // For repo:commit_comment_created + PullRequest *PullRequest `json:"pullrequest,omitempty"` // For pullrequest:comment_created +} + +type Comment struct { + ID int `json:"id"` + Content Content `json:"content"` + CreatedOn time.Time `json:"created_on"` + UpdatedOn time.Time `json:"updated_on"` + User Actor `json:"user"` + Links Links `json:"links"` + Parent *Comment `json:"parent,omitempty"` // For replies to comments + Inline *Inline `json:"inline,omitempty"` // For inline comments +} + +type Inline struct { + Path string `json:"path"` + From *int `json:"from,omitempty"` + To *int `json:"to,omitempty"` +} diff --git a/ee/backend/controllers/github.go b/ee/backend/controllers/github.go index febe52e97..1bcf975f8 100644 --- a/ee/backend/controllers/github.go +++ b/ee/backend/controllers/github.go @@ -103,7 +103,7 @@ func GithubAppConnections(c *gin.Context) { return } - var connections []models.GithubAppConnection + var connections []models.VCSConnection // GORM query result := models.DB.GormDB.Where("organisation_id = ?", orgId).Find(&connections) @@ -192,7 +192,7 @@ func (d DiggerEEController) GithubAppConnectionsConfirm(c *gin.Context) { return } - _, err = models.DB.CreateGithubAppConnection(cfg.GetName(), cfg.GetID(), cfg.GetClientID(), clientSecretEnc, webhookSecretEnc, PEMEnc, PEM64Enc, *cfg.Owner.Login, cfg.GetHTMLURL(), orgId) + _, err = models.DB.CreateVCSConnection(cfg.GetName(), cfg.GetID(), cfg.GetClientID(), clientSecretEnc, webhookSecretEnc, PEMEnc, PEM64Enc, *cfg.Owner.Login, cfg.GetHTMLURL(), "", "", orgId) if err != nil { log.Printf("failed to create github app connection record: %v", err) c.String(500, fmt.Sprintf("Failed to create github app record on callback")) @@ -215,7 +215,7 @@ func (d DiggerEEController) GithubAppConnectionsDelete(c *gin.Context) { c.Error(fmt.Errorf("Ignoring callback, missing code query parameter")) } - connection := models.GithubAppConnection{} + connection := models.VCSConnection{} result := models.DB.GormDB.Where("id = ?", connectionId).Delete(&connection) if result.Error != nil { log.Printf("error while deleting record %v: %v", connectionId, result.Error.Error()) diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index e7c71aa24..344e92d7f 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -30,6 +30,7 @@ import ( type DiggerEEController struct { GithubClientProvider utils.GithubClientProvider GitlabProvider utils.GitlabProvider + BitbucketProvider utils.BitbucketProvider CiBackendProvider ci_backends.CiBackendProvider } @@ -164,7 +165,7 @@ func handlePushEvent(gh utils.GitlabProvider, payload *gitlab.PushEvent, organis } models.DB.UpdateRepoDiggerConfig(organisationId, *config, repo, isMainBranch) return nil - }) + }, "") if err != nil { return fmt.Errorf("error while cloning repo: %v", err) } @@ -218,7 +219,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab return fmt.Errorf("error getting ghService to post error comment") } - diggeryamlStr, config, projectsGraph, err := utils.GetDiggerConfigForBranch(gitlabProvider, projectId, repoFullName, repoOwner, repoName, cloneURL, branch, prNumber, discussionId) + diggeryamlStr, config, projectsGraph, err := utils.GetDiggerConfigForBranchGitlab(gitlabProvider, projectId, repoFullName, repoOwner, repoName, cloneURL, branch, prNumber, discussionId) if err != nil { log.Printf("getDiggerConfigForPR error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: Could not load digger config, error: %v", err)) @@ -407,7 +408,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla return fmt.Errorf("error getting ghService to post error comment") } - diggerYmlStr, config, projectsGraph, err := utils.GetDiggerConfigForBranch(gitlabProvider, projectId, repoFullName, repoOwner, repoName, cloneURL, branch, issueNumber, discussionId) + diggerYmlStr, config, projectsGraph, err := utils.GetDiggerConfigForBranchGitlab(gitlabProvider, projectId, repoFullName, repoOwner, repoName, cloneURL, branch, issueNumber, discussionId) if err != nil { log.Printf("getDiggerConfigForPR error: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: Could not load digger config, error: %v", err)) diff --git a/ee/backend/main.go b/ee/backend/main.go index fe97ae1f0..775b99887 100644 --- a/ee/backend/main.go +++ b/ee/backend/main.go @@ -44,11 +44,13 @@ func main() { eeController := controllers.DiggerEEController{ GithubClientProvider: githubProvider, GitlabProvider: utils.GitlabClientProvider{}, + BitbucketProvider: utils.BitbucketClientProvider{}, CiBackendProvider: ci_backends2.EEBackendProvider{}, } r.POST("/get-spec", eeController.GetSpec) r.POST("/gitlab-webhook", eeController.GitlabWebHookHandler) + r.POST("/bitbucket-webhook", eeController.BitbucketWebhookHandler) githubGroup := r.Group("/github") githubGroup.Use(middleware.GetWebMiddleware()) diff --git a/ee/backend/providers/github/providers.go b/ee/backend/providers/github/providers.go index 47a836793..845c2506b 100644 --- a/ee/backend/providers/github/providers.go +++ b/ee/backend/providers/github/providers.go @@ -44,7 +44,7 @@ func (gh DiggerGithubEEClientProvider) Get(githubAppId int64, installationId int if githubAppPrivateKeyB64 == "" { log.Printf("Info: could not find env variable for private key, attempting to find via connection ID") - connectionEnc, err := models.DB.GetGithubAppConnection(githubAppId) + connectionEnc, err := models.DB.GetVCSConnection(githubAppId) if err != nil { log.Printf("could not find app using app id: %v", githubAppId) return nil, nil, fmt.Errorf("could not find app using app id: %v", githubAppId) @@ -108,7 +108,7 @@ func (gh DiggerGithubEEClientProvider) FetchCredentials(githubAppId string) (str log.Printf("Info: client ID and secret env variables not set, trying to find app via connection ID") - connectionEnc, err := models.DB.GetGithubAppConnection(githubAppId) + connectionEnc, err := models.DB.GetVCSConnection(githubAppId) if err != nil { log.Printf("could not find app using githubAppId id: %v", githubAppId) return "", "", "", "", fmt.Errorf("could not find app using githubAppId id: %v", githubAppId) diff --git a/ee/cli/cmd/digger/root.go b/ee/cli/cmd/digger/root.go index c89d932c2..30ca27146 100644 --- a/ee/cli/cmd/digger/root.go +++ b/ee/cli/cmd/digger/root.go @@ -2,10 +2,10 @@ package main import ( "fmt" - "github.com/diggerhq/digger/cli/pkg/bitbucket" "github.com/diggerhq/digger/cli/pkg/utils" "github.com/diggerhq/digger/libs/backendapi" "github.com/diggerhq/digger/libs/ci" + "github.com/diggerhq/digger/libs/ci/bitbucket" orchestrator_github "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/comment_utils/reporting" "github.com/diggerhq/digger/libs/locking" diff --git a/ee/cli/pkg/policy/policy.go b/ee/cli/pkg/policy/policy.go index 720dbbd8b..c53983a5d 100644 --- a/ee/cli/pkg/policy/policy.go +++ b/ee/cli/pkg/policy/policy.go @@ -62,7 +62,7 @@ func GetPrefixesForPath(path string, fileName string) []string { func (p DiggerRepoPolicyProvider) getPolicyFileContents(repo string, projectName string, projectDir string, fileName string) (string, error) { var contents string - err := utils.CloneGitRepoAndDoAction(p.ManagementRepoUrl, "main", p.GitToken, func(basePath string) error { + err := utils.CloneGitRepoAndDoAction(p.ManagementRepoUrl, "main", p.GitToken, "", func(basePath string) error { // we start with the project directory path prefixes as the highest priority prefixes := GetPrefixesForPath(path.Join(basePath, projectDir), fileName) diff --git a/ee/cli/pkg/utils/github.go b/ee/cli/pkg/utils/github.go index e977c81ef..3655ac6f3 100644 --- a/ee/cli/pkg/utils/github.go +++ b/ee/cli/pkg/utils/github.go @@ -16,9 +16,9 @@ func createTempDir() string { type action func(string) error -func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action action) error { +func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, tokenUsername string, action action) error { dir := createTempDir() - git := utils.NewGitShellWithTokenAuth(dir, token) + git := utils.NewGitShellWithTokenAuth(dir, token, tokenUsername) err := git.Clone(repoUrl, branch) if err != nil { return err diff --git a/ee/drift/tasks/github.go b/ee/drift/tasks/github.go index c54468bed..f7e6af4b9 100644 --- a/ee/drift/tasks/github.go +++ b/ee/drift/tasks/github.go @@ -42,7 +42,7 @@ func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId s return fmt.Errorf("error getting github service") } - err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error { + err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) if err != nil { log.Printf("ERROR load digger.yml: %v", err) diff --git a/ee/drift/utils/github.go b/ee/drift/utils/github.go index cbcaf2120..afb200811 100644 --- a/ee/drift/utils/github.go +++ b/ee/drift/utils/github.go @@ -112,7 +112,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 return err } return nil - }) + }, "") if err != nil { log.Printf("Error cloning and loading config: %v", err) return "", nil, nil, nil, fmt.Errorf("error cloning and loading config") diff --git a/go.work.sum b/go.work.sum index ce29a722f..b8c126663 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1167,6 +1167,8 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kataras/blocks v0.0.7 h1:cF3RDY/vxnSRezc7vLFlQFTYXG/yAr1o7WImJuZbzC4= github.com/kataras/blocks v0.0.7/go.mod h1:UJIU97CluDo0f+zEjbnbkeMRlvYORtmc1304EeyXf4I= @@ -1199,7 +1201,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/labstack/echo/v4 v4.10.0 h1:5CiyngihEO4HXsz3vVsJn7f8xAlWwRr3aY6Ih280ZKA= github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjFMSUTkDRNQ= @@ -1369,7 +1370,6 @@ github.com/pterm/pterm v0.12.41/go.mod h1:LW/G4J2A42XlTaPTAGRPvbBfF4UXvHWhC6SN7u github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b h1:aUNXCGgukb4gtY99imuIeoh8Vr0GSwAlYxPAhqZrpFc= github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= @@ -1579,10 +1579,8 @@ golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72 golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -1599,11 +1597,9 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= @@ -1631,10 +1627,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1649,7 +1643,6 @@ golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/cli/pkg/bitbucket/bitbucket.go b/libs/ci/bitbucket/bitbucket.go similarity index 93% rename from cli/pkg/bitbucket/bitbucket.go rename to libs/ci/bitbucket/bitbucket.go index 01afa63e1..03c45410d 100644 --- a/cli/pkg/bitbucket/bitbucket.go +++ b/libs/ci/bitbucket/bitbucket.go @@ -4,11 +4,13 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/diggerhq/digger/libs/ci" - configuration "github.com/diggerhq/digger/libs/digger_config" + "log" "net/http" "strconv" "time" + + "github.com/diggerhq/digger/libs/ci" + configuration "github.com/diggerhq/digger/libs/digger_config" ) // Define the base URL for the Bitbucket API. @@ -41,35 +43,16 @@ func (b BitbucketAPI) sendRequest(method, url string, body []byte) (*http.Respon } type DiffStat struct { - Pagelen int `json:"pagelen"` - Values []struct { - Type string `json:"type"` - Status string `json:"status"` - LinesRemoved int `json:"lines_removed"` - LinesAdded int `json:"lines_added"` - Old struct { - Path string `json:"path"` - EscapedPath string `json:"escaped_path"` - Type string `json:"type"` - Links struct { - Self struct { - Href string `json:"href"` - } `json:"self"` - } `json:"links"` - } `json:"old"` + Values []struct { + Status string `json:"status"` + Old struct { + Path string `json:"path"` + } `json:"old,omitempty"` New struct { - Path string `json:"path"` - EscapedPath string `json:"escaped_path"` - Type string `json:"type"` - Links struct { - Self struct { - Href string `json:"href"` - } `json:"self"` - } `json:"links"` - } `json:"new"` + Path string `json:"path"` + } `json:"new,omitempty"` } `json:"values"` - Page int `json:"page"` - Size int `json:"size"` + Next string `json:"next,omitempty"` } func (b BitbucketAPI) GetChangedFiles(prNumber int) ([]string, error) { @@ -81,10 +64,10 @@ func (b BitbucketAPI) GetChangedFiles(prNumber int) ([]string, error) { } defer resp.Body.Close() + log.Printf("url %v", url) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to get changed files. Status code: %d", resp.StatusCode) } - diffStat := &DiffStat{} err = json.NewDecoder(resp.Body).Decode(diffStat) if err != nil { diff --git a/libs/ci/bitbucket/bitbucket_service.go b/libs/ci/bitbucket/bitbucket_service.go new file mode 100644 index 000000000..7fcf35a48 --- /dev/null +++ b/libs/ci/bitbucket/bitbucket_service.go @@ -0,0 +1,193 @@ +package bitbucket + +import ( + "fmt" + "github.com/caarlos0/env/v11" + "github.com/diggerhq/digger/libs/ci" + "github.com/diggerhq/digger/libs/digger_config" + "github.com/diggerhq/digger/libs/scheduler" + "log" + "strings" +) + +type BitbucketContext struct { + PullRequestID *int `env:"BITBUCKET_PR_ID"` + RepositoryOWNER string `env:"BITBUCKET_REPO_OWNER"` + RepositoryName string `env:"BITBUCKET_REPO_NAME"` + RepositoryFullName string `env:"BITBUCKET_REPO_FULL_NAME"` + Token string `env:"BITBUCKET_TOKEN"` + BitbucketUserName string `env:"BITBUCKET_USER_NAME"` + DiggerCommand string `env:"DIGGER_COMMAND"` + EventType BitbucketEventType `env:"BITBUCKET_EVENT_TYPE"` +} + +type BitbucketEventType string + +func (e BitbucketEventType) String() string { + return string(e) +} + +const ( + PullRequestOpened = BitbucketEventType("pullrequest:created") + PullRequestUpdated = BitbucketEventType("pullrequest:updated") + PullRequestMerged = BitbucketEventType("pullrequest:merged") + PullRequestDeclined = BitbucketEventType("pullrequest:declined") + PullRequestComment = BitbucketEventType("pullrequest:comment") +) + +func ParseBitbucketContext() (*BitbucketContext, error) { + var parsedContext BitbucketContext + + if err := env.Parse(&parsedContext); err != nil { + log.Printf("Error parsing Bitbucket context: %+v\n", err) + return nil, err + } + + log.Printf("Parsed Bitbucket context: %+v\n", parsedContext) + return &parsedContext, nil +} + +func ProcessBitbucketEvent(context *BitbucketContext, diggerConfig *digger_config.DiggerConfig, api BitbucketAPI) ([]digger_config.Project, *digger_config.Project, error) { + var impactedProjects []digger_config.Project + + if context.PullRequestID == nil { + return nil, nil, fmt.Errorf("pull request ID not found") + } + + changedFiles, err := api.GetChangedFiles(*context.PullRequestID) + if err != nil { + return nil, nil, fmt.Errorf("could not get changed files: %v", err) + } + + impactedProjects, _ = diggerConfig.GetModifiedProjects(changedFiles) + + switch context.EventType { + case PullRequestComment: + diggerCommand := strings.ToLower(context.DiggerCommand) + diggerCommand = strings.TrimSpace(diggerCommand) + requestedProject := ci.ParseProjectName(diggerCommand) + + if requestedProject == "" { + return impactedProjects, nil, nil + } + + for _, project := range impactedProjects { + if project.Name == requestedProject { + return impactedProjects, &project, nil + } + } + return nil, nil, fmt.Errorf("requested project not found in modified projects") + default: + return impactedProjects, nil, nil + } +} + +func ConvertBitbucketEventToCommands(event BitbucketEventType, context *BitbucketContext, impactedProjects []digger_config.Project, requestedProject *digger_config.Project, workflows map[string]digger_config.Workflow) ([]scheduler.Job, bool, error) { + jobs := make([]scheduler.Job, 0) + + log.Printf("Converting Bitbucket event to commands, event type: %s\n", event) + + switch event { + case PullRequestOpened, PullRequestUpdated: + for _, project := range impactedProjects { + workflow, ok := workflows[project.Workflow] + if !ok { + return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) + } + + var skipMerge bool + if workflow.Configuration != nil { + skipMerge = workflow.Configuration.SkipMergeCheck + } + + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) + StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) + + jobs = append(jobs, scheduler.Job{ + ProjectName: project.Name, + ProjectDir: project.Dir, + ProjectWorkspace: project.Workspace, + Terragrunt: project.Terragrunt, + OpenTofu: project.OpenTofu, + Pulumi: project.Pulumi, + Commands: workflow.Configuration.OnPullRequestPushed, + ApplyStage: scheduler.ToConfigStage(workflow.Apply), + PlanStage: scheduler.ToConfigStage(workflow.Plan), + PullRequestNumber: context.PullRequestID, + EventName: string(event), + RequestedBy: context.BitbucketUserName, + Namespace: context.RepositoryFullName, + StateEnvVars: stateEnvVars, + CommandEnvVars: commandEnvVars, + StateEnvProvider: StateEnvProvider, + CommandEnvProvider: CommandEnvProvider, + SkipMergeCheck: skipMerge, + }) + } + return jobs, true, nil + + case PullRequestComment: + supportedCommands := []string{"digger plan", "digger apply", "digger unlock", "digger lock"} + coversAllImpactedProjects := true + runForProjects := impactedProjects + + if requestedProject != nil { + if len(impactedProjects) > 1 { + coversAllImpactedProjects = false + runForProjects = []digger_config.Project{*requestedProject} + } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { + return jobs, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) + } + } + + diggerCommand := strings.ToLower(context.DiggerCommand) + diggerCommand = strings.TrimSpace(diggerCommand) + + for _, command := range supportedCommands { + if strings.Contains(diggerCommand, command) { + for _, project := range runForProjects { + workflow, ok := workflows[project.Workflow] + if !ok { + workflow = workflows["default"] + } + + workspace := project.Workspace + workspaceOverride, err := ci.ParseWorkspace(diggerCommand) + if err != nil { + return []scheduler.Job{}, false, err + } + if workspaceOverride != "" { + workspace = workspaceOverride + } + + stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true) + StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(project) + + jobs = append(jobs, scheduler.Job{ + ProjectName: project.Name, + ProjectDir: project.Dir, + ProjectWorkspace: workspace, + Terragrunt: project.Terragrunt, + OpenTofu: project.OpenTofu, + Pulumi: project.Pulumi, + Commands: []string{command}, + ApplyStage: scheduler.ToConfigStage(workflow.Apply), + PlanStage: scheduler.ToConfigStage(workflow.Plan), + PullRequestNumber: context.PullRequestID, + EventName: string(event), + RequestedBy: context.BitbucketUserName, + Namespace: context.RepositoryFullName, + StateEnvVars: stateEnvVars, + CommandEnvVars: commandEnvVars, + StateEnvProvider: StateEnvProvider, + CommandEnvProvider: CommandEnvProvider, + }) + } + } + } + return jobs, coversAllImpactedProjects, nil + + default: + return []scheduler.Job{}, false, fmt.Errorf("unsupported Bitbucket event type: %v", event) + } +} diff --git a/next/controllers/github.go b/next/controllers/github.go index 255fb6007..b14e84bb2 100644 --- a/next/controllers/github.go +++ b/next/controllers/github.go @@ -538,7 +538,7 @@ func getDiggerConfigForBranch(gh next_utils.GithubClientProvider, installationId log.Printf("Error getting changed files: %v", err) return "", nil, nil, nil, fmt.Errorf("error getting changed files") } - err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error { + err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles) diff --git a/next/services/config.go b/next/services/config.go index 8e8fa6329..f00eee483 100644 --- a/next/services/config.go +++ b/next/services/config.go @@ -39,7 +39,7 @@ func GetWorkflowsForRepoAndBranch(gh utils.GithubClientProvider, repoId int64, b var config *dg_configuration.DiggerConfig - err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, commitHash, *token, func(dir string) error { + err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, commitHash, *token, "", func(dir string) error { // we create a blank file if it does not exist err := dg_configuration.CheckOrCreateDiggerFile(dir) if err != nil { From aa74fed67b95ff2480ab93122a011ac1621cc38c Mon Sep 17 00:00:00 2001 From: motatoes Date: Sun, 2 Mar 2025 10:47:34 -0800 Subject: [PATCH 2/6] support bb --- ee/backend/ci_backends/bitbucket_pipeline.go | 39 ++++++++++++++++++++ ee/backend/controllers/bitbucket.go | 38 ++++++++----------- ee/backend/controllers/gitlab.go | 4 +- 3 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 ee/backend/ci_backends/bitbucket_pipeline.go diff --git a/ee/backend/ci_backends/bitbucket_pipeline.go b/ee/backend/ci_backends/bitbucket_pipeline.go new file mode 100644 index 000000000..639cfc439 --- /dev/null +++ b/ee/backend/ci_backends/bitbucket_pipeline.go @@ -0,0 +1,39 @@ +package ci_backends + +import ( + "encoding/json" + "fmt" + orchestrator_bitbucket "github.com/diggerhq/digger/libs/ci/bitbucket" + "github.com/diggerhq/digger/libs/spec" +) + +type BitbucketPipelineCI struct { + Client *orchestrator_bitbucket.BitbucketAPI + RepoOwner string + RepoName string + Branch string +} + +func (bbp BitbucketPipelineCI) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error { + specBytes, err := json.Marshal(spec) + if err != nil { + return fmt.Errorf("could not serialize spec: %v", err) + } + + variables := []interface{}{ + map[string]string{ + "key": "ENVIRONMENT", + "value": "production", + }, + map[string]string{ + "key": "DIGGER_RUN_SPEC", + "value": string(specBytes), + }, + map[string]string{ + "key": "DIGGER_BITBUCKET_ACCESS_TOKEN", + "value": vcsToken, + }, + } + _, err = bbp.Client.TriggerPipeline(bbp.Branch, variables) + return err +} diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go index fcc2f381b..237fd536a 100644 --- a/ee/backend/controllers/bitbucket.go +++ b/ee/backend/controllers/bitbucket.go @@ -12,6 +12,7 @@ import ( "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/segment" "github.com/diggerhq/digger/backend/utils" + ci_backends2 "github.com/diggerhq/digger/ee/backend/ci_backends" "github.com/diggerhq/digger/libs/ci/generic" dg_github "github.com/diggerhq/digger/libs/ci/github" comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting" @@ -116,26 +117,25 @@ func (ee DiggerEEController) BitbucketWebhookHandler(c *gin.Context) { log.Printf("error parsing pullrequest:comment_created event: %v", err) log.Printf("error parsing pullrequest:comment_created event: %v", err) } - handleIssueCommentEventBB(ee.BitbucketProvider, &pullRequestCommentCreated, ee.CiBackendProvider, orgId, bitbucketAccessToken) - return + go handleIssueCommentEventBB(ee.BitbucketProvider, &pullRequestCommentCreated, ee.CiBackendProvider, orgId, bitbucketAccessToken) case "pullrequest:created": err := json.Unmarshal(bodyBytes, &pullRequestCreated) if err != nil { log.Printf("error parsing pullrequest:created event: %v", err) } log.Printf("pullrequest:created") - return case "repo:push": err := json.Unmarshal(bodyBytes, &repoPush) if err != nil { log.Printf("error parsing repo:push event: %v", err) } log.Printf("repo:push") - return default: log.Printf("unknown event key: %s", eventKey) return } + + c.String(http.StatusAccepted, "ok") } func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payload *BitbucketCommentCreatedEvent, ciBackendProvider ci_backends.CiBackendProvider, organisationId uint, bbAccessToken string) error { @@ -149,15 +149,12 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa commentId := payload.Comment.ID commentBody := payload.Comment.Content.Raw branch := payload.PullRequest.Source.Branch.Name - commitSha := payload.PullRequest.Source.Commit.Hash + // TODO: figure why git fetch fails in bb pipeline + commitSha := "" //payload.PullRequest.Source.Commit.Hash defaultBranch := payload.PullRequest.Source.Branch.Name actor := payload.Actor.Nickname //discussionId := payload.Comment.ID - log.Printf("clone url %v", cloneURL) - log.Printf("access token :%v", bbAccessToken) - log.Printf("repoowner %v reponame %v", repoOwner, repoName) - if !strings.HasPrefix(commentBody, "digger") { log.Printf("comment is not a Digger command, ignoring") return nil @@ -212,7 +209,7 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: Error processing event: %v", err)) return fmt.Errorf("error processing event") } - log.Printf("GitHub IssueComment event processed successfully\n") + log.Printf("Bitbucket IssueComment event processed successfully\n") // perform unlocking in backend if config.PrLocks { @@ -286,7 +283,7 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa log.Printf("ParseInt err: %v", err) return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -325,18 +322,13 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa segment.Track(strconv.Itoa(int(organisationId)), "backend_trigger_job") - ciBackend, err := ciBackendProvider.GetCiBackend( - ci_backends.CiBackendOptions{ - RepoName: repoName, - RepoOwner: repoOwner, - RepoFullName: repoFullName, - // BB data - }, - ) - if err != nil { - log.Printf("GetCiBackend error: %v", err) - utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: GetCiBackend error: %v", err)) - return fmt.Errorf("error fetching ci backed %v", err) + // hardcoded bitbucket ci backend for this controller + // TODO: making this configurable based on env variable and connection + ciBackend := ci_backends2.BitbucketPipelineCI{ + RepoName: repoName, + RepoOwner: repoOwner, + Branch: branch, + Client: bbService, } err = controllers.TriggerDiggerJobs(ciBackend, repoFullName, repoOwner, repoName, batchId, issueNumber, bbService, nil) if err != nil { diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index 344e92d7f..54dd780c6 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -157,7 +157,7 @@ func handlePushEvent(gh utils.GitlabProvider, payload *gitlab.PushEvent, organis isMainBranch = false } - err = utils.CloneGitRepoAndDoAction(cloneURL, pushBranch, "", token, func(dir string) error { + err = utils.CloneGitRepoAndDoAction(cloneURL, pushBranch, "", token, "", func(dir string) error { config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) if err != nil { log.Printf("ERROR load digger.yml: %v", err) @@ -165,7 +165,7 @@ func handlePushEvent(gh utils.GitlabProvider, payload *gitlab.PushEvent, organis } models.DB.UpdateRepoDiggerConfig(organisationId, *config, repo, isMainBranch) return nil - }, "") + }) if err != nil { return fmt.Errorf("error while cloning repo: %v", err) } From 1716c40f233c1f18e213f985e2867c171e9ad686 Mon Sep 17 00:00:00 2001 From: motatoes Date: Sun, 2 Mar 2025 10:48:02 -0800 Subject: [PATCH 3/6] bb support --- backend/controllers/cache.go | 4 +- backend/controllers/github.go | 4 +- backend/models/scheduler.go | 1 + backend/utils/bitbucket.go | 2 +- backend/utils/github_test.go | 8 +- backend/utils/gitshell.go | 2 - libs/ci/bitbucket/bitbucket.go | 179 ++++++++++++++++++++++++++++++++- libs/spec/providers.go | 13 +++ 8 files changed, 199 insertions(+), 14 deletions(-) diff --git a/backend/controllers/cache.go b/backend/controllers/cache.go index b72fec23a..1e9edc667 100644 --- a/backend/controllers/cache.go +++ b/backend/controllers/cache.go @@ -67,7 +67,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) { // update the cache here, do it async for immediate response go func() { - err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error { + err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil) @@ -76,7 +76,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) { return err } return nil - }, "") + }) if err != nil { log.Printf("could not load digger config :%v", err) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 4f04b9bb9..16ce41115 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -595,7 +595,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 var diggerYmlStr string var dependencyGraph graph.Graph[string, dg_configuration.Project] - err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error { + err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error { diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir) if err != nil { log.Printf("could not load digger config: %v", err) @@ -607,7 +607,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 return err } return nil - }, "") + }) if err != nil { log.Printf("Error cloning and loading config: %v", err) return "", nil, nil, nil, fmt.Errorf("error cloning and loading config %v", err) diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index c371b106f..6c75259e7 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -20,6 +20,7 @@ type DiggerVCSType string const DiggerVCSGithub DiggerVCSType = "github" const DiggerVCSGitlab DiggerVCSType = "gitlab" +const DiggerVCSBitbucket DiggerVCSType = "bitbucket" type DiggerBatch struct { ID uuid.UUID `gorm:"primary_key"` diff --git a/backend/utils/bitbucket.go b/backend/utils/bitbucket.go index ab8b751bd..144f372f3 100644 --- a/backend/utils/bitbucket.go +++ b/backend/utils/bitbucket.go @@ -59,7 +59,7 @@ func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoF return "", nil, nil, fmt.Errorf("error getting changed files") } - err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error { + err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error { diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml")) diggerYmlStr = string(diggerYmlBytes) config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles) diff --git a/backend/utils/github_test.go b/backend/utils/github_test.go index 17b0f26d9..676f6edda 100644 --- a/backend/utils/github_test.go +++ b/backend/utils/github_test.go @@ -14,14 +14,14 @@ func init() { func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) { f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", f, "") + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", "", f) assert.NotNil(t, err) } func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, f, "") + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, "", f) assert.Nil(t, err) } @@ -32,13 +32,13 @@ func TestGithubCloneWithPrivateRepoAndValidTokenThrowsNoError(t *testing.T) { return } f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, f, "") + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, "", f) assert.Nil(t, err) } func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) { token := os.Getenv("GITHUB_PAT_TOKEN") f := func(d string) error { return nil } - err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, f, "") + err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, "", f) assert.NotNil(t, err) } diff --git a/backend/utils/gitshell.go b/backend/utils/gitshell.go index 6e3ca6477..9abcca0b6 100644 --- a/backend/utils/gitshell.go +++ b/backend/utils/gitshell.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "log" "net/url" "os" "os/exec" @@ -129,7 +128,6 @@ func (g *GitShell) Clone(repoURL, branch string) error { args = append(args, "-b", branch) } - log.Printf("auth url: %v", authURL) args = append(args, "--depth", "1") args = append(args, "--single-branch", authURL, g.workDir) diff --git a/libs/ci/bitbucket/bitbucket.go b/libs/ci/bitbucket/bitbucket.go index 03c45410d..95d088a7d 100644 --- a/libs/ci/bitbucket/bitbucket.go +++ b/libs/ci/bitbucket/bitbucket.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "fmt" + "io" + "io/ioutil" "log" "net/http" "strconv" @@ -81,6 +83,26 @@ func (b BitbucketAPI) GetChangedFiles(prNumber int) ([]string, error) { return files, nil } +type BitbucketCommentResponse struct { + ID int `json:"id"` + Content struct { + Raw string `json:"raw"` + } `json:"content"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + CreatedOn string `json:"created_on"` + User struct { + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + } `json:"user"` +} + func (b BitbucketAPI) PublishComment(prNumber int, comment string) (*ci.Comment, error) { url := fmt.Sprintf("%s/repositories/%s/%s/pullrequests/%d/comments", bitbucketBaseURL, b.RepoWorkspace, b.RepoName, prNumber) @@ -105,7 +127,25 @@ func (b BitbucketAPI) PublishComment(prNumber int, comment string) (*ci.Comment, return nil, fmt.Errorf("failed to publish comment. Status code: %d", resp.StatusCode) } - return nil, nil + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response:", err) + return nil, fmt.Errorf("error reading response: %v", err) + } + + var commentResponse BitbucketCommentResponse + if err := json.Unmarshal(body, &commentResponse); err != nil { + fmt.Println("Error parsing response:", err) + return nil, fmt.Errorf("error parsing response: %v", err) + } + + res := ci.Comment{ + Id: strconv.Itoa(commentResponse.ID), + DiscussionId: "", + Body: &comment, + Url: commentResponse.Links.HTML.Href, + } + return &res, nil } func (svc BitbucketAPI) ListIssues() ([]*ci.Issue, error) { @@ -123,8 +163,10 @@ func (svc BitbucketAPI) UpdateIssue(ID int64, title string, body string) (int64, func (b BitbucketAPI) EditComment(prNumber int, id string, comment string) error { url := fmt.Sprintf("%s/repositories/%s/%s/pullrequests/%d/comments/%s", bitbucketBaseURL, b.RepoWorkspace, b.RepoName, prNumber, id) - commentBody := map[string]string{ - "content": comment, + commentBody := map[string]interface{}{ + "content": map[string]string{ + "raw": comment, + }, } commentJSON, err := json.Marshal(commentBody) @@ -475,6 +517,137 @@ func (b BitbucketAPI) GetUserTeams(organisation string, user string) ([]string, return nil, fmt.Errorf("not implemented") } +type PipelineResponse struct { + UUID string `json:"uuid"` + BuildNumber int `json:"build_number"` + CreatedOn string `json:"created_on"` + Creator struct { + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + AccountID string `json:"account_id"` + Nickname string `json:"nickname"` + Type string `json:"type"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + } `json:"creator"` + Repository struct { + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + Type string `json:"type"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + } `json:"repository"` + Target struct { + Type string `json:"type"` + RefName string `json:"ref_name"` + RefType string `json:"ref_type"` + Selector struct { + Type string `json:"type"` + } `json:"selector"` + Commit struct { + Type string `json:"type"` + Hash string `json:"hash"` + } `json:"commit"` + } `json:"target"` + Trigger struct { + Name string `json:"name"` + Type string `json:"type"` + } `json:"trigger"` + State struct { + Name string `json:"name"` + Type string `json:"type"` + Stage struct { + Name string `json:"name"` + Type string `json:"type"` + } `json:"stage"` + } `json:"state"` + Variables []struct { + Key string `json:"key"` + Value string `json:"value"` + Secured bool `json:"secured"` + UUID string `json:"uuid"` + } `json:"variables"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` +} + +// trigger pipeline from a specific branch +func (b BitbucketAPI) TriggerPipeline(branch string, variables []interface{}) (string, error) { + url := fmt.Sprintf("%s/repositories/%s/%s/pipelines", bitbucketBaseURL, b.RepoWorkspace, b.RepoName) + + log.Printf("pipeline trigger url: %v branch %v", url, branch) + triggerOptions := map[string]interface{}{ + "target": map[string]interface{}{ + "ref_type": "branch", + "type": "pipeline_ref_target", + "ref_name": branch, + "selector": map[string]interface{}{ + "type": "custom", + "pattern": "digger", + }, + }, + "variables": variables, + } + + triggerJSON, err := json.Marshal(triggerOptions) + if err != nil { + return "", err + } + + resp, err := b.sendRequest("POST", url, triggerJSON) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + log.Printf("the response from bitbucket is: %v", string(body)) + return "", fmt.Errorf("failed to trigger pipeline: %d", resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response:", err) + return "", fmt.Errorf("error reading response: %v", err) + } + + var triggerPipelineResponse PipelineResponse + if err := json.Unmarshal(body, &triggerPipelineResponse); err != nil { + fmt.Println("Error parsing response:", err) + return "", fmt.Errorf("error parsing response: %v", err) + } + + log.Printf(triggerPipelineResponse.Links.HTML.Href) + return "", nil + +} + func FindImpactedProjectsInBitbucket(diggerConfig *configuration.DiggerConfig, prNumber int, prService ci.PullRequestService) ([]configuration.Project, error) { changedFiles, err := prService.GetChangedFiles(prNumber) diff --git a/libs/spec/providers.go b/libs/spec/providers.go index 6ad1aac49..15b62f3d9 100644 --- a/libs/spec/providers.go +++ b/libs/spec/providers.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" backend2 "github.com/diggerhq/digger/libs/backendapi" "github.com/diggerhq/digger/libs/ci" + "github.com/diggerhq/digger/libs/ci/bitbucket" "github.com/diggerhq/digger/libs/ci/github" "github.com/diggerhq/digger/libs/ci/gitlab" "github.com/diggerhq/digger/libs/comment_utils/reporting" @@ -195,6 +196,18 @@ func (v VCSProviderBasic) GetPrService(vcsSpec VcsSpec) (ci.PullRequestService, return nil, fmt.Errorf("failed to get gitlab service, could not parse context: %v", err) } return gitlab.NewGitLabService(token, context, "") + case "bitbucket": + token := os.Getenv("DIGGER_BITBUCKET_ACCESS_TOKEN") + if token == "" { + return nil, fmt.Errorf("failed to get bitbucket service: GITLAB_TOKEN not specified") + } + return bitbucket.BitbucketAPI{ + AuthToken: token, + HttpClient: http.Client{}, + RepoWorkspace: vcsSpec.RepoOwner, + RepoName: vcsSpec.RepoName, + }, nil + default: return nil, fmt.Errorf("could not get PRService, unknown type %v", vcsSpec.VcsType) } From 9831a4884af20848f3f11a990760882e023a9091 Mon Sep 17 00:00:00 2001 From: motatoes Date: Sun, 2 Mar 2025 10:54:37 -0800 Subject: [PATCH 4/6] support org service --- libs/spec/providers.go | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/libs/spec/providers.go b/libs/spec/providers.go index 15b62f3d9..0e39138c5 100644 --- a/libs/spec/providers.go +++ b/libs/spec/providers.go @@ -186,16 +186,6 @@ func (v VCSProviderBasic) GetPrService(vcsSpec VcsSpec) (ci.PullRequestService, return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified") } return github.GithubServiceProviderBasic{}.NewService(token, vcsSpec.RepoName, vcsSpec.RepoOwner) - case "gitlab": - token := os.Getenv("GITLAB_TOKEN") - if token == "" { - return nil, fmt.Errorf("failed to get gitlab service: GITLAB_TOKEN not specified") - } - context, err := gitlab.ParseGitLabContext() - if err != nil { - return nil, fmt.Errorf("failed to get gitlab service, could not parse context: %v", err) - } - return gitlab.NewGitLabService(token, context, "") case "bitbucket": token := os.Getenv("DIGGER_BITBUCKET_ACCESS_TOKEN") if token == "" { @@ -207,6 +197,16 @@ func (v VCSProviderBasic) GetPrService(vcsSpec VcsSpec) (ci.PullRequestService, RepoWorkspace: vcsSpec.RepoOwner, RepoName: vcsSpec.RepoName, }, nil + case "gitlab": + token := os.Getenv("GITLAB_TOKEN") + if token == "" { + return nil, fmt.Errorf("failed to get gitlab service: GITLAB_TOKEN not specified") + } + context, err := gitlab.ParseGitLabContext() + if err != nil { + return nil, fmt.Errorf("failed to get gitlab service, could not parse context: %v", err) + } + return gitlab.NewGitLabService(token, context, "") default: return nil, fmt.Errorf("could not get PRService, unknown type %v", vcsSpec.VcsType) @@ -223,6 +223,17 @@ func (v VCSProviderBasic) GetOrgService(vcsSpec VcsSpec) (ci.OrgService, error) return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified") } return github.GithubServiceProviderBasic{}.NewService(token, vcsSpec.RepoName, vcsSpec.RepoOwner) + case "bitbucket": + token := os.Getenv("DIGGER_BITBUCKET_ACCESS_TOKEN") + if token == "" { + return nil, fmt.Errorf("failed to get bitbucket service: GITLAB_TOKEN not specified") + } + return bitbucket.BitbucketAPI{ + AuthToken: token, + HttpClient: http.Client{}, + RepoWorkspace: vcsSpec.RepoOwner, + RepoName: vcsSpec.RepoName, + }, nil case "gitlab": token := os.Getenv("GITLAB_TOKEN") if token == "" { From 81c59cc1649f5a4164a10679203e149644c72a05 Mon Sep 17 00:00:00 2001 From: motatoes Date: Sun, 2 Mar 2025 13:31:56 -0800 Subject: [PATCH 5/6] add is support markdown support --- backend/controllers/github.go | 4 ++-- backend/controllers/github_test.go | 8 ++++---- backend/migrations/20250302190926.sql | 2 ++ backend/migrations/atlas.sum | 3 ++- backend/models/scheduler.go | 4 +++- backend/models/storage.go | 3 ++- backend/models/storage_test.go | 2 +- backend/services/spec.go | 25 +++++++++++++++++++++++++ backend/tasks/runs_test.go | 2 +- backend/utils/graphs.go | 4 ++-- cli/pkg/spec/spec.go | 2 +- ee/backend/controllers/bitbucket.go | 6 +++--- ee/backend/controllers/gitlab.go | 4 ++-- ee/backend/hooks/github.go | 2 +- libs/ci/bitbucket/bitbucket.go | 12 ++---------- libs/spec/providers.go | 10 +++++++--- 16 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 backend/migrations/20250302190926.sql diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 16ce41115..13614224a 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -522,7 +522,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR aiSummaryCommentId = aiSummaryComment.Id } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -930,7 +930,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu aiSummaryCommentId = aiSummaryComment.Id } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index 2ddf74216..2d6fab6e6 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -731,7 +731,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) { graph, err := configuration.CreateProjectDependencyGraph(projects) assert.NoError(t, err) - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, nil) assert.NoError(t, err) assert.Equal(t, 1, len(result)) parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID) @@ -760,7 +760,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -793,7 +793,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil) assert.NoError(t, err) assert.Equal(t, 2, len(result)) parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID) @@ -838,7 +838,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) { projectMap["555"] = project5 projectMap["666"] = project6 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, nil) assert.NoError(t, err) assert.Equal(t, 6, len(result)) parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID) diff --git a/backend/migrations/20250302190926.sql b/backend/migrations/20250302190926.sql new file mode 100644 index 000000000..47e2170c0 --- /dev/null +++ b/backend/migrations/20250302190926.sql @@ -0,0 +1,2 @@ +-- Modify "digger_batches" table +ALTER TABLE "public"."digger_batches" ADD COLUMN "vcs_connection_id" bigint NULL, ADD CONSTRAINT "fk_digger_batches_vcs_connection" FOREIGN KEY ("vcs_connection_id") REFERENCES "public"."github_app_connections" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index 2ba62ac92..9036fbdf1 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:uESS+Cvtzp9ftW1x7GX98EeuQMvEHX6PZgBN0UNOzMM= +h1:pGuhEh2gQrZkYbjLCxVK+ZHO3jlFj99jjAO96gKlWlc= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -43,3 +43,4 @@ h1:uESS+Cvtzp9ftW1x7GX98EeuQMvEHX6PZgBN0UNOzMM= 20250221044813.sql h1:PPcVcoMaMY5DXyoptQYeBOEwofrzIfyKcWRd3S12I2U= 20250224152926.sql h1:EjoFpfeoCpk/SjSo2i7sajKCR3t7YCn+1ZgGJrT0L9Y= 20250226185150.sql h1:K7e/3Zy2wSTqKa3iYpIb02GTAniYSXHObTIqOV9aOhM= +20250302190926.sql h1:F3FnaGnZv1Hwmg6W9Nacg5fbdiYbZGgS/mkuogtCso0= diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 6c75259e7..7b5b0f97e 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -39,7 +39,9 @@ type DiggerBatch struct { BatchType orchestrator_scheduler.DiggerCommand ReportTerraformOutputs bool // used for module source grouping comments - SourceDetails []byte + SourceDetails []byte + VCSConnectionId *uint `` + VCSConnection VCSConnection } type DiggerJob struct { diff --git a/backend/models/storage.go b/backend/models/storage.go index bc6cacde7..a93882249 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -620,11 +620,12 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) { return batch, nil } -func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool) (*DiggerBatch, error) { +func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, VCSConnectionId *uint) (*DiggerBatch, error) { uid := uuid.New() batch := &DiggerBatch{ ID: uid, VCS: vcsType, + VCSConnectionId: VCSConnectionId, GithubInstallationId: githubInstallationId, RepoOwner: repoOwner, RepoName: repoName, diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index 509ed2343..17357cad6 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -151,7 +151,7 @@ func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { resourcesUpdated := uint(2) resourcesDeleted := uint(3) - batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false) + batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, nil) assert.NoError(t, err) job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml") diff --git a/backend/services/spec.go b/backend/services/spec.go index 152f7199c..d8a701e51 100644 --- a/backend/services/spec.go +++ b/backend/services/spec.go @@ -34,6 +34,31 @@ func GetVCSTokenFromJob(job models.DiggerJob, gh utils.GithubClientProvider) (*s } case models.DiggerVCSGitlab: token = os.Getenv("DIGGER_GITLAB_ACCESS_TOKEN") + case models.DiggerVCSBitbucket: + // TODO: Refactor this peice into its own + if batch.VCSConnectionId == nil { + return nil, fmt.Errorf("connection ID not set, could not get vcs token") + } + log.Printf("vcs connection id %v", batch.VCSConnectionId) + connectionId := strconv.Itoa(int(*batch.VCSConnectionId)) + connectionEncrypted, err := models.DB.GetVCSConnectionById(connectionId) + if err != nil { + log.Printf("failed to fetch connection: %v", err) + return nil, fmt.Errorf("failed to fetch connection: %v", err) + } + + secret := os.Getenv("DIGGER_ENCRYPTION_SECRET") + if secret == "" { + log.Printf("ERROR: no encryption secret specified, please specify DIGGER_ENCRYPTION_SECRET as 32 bytes base64 string") + return nil, fmt.Errorf("ERROR: no encryption secret specified, please specify DIGGER_ENCRYPTION_SECRET as 32 bytes base64 string") + } + connectionDecrypted, err := utils.DecryptConnection(connectionEncrypted, []byte(secret)) + if err != nil { + log.Printf("ERROR: could not perform decryption: %v", err) + return nil, fmt.Errorf("ERROR: could not perform decryption: %v", err) + } + bitbucketAccessToken := connectionDecrypted.BitbucketAccessToken + token = bitbucketAccessToken default: return nil, fmt.Errorf("unknown batch VCS: %v", batch.VCS) } diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index f9c36bc4c..1f7ab8954 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -143,7 +143,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { for i, testParam := range testParameters { ciService := github2.MockCiService{} - batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false) + batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, nil) project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, nil, false, false) planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) diff --git a/backend/utils/graphs.go b/backend/utils/graphs.go index 227c31907..d30fddb35 100644 --- a/backend/utils/graphs.go +++ b/backend/utils/graphs.go @@ -14,7 +14,7 @@ import ( ) // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value -func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool) (*uuid.UUID, map[string]*models.DiggerJob, error) { +func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, VCSConnectionId *uint) (*uuid.UUID, map[string]*models.DiggerJob, error) { result := make(map[string]*models.DiggerJob) organisation, err := models.DB.GetOrganisationById(organisationId) if err != nil { @@ -43,7 +43,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig log.Printf("marshalledJobsMap: %v\n", marshalledJobsMap) - batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput) + batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, VCSConnectionId) if err != nil { return nil, nil, fmt.Errorf("failed to create batch: %v", err) } diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index bfbcf4d77..745726cc0 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -99,7 +99,7 @@ func RunSpec( reportError(spec, backendApi, message, err) } - reporter, err := reporterProvider.GetReporter(fmt.Sprintf("%v for %v", spec.Job.JobType, job.ProjectName), spec.Reporter, prService, *spec.Job.PullRequestNumber) + reporter, err := reporterProvider.GetReporter(fmt.Sprintf("%v for %v", spec.Job.JobType, job.ProjectName), spec.Reporter, prService, *spec.Job.PullRequestNumber, "") if err != nil { message := fmt.Sprintf("could not get reporter: %v", err) reportError(spec, backendApi, message, err) diff --git a/ee/backend/controllers/bitbucket.go b/ee/backend/controllers/bitbucket.go index 237fd536a..4c924385c 100644 --- a/ee/backend/controllers/bitbucket.go +++ b/ee/backend/controllers/bitbucket.go @@ -117,7 +117,7 @@ func (ee DiggerEEController) BitbucketWebhookHandler(c *gin.Context) { log.Printf("error parsing pullrequest:comment_created event: %v", err) log.Printf("error parsing pullrequest:comment_created event: %v", err) } - go handleIssueCommentEventBB(ee.BitbucketProvider, &pullRequestCommentCreated, ee.CiBackendProvider, orgId, bitbucketAccessToken) + go handleIssueCommentEventBB(ee.BitbucketProvider, &pullRequestCommentCreated, ee.CiBackendProvider, orgId, &connectionEncrypted.ID, bitbucketAccessToken) case "pullrequest:created": err := json.Unmarshal(bodyBytes, &pullRequestCreated) if err != nil { @@ -138,7 +138,7 @@ func (ee DiggerEEController) BitbucketWebhookHandler(c *gin.Context) { c.String(http.StatusAccepted, "ok") } -func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payload *BitbucketCommentCreatedEvent, ciBackendProvider ci_backends.CiBackendProvider, organisationId uint, bbAccessToken string) error { +func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payload *BitbucketCommentCreatedEvent, ciBackendProvider ci_backends.CiBackendProvider, organisationId uint, vcsConnectionId *uint, bbAccessToken string) error { repoFullName := payload.Repository.FullName repoOwner := payload.Repository.Owner.Username repoName := payload.Repository.Name @@ -283,7 +283,7 @@ func handleIssueCommentEventBB(bitbucketProvider utils.BitbucketProvider, payloa log.Printf("ParseInt err: %v", err) return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSBitbucket, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, 0, "", false, vcsConnectionId) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(bbService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index 54dd780c6..7f19ffedd 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -334,7 +334,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab log.Printf("strconv.ParseInt error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -525,7 +525,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla log.Printf("ParseInt err: %v", err) return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/hooks/github.go b/ee/backend/hooks/github.go index d6fb07dc7..bac444bdd 100644 --- a/ee/backend/hooks/github.go +++ b/ee/backend/hooks/github.go @@ -151,7 +151,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, nil) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/libs/ci/bitbucket/bitbucket.go b/libs/ci/bitbucket/bitbucket.go index 95d088a7d..0b5f9e067 100644 --- a/libs/ci/bitbucket/bitbucket.go +++ b/libs/ci/bitbucket/bitbucket.go @@ -4,15 +4,12 @@ import ( "bytes" "encoding/json" "fmt" - "io" + "github.com/diggerhq/digger/libs/ci" + configuration "github.com/diggerhq/digger/libs/digger_config" "io/ioutil" - "log" "net/http" "strconv" "time" - - "github.com/diggerhq/digger/libs/ci" - configuration "github.com/diggerhq/digger/libs/digger_config" ) // Define the base URL for the Bitbucket API. @@ -66,7 +63,6 @@ func (b BitbucketAPI) GetChangedFiles(prNumber int) ([]string, error) { } defer resp.Body.Close() - log.Printf("url %v", url) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to get changed files. Status code: %d", resp.StatusCode) } @@ -600,7 +596,6 @@ type PipelineResponse struct { func (b BitbucketAPI) TriggerPipeline(branch string, variables []interface{}) (string, error) { url := fmt.Sprintf("%s/repositories/%s/%s/pipelines", bitbucketBaseURL, b.RepoWorkspace, b.RepoName) - log.Printf("pipeline trigger url: %v branch %v", url, branch) triggerOptions := map[string]interface{}{ "target": map[string]interface{}{ "ref_type": "branch", @@ -626,8 +621,6 @@ func (b BitbucketAPI) TriggerPipeline(branch string, variables []interface{}) (s defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { - body, _ := io.ReadAll(resp.Body) - log.Printf("the response from bitbucket is: %v", string(body)) return "", fmt.Errorf("failed to trigger pipeline: %d", resp.StatusCode) } @@ -643,7 +636,6 @@ func (b BitbucketAPI) TriggerPipeline(branch string, variables []interface{}) (s return "", fmt.Errorf("error parsing response: %v", err) } - log.Printf(triggerPipelineResponse.Links.HTML.Href) return "", nil } diff --git a/libs/spec/providers.go b/libs/spec/providers.go index 0e39138c5..37c87a6b1 100644 --- a/libs/spec/providers.go +++ b/libs/spec/providers.go @@ -114,7 +114,7 @@ func (l LockProvider) GetLock(lockSpec LockSpec) (locking.Lock, error) { type ReporterProvider struct{} -func (r ReporterProvider) GetReporter(title string, reporterSpec ReporterSpec, ciService ci.PullRequestService, prNumber int) (reporting.Reporter, error) { +func (r ReporterProvider) GetReporter(title string, reporterSpec ReporterSpec, ciService ci.PullRequestService, prNumber int, vcs string) (reporting.Reporter, error) { getStrategy := func(strategy string) reporting.ReportStrategy { switch reporterSpec.ReportingStrategy { case "comments_per_run": @@ -131,6 +131,10 @@ func (r ReporterProvider) GetReporter(title string, reporterSpec ReporterSpec, c } } + isSupportMarkdown := true + if vcs == "bitbucket" { + isSupportMarkdown = false + } switch reporterSpec.ReporterType { case "noop": return reporting.NoopReporter{}, nil @@ -139,7 +143,7 @@ func (r ReporterProvider) GetReporter(title string, reporterSpec ReporterSpec, c return reporting.CiReporter{ CiService: ciService, PrNumber: prNumber, - IsSupportMarkdown: true, + IsSupportMarkdown: isSupportMarkdown, ReportStrategy: strategy, }, nil case "lazy": @@ -147,7 +151,7 @@ func (r ReporterProvider) GetReporter(title string, reporterSpec ReporterSpec, c ciReporter := reporting.CiReporter{ CiService: ciService, PrNumber: prNumber, - IsSupportMarkdown: true, + IsSupportMarkdown: isSupportMarkdown, ReportStrategy: strategy, } return reporting.NewCiReporterLazy(ciReporter), nil From 9906d0b9f2c26201c94ba00990e30af58d914f3a Mon Sep 17 00:00:00 2001 From: motatoes Date: Sun, 2 Mar 2025 13:34:34 -0800 Subject: [PATCH 6/6] set vcs type --- cli/pkg/spec/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 745726cc0..361fc6a5d 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -99,7 +99,7 @@ func RunSpec( reportError(spec, backendApi, message, err) } - reporter, err := reporterProvider.GetReporter(fmt.Sprintf("%v for %v", spec.Job.JobType, job.ProjectName), spec.Reporter, prService, *spec.Job.PullRequestNumber, "") + reporter, err := reporterProvider.GetReporter(fmt.Sprintf("%v for %v", spec.Job.JobType, job.ProjectName), spec.Reporter, prService, *spec.Job.PullRequestNumber, spec.VCS.VcsType) if err != nil { message := fmt.Sprintf("could not get reporter: %v", err) reportError(spec, backendApi, message, err)