Skip to content

Commit 17e2d8c

Browse files
motatoesmotatoes
authored
Add github connections to EE (#1803)
* supporting github connections in backend ee
1 parent f93c191 commit 17e2d8c

23 files changed

+862
-67
lines changed

‎backend/controllers/github.go

+25-29
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,11 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
5353
gh := d.GithubClientProvider
5454
log.Printf("GithubAppWebHook")
5555

56-
payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")))
56+
appID := c.GetHeader("X-GitHub-Hook-Installation-Target-ID")
57+
log.Printf("app id from header is: %v", appID)
58+
_, _, webhookSecret, _, err := d.GithubClientProvider.FetchCredentials(appID)
59+
60+
payload, err := github.ValidatePayload(c.Request, []byte(webhookSecret))
5761
if err != nil {
5862
log.Printf("Error validating github app webhook's payload: %v", err)
5963
c.String(http.StatusBadRequest, "Error validating github app webhook's payload")
@@ -70,12 +74,17 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
7074

7175
log.Printf("github event type: %v\n", reflect.TypeOf(event))
7276

77+
appId64, err := strconv.ParseInt(appID, 10, 64)
78+
if err != nil {
79+
log.Printf("Error converting appId string to int64: %v", err)
80+
return
81+
}
82+
7383
switch event := event.(type) {
7484
case *github.InstallationEvent:
7585
log.Printf("InstallationEvent, action: %v\n", *event.Action)
76-
7786
if *event.Action == "deleted" {
78-
err := handleInstallationDeletedEvent(event)
87+
err := handleInstallationDeletedEvent(event, appId64)
7988
if err != nil {
8089
c.String(http.StatusAccepted, "Failed to handle webhook event.")
8190
return
@@ -87,7 +96,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
8796
c.String(http.StatusOK, "OK")
8897
return
8998
}
90-
err := handleIssueCommentEvent(gh, event, d.CiBackendProvider)
99+
err = handleIssueCommentEvent(gh, event, d.CiBackendProvider, appId64)
91100
if err != nil {
92101
log.Printf("handleIssueCommentEvent error: %v", err)
93102
c.String(http.StatusAccepted, err.Error())
@@ -105,7 +114,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
105114
}
106115
case *github.PullRequestEvent:
107116
log.Printf("Got pull request event for %d", *event.PullRequest.ID)
108-
err := handlePullRequestEvent(gh, event, d.CiBackendProvider)
117+
err = handlePullRequestEvent(gh, event, d.CiBackendProvider, appId64)
109118
if err != nil {
110119
log.Printf("handlePullRequestEvent error: %v", err)
111120
c.String(http.StatusAccepted, err.Error())
@@ -231,11 +240,6 @@ func (d DiggerController) GithubSetupExchangeCode(c *gin.Context) {
231240
}
232241
log.Printf("Found credentials for GitHub app %v with id %d", *cfg.Name, cfg.GetID())
233242

234-
_, err = models.DB.CreateGithubApp(cfg.GetName(), cfg.GetID(), cfg.GetHTMLURL())
235-
if err != nil {
236-
c.Error(fmt.Errorf("Failed to create github app record on callback"))
237-
}
238-
239243
PEM := cfg.GetPEM()
240244
PemBase64 := base64.StdEncoding.EncodeToString([]byte(PEM))
241245
c.HTML(http.StatusOK, "github_setup.tmpl", gin.H{
@@ -299,10 +303,8 @@ generate_projects:
299303
return repo, org, nil
300304
}
301305

302-
func handleInstallationDeletedEvent(installation *github.InstallationEvent) error {
306+
func handleInstallationDeletedEvent(installation *github.InstallationEvent, appId int64) error {
303307
installationId := *installation.Installation.ID
304-
appId := *installation.Installation.AppID
305-
306308
link, err := models.DB.GetGithubInstallationLinkForInstallationId(installationId)
307309
if err != nil {
308310
return err
@@ -323,13 +325,7 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent) erro
323325
return nil
324326
}
325327

326-
func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
327-
appId, err := strconv.ParseInt(os.Getenv("GITHUB_APP_ID"), 10, 64)
328-
if err != nil {
329-
log.Printf("error getting github app installation id: %v", err)
330-
return fmt.Errorf("error getting github app installation id")
331-
}
332-
328+
func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
333329
installationId := *payload.Installation.ID
334330
repoName := *payload.Repo.Name
335331
repoOwner := *payload.Repo.Owner.Login
@@ -645,13 +641,7 @@ func getBatchType(jobs []orchestrator_scheduler.Job) orchestrator_scheduler.Digg
645641
}
646642
}
647643

648-
func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
649-
appId, err := strconv.ParseInt(os.Getenv("GITHUB_APP_ID"), 10, 64)
650-
if err != nil {
651-
log.Printf("error getting github app installation id: %v", err)
652-
return fmt.Errorf("error getting github app installation id")
653-
}
654-
644+
func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
655645
installationId := *payload.Installation.ID
656646
repoName := *payload.Repo.Name
657647
repoOwner := *payload.Repo.Owner.Login
@@ -1028,8 +1018,14 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
10281018
installationId := c.Request.URL.Query()["installation_id"][0]
10291019
//setupAction := c.Request.URL.Query()["setup_action"][0]
10301020
code := c.Request.URL.Query()["code"][0]
1031-
clientId := os.Getenv("GITHUB_APP_CLIENT_ID")
1032-
clientSecret := os.Getenv("GITHUB_APP_CLIENT_SECRET")
1021+
appId := c.Request.URL.Query().Get("state")
1022+
1023+
clientId, clientSecret, _, _, err := d.GithubClientProvider.FetchCredentials(appId)
1024+
if err != nil {
1025+
log.Printf("could not fetch credentials for the app: %v", err)
1026+
c.String(500, "could not find credentials for github app")
1027+
return
1028+
}
10331029

10341030
installationId64, err := strconv.ParseInt(installationId, 10, 64)
10351031
if err != nil {

‎backend/controllers/github_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
589589

590590
// migrate tables
591591
err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
592-
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubApp{}, &models.GithubAppInstallationLink{},
592+
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{},
593593
&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.JobToken{})
594594
if err != nil {
595595
log.Fatal(err)
@@ -706,7 +706,7 @@ func TestGithubHandleIssueCommentEvent(t *testing.T) {
706706
var payload github.IssueCommentEvent
707707
err := json.Unmarshal([]byte(issueCommentPayload), &payload)
708708
assert.NoError(t, err)
709-
err = handleIssueCommentEvent(gh, &payload, nil)
709+
err = handleIssueCommentEvent(gh, &payload, nil, 0)
710710
assert.NoError(t, err)
711711

712712
jobs, err := models.DB.GetPendingParentDiggerJobs(nil)

‎backend/go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,8 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7l
299299
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
300300
github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
301301
github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
302+
github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
303+
github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
302304
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
303305
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
304306
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

‎backend/migrations/20241107162605.sql

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- Modify "github_apps" table
2+
ALTER TABLE "public"."github_apps" ADD COLUMN "client_id" text NULL, ADD COLUMN "client_secret_encrypted" text NULL, ADD COLUMN "webhook_secret_encrypted" text NULL, ADD COLUMN "private_key_encrypted" text NULL, ADD COLUMN "private_key_base64_encrypted" text NULL, ADD COLUMN "org" text NULL;

‎backend/migrations/20241107163722.sql

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
-- Create "github_app_connections" table
2+
CREATE TABLE "public"."github_app_connections" (
3+
"id" bigserial NOT NULL,
4+
"created_at" timestamptz NULL,
5+
"updated_at" timestamptz NULL,
6+
"deleted_at" timestamptz NULL,
7+
"github_id" bigint NULL,
8+
"client_id" text NULL,
9+
"client_secret_encrypted" text NULL,
10+
"webhook_secret_encrypted" text NULL,
11+
"private_key_encrypted" text NULL,
12+
"private_key_base64_encrypted" text NULL,
13+
"org" text NULL,
14+
"name" text NULL,
15+
"github_app_url" text NULL,
16+
PRIMARY KEY ("id")
17+
);
18+
-- Create index "idx_github_app_connections_deleted_at" to table: "github_app_connections"
19+
CREATE INDEX "idx_github_app_connections_deleted_at" ON "public"."github_app_connections" ("deleted_at");
20+
-- Drop "github_apps" table
21+
DROP TABLE "public"."github_apps";

‎backend/migrations/20241107172343.sql

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
-- Modify "github_app_connections" table
2+
ALTER TABLE "public"."github_app_connections" ADD COLUMN "organisation_id" bigint NULL, ADD
3+
CONSTRAINT "fk_github_app_connections_organisation" FOREIGN KEY ("organisation_id") REFERENCES "public"."organisations" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION;

‎backend/migrations/atlas.sum

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68=
1+
h1:LczvJY4QB28QrHCgIm+p3umL5Pkm1NF+iBM0pG6Ij4E=
22
20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
33
20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
44
20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -30,3 +30,6 @@ h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68=
3030
20240729155442.sql h1:s7PCALP3SgPz5y9Ya7HkDzYjeN86Q5NniW2YIkOMBrQ=
3131
20240729155926.sql h1:8vsDrpy/R1UDI+meIp6KoDfhS60t+ngu8aPB+uonFZ4=
3232
20240729160028.sql h1:snkkxhA2aEQhqBmIhN8l+nPlBhrPOZiPP+dnyhobwD8=
33+
20241107162605.sql h1:UwkkGz6bQ7nqSCCA9lVS+JS01YDONLVV0yaM8jJqzlQ=
34+
20241107163722.sql h1:tk28AgXggvpEigTkWMYMxIVDPNdEUFGijaFWBqvlZhA=
35+
20241107172343.sql h1:wtM1+uJZY6NiiDYabuzj/LAANAV7+xyUCL5U23v3e+c=

‎backend/models/github.go

+15-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
11
package models
22

3-
import "gorm.io/gorm"
3+
import (
4+
"gorm.io/gorm"
5+
)
46

5-
type GithubApp struct {
7+
type GithubAppConnection struct {
68
gorm.Model
7-
GithubId int64
8-
Name string
9-
GithubAppUrl string
9+
GithubId int64 // app id
10+
ClientID string
11+
ClientSecretEncrypted string
12+
WebhookSecretEncrypted string
13+
PrivateKeyEncrypted string
14+
PrivateKeyBase64Encrypted string
15+
Org string
16+
Name string
17+
GithubAppUrl string
18+
OrganisationID uint
19+
Organisation Organisation
1020
}
1121

1222
type GithubAppInstallStatus int

‎backend/models/scheduler_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) {
3838

3939
// migrate tables
4040
err = gdb.AutoMigrate(&Policy{}, &Organisation{}, &Repo{}, &Project{}, &Token{},
41-
&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubApp{}, &GithubAppInstallationLink{},
41+
&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubAppConnection{}, &GithubAppInstallationLink{},
4242
&GithubDiggerJobLink{}, &DiggerJob{}, &DiggerJobParentLink{})
4343
if err != nil {
4444
log.Fatal(err)

‎backend/models/storage.go

+25-4
Original file line numberDiff line numberDiff line change
@@ -436,8 +436,19 @@ func (db *Database) GetGithubAppInstallationLink(installationId int64) (*GithubA
436436
return &link, nil
437437
}
438438

439-
func (db *Database) CreateGithubApp(name string, githubId int64, url string) (*GithubApp, error) {
440-
app := GithubApp{Name: name, GithubId: githubId, GithubAppUrl: url}
439+
func (db *Database) CreateGithubAppConnection(name string, githubId int64, ClientID string, ClientSecretEncrypted string, WebhookSecretEncrypted string, PrivateKeyEncrypted string, PrivateKeyBase64Encrypted string, Org string, url string, orgId uint) (*GithubAppConnection, error) {
440+
app := GithubAppConnection{
441+
Name: name,
442+
GithubId: githubId,
443+
ClientID: ClientID,
444+
ClientSecretEncrypted: ClientSecretEncrypted,
445+
WebhookSecretEncrypted: WebhookSecretEncrypted,
446+
PrivateKeyEncrypted: PrivateKeyEncrypted,
447+
PrivateKeyBase64Encrypted: PrivateKeyBase64Encrypted,
448+
Org: Org,
449+
GithubAppUrl: url,
450+
OrganisationID: orgId,
451+
}
441452
result := db.GormDB.Save(&app)
442453
if result.Error != nil {
443454
return nil, result.Error
@@ -446,9 +457,19 @@ func (db *Database) CreateGithubApp(name string, githubId int64, url string) (*G
446457
return &app, nil
447458
}
448459

460+
func (db *Database) GetGithubAppConnectionById(id string) (*GithubAppConnection, error) {
461+
app := GithubAppConnection{}
462+
result := db.GormDB.Where("id = ?", id).Find(&app)
463+
if result.Error != nil {
464+
log.Printf("Failed to find GitHub App for id: %v, error: %v\n", id, result.Error)
465+
return nil, result.Error
466+
}
467+
return &app, nil
468+
}
469+
449470
// GetGithubApp return GithubApp by Id
450-
func (db *Database) GetGithubApp(gitHubAppId any) (*GithubApp, error) {
451-
app := GithubApp{}
471+
func (db *Database) GetGithubAppConnection(gitHubAppId any) (*GithubAppConnection, error) {
472+
app := GithubAppConnection{}
452473
result := db.GormDB.Where("github_id = ?", gitHubAppId).Find(&app)
453474
if result.Error != nil {
454475
log.Printf("Failed to find GitHub App for id: %v, error: %v\n", gitHubAppId, result.Error)

‎backend/models/storage_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *Database, *Organisation) {
3636

3737
// migrate tables
3838
err = gdb.AutoMigrate(&Policy{}, &Organisation{}, &Repo{}, &Project{}, &Token{},
39-
&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubApp{}, &GithubAppInstallationLink{},
39+
&User{}, &ProjectRun{}, &GithubAppInstallation{}, &GithubAppConnection{}, &GithubAppInstallationLink{},
4040
&GithubDiggerJobLink{}, &DiggerJob{}, &DiggerJobParentLink{})
4141
if err != nil {
4242
log.Fatal(err)

‎backend/tasks/runs_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) {
5050

5151
// migrate tables
5252
err = gdb.AutoMigrate(&models.Policy{}, &models.Organisation{}, &models.Repo{}, &models.Project{}, &models.Token{},
53-
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubApp{}, &models.GithubAppInstallationLink{},
53+
&models.User{}, &models.ProjectRun{}, &models.GithubAppInstallation{}, &models.GithubAppConnection{}, &models.GithubAppInstallationLink{},
5454
&models.GithubDiggerJobLink{}, &models.DiggerJob{}, &models.DiggerJobParentLink{}, &models.DiggerRun{}, &models.DiggerRunQueueItem{})
5555
if err != nil {
5656
log.Fatal(err)

0 commit comments

Comments
 (0)