Skip to content

Commit 1c83398

Browse files
committed
cli: download and chdir to artefact
1 parent d77cbc8 commit 1c83398

File tree

7 files changed

+231
-5
lines changed

7 files changed

+231
-5
lines changed

cli/pkg/spec/spec.go

+25-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"github.com/diggerhq/digger/cli/pkg/digger"
66
"github.com/diggerhq/digger/cli/pkg/usage"
7+
"github.com/diggerhq/digger/cli/pkg/utils"
78
"github.com/diggerhq/digger/libs/ci"
89
"github.com/diggerhq/digger/libs/comment_utils/reporting"
910
comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary"
@@ -14,6 +15,8 @@ import (
1415
"github.com/samber/lo"
1516
"log"
1617
"os"
18+
"os/exec"
19+
"strings"
1720
"time"
1821
)
1922

@@ -169,6 +172,27 @@ func RunSpecManualCommand(
169172
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1)
170173
}
171174

175+
// download zip artefact, git init and prepare for job execution
176+
tempDir, err := os.MkdirTemp("", "downloaded-zip-")
177+
if err != nil {
178+
log.Printf("failed to create temp dir: %w", err)
179+
os.Exit(1)
180+
}
181+
182+
// downloading artefact zip , extracting, git init and then chdir to that directory for job execution
183+
absoluteFileName, err := backendApi.DownloadJobArtefact(tempDir)
184+
if err != nil {
185+
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not download job artefact: %v", err), 1)
186+
}
187+
zipPath := *absoluteFileName
188+
utils.ExtractZip(zipPath, tempDir)
189+
// Transforming: /var/temp/xxx/yyy/blabla.zip -> /var/temp/xxx/yyy/blabla
190+
gitLocation := strings.TrimSuffix(zipPath, ".zip")
191+
os.Chdir(gitLocation)
192+
cmd := exec.Command("git", "init")
193+
cmd.Stdout = os.Stdout
194+
cmd.Stdout = os.Stderr
195+
172196
policyChecker, err := policyProvider.GetPolicyProvider(spec.Policy, spec.Backend.BackendHostname, spec.Backend.BackendOrganisationName, spec.Backend.BackendJobToken)
173197
if err != nil {
174198
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get policy provider: %v", err), 1)
@@ -199,7 +223,7 @@ func RunSpecManualCommand(
199223
}
200224

201225
commentUpdater := comment_summary.NoopCommentUpdater{}
202-
// TODO: do not require conversion to gh service
226+
// do not change these placeholders as they are parsed by dgctl to stream logs
203227
log.Printf("<========= DIGGER RUNNING IN MANUAL MODE =========>")
204228
allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, false, false, commentId, currentDir)
205229
log.Printf("<========= DIGGER COMPLETED =========>")

cli/pkg/utils/io.go

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package utils
2+
3+
import (
4+
"archive/zip"
5+
"fmt"
6+
"io"
7+
"os"
8+
"path/filepath"
9+
)
10+
11+
func ExtractZip(zipFilePath string, outDir string) error {
12+
13+
// Open the zip file
14+
zipReader, err := zip.OpenReader(zipFilePath)
15+
if err != nil {
16+
return fmt.Errorf("failed to open zip: %w", err)
17+
}
18+
defer zipReader.Close()
19+
20+
for _, file := range zipReader.File {
21+
path := filepath.Join(outDir, file.Name)
22+
23+
if file.FileInfo().IsDir() {
24+
os.MkdirAll(path, os.ModePerm)
25+
continue
26+
}
27+
28+
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
29+
return fmt.Errorf("failed to create directory: %w", err)
30+
}
31+
32+
dstFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
33+
if err != nil {
34+
return fmt.Errorf("failed to create file: %w", err)
35+
}
36+
37+
srcFile, err := file.Open()
38+
if err != nil {
39+
dstFile.Close()
40+
return fmt.Errorf("failed to open zip file: %w", err)
41+
}
42+
43+
_, err = io.Copy(dstFile, srcFile)
44+
srcFile.Close()
45+
dstFile.Close()
46+
47+
if err != nil {
48+
return fmt.Errorf("failed to extract file: %w", err)
49+
}
50+
}
51+
return nil
52+
}

dgctl/cmd/exec.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,8 @@ var execCmd = &cobra.Command{
278278
// attach zip archive to backend
279279
backendToken := spec.Job.BackendJobToken
280280
zipLocation, err := utils.ArchiveGitRepo("./")
281-
statusCode, respBody, err := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken}.UploadJobArtefact(zipLocation)
282-
281+
backendApi := backendapi.DiggerApi{DiggerHost: diggerHostname, AuthToken: backendToken}
282+
statusCode, respBody, err := backendApi.UploadJobArtefact(zipLocation)
283283
if err != nil {
284284
log.Printf("could not attach zip artefact: %v", err)
285285
os.Exit(1)
@@ -289,7 +289,7 @@ var execCmd = &cobra.Command{
289289
log.Printf("server response: %v", *respBody)
290290
os.Exit(1)
291291
}
292-
292+
293293
token := os.Getenv("GITHUB_PAT_TOKEN")
294294
if token == "" {
295295
log.Printf("missing variable: GITHUB_PAT_TOKEN")

ee/backend/controllers/artefacts.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ func SetJobArtefact(c *gin.Context) {
5757
// Create a new File record
5858
artefactRecord := models.JobArtefact{
5959
JobTokenID: jobToken.ID,
60-
Filename: uuid.NewString(),
60+
Filename: uuid.NewString() + ".zip",
6161
Contents: content,
6262
Size: file.Size,
6363
ContentType: file.Header.Get("Content-Type"),

libs/backendapi/backend.go

+2
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,6 @@ type Api interface {
1010
ReportProject(repo string, projectName string, configuration string) error
1111
ReportProjectRun(repo string, projectName string, startedAt time.Time, endedAt time.Time, status string, command string, output string) error
1212
ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error)
13+
UploadJobArtefact(zipLocation string) (*int, *string, error)
14+
DownloadJobArtefact(downloadTo string) (*string, error)
1315
}

libs/backendapi/diggerapi.go

+140
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@ import (
99
"github.com/diggerhq/digger/libs/terraform_utils"
1010
"io"
1111
"log"
12+
"mime"
13+
"mime/multipart"
1214
"net/http"
1315
"net/url"
1416
"os"
17+
"path"
1518
"path/filepath"
1619
"time"
1720
)
@@ -31,6 +34,14 @@ func (n NoopApi) ReportProjectJobStatus(repo string, projectName string, jobId s
3134
return nil, nil
3235
}
3336

37+
func (n NoopApi) UploadJobArtefact(zipLocation string) (*int, *string, error) {
38+
return nil, nil, nil
39+
}
40+
41+
func (n NoopApi) DownloadJobArtefact(downloadTo string) (*string, error) {
42+
return nil, nil
43+
}
44+
3445
type DiggerApi struct {
3546
DiggerHost string
3647
AuthToken string
@@ -187,6 +198,135 @@ func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId
187198
return &response, nil
188199
}
189200

201+
func (d DiggerApi) UploadJobArtefact(zipLocation string) (*int, *string, error) {
202+
u, err := url.Parse(d.DiggerHost)
203+
if err != nil {
204+
return nil, nil, err
205+
}
206+
u.Path = path.Join(u.Path, "job_artefacts")
207+
url := u.String()
208+
filePath := zipLocation
209+
210+
// Open the file
211+
file, err := os.Open(filePath)
212+
if err != nil {
213+
fmt.Println("Error opening file:", err)
214+
return nil, nil, fmt.Errorf("Error opening file:", err)
215+
}
216+
defer file.Close()
217+
218+
// Create a buffer to store our request body as bytes
219+
var requestBody bytes.Buffer
220+
221+
// Create a multipart writer
222+
multipartWriter := multipart.NewWriter(&requestBody)
223+
224+
// Create a form file writer for our file field
225+
fileWriter, err := multipartWriter.CreateFormFile("file", filepath.Base(filePath))
226+
if err != nil {
227+
fmt.Println("Error creating form file:", err)
228+
return nil, nil, fmt.Errorf("Error creating form file:", err)
229+
}
230+
231+
// Copy the file content to the form file writer
232+
_, err = io.Copy(fileWriter, file)
233+
if err != nil {
234+
fmt.Println("Error copying file content:", err)
235+
return nil, nil, fmt.Errorf("Error copying file content:", err)
236+
}
237+
238+
// Close the multipart writer to finalize the request body
239+
multipartWriter.Close()
240+
241+
// Create a new HTTP request
242+
req, err := http.NewRequest("PUT", url, &requestBody)
243+
if err != nil {
244+
fmt.Println("Error creating request:", err)
245+
return nil, nil, fmt.Errorf("Error creating request:", err)
246+
}
247+
248+
// Set the content type header
249+
req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
250+
req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", d.AuthToken))
251+
252+
// Send the request
253+
client := &http.Client{}
254+
resp, err := client.Do(req)
255+
if err != nil {
256+
fmt.Println("Error sending request:", err)
257+
return nil, nil, fmt.Errorf("Error sending request:", err)
258+
}
259+
defer resp.Body.Close()
260+
261+
// Read and print the response
262+
body, err := io.ReadAll(resp.Body)
263+
if err != nil {
264+
fmt.Println("Error reading response:", err)
265+
return nil, nil, fmt.Errorf("Error reading response: %v", err)
266+
}
267+
268+
b := string(body)
269+
return &resp.StatusCode, &b, nil
270+
}
271+
272+
func getFilename(resp *http.Response) string {
273+
// Check the Content-Disposition header
274+
if cd := resp.Header.Get("Content-Disposition"); cd != "" {
275+
if _, params, err := mime.ParseMediaType(cd); err == nil {
276+
if filename, ok := params["filename"]; ok {
277+
return filename
278+
}
279+
}
280+
}
281+
// Fallback to the last part of the URL path
282+
return path.Base(resp.Request.URL.Path)
283+
}
284+
285+
func (d DiggerApi) DownloadJobArtefact(downloadTo string) (*string, error) {
286+
// Download the zip file
287+
url, err := url.JoinPath(d.DiggerHost, "job_artefacts")
288+
if err != nil {
289+
log.Printf("failed to create url: %v", err)
290+
return nil, err
291+
}
292+
293+
// Create a new HTTP request
294+
req, err := http.NewRequest("GET", url, nil)
295+
if err != nil {
296+
fmt.Println("Error creating request:", err)
297+
return nil, fmt.Errorf("Error creating request:", err)
298+
}
299+
300+
// Set the content type header
301+
req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", d.AuthToken))
302+
303+
// Send the request
304+
client := &http.Client{}
305+
resp, err := client.Do(req)
306+
if err != nil {
307+
return nil, fmt.Errorf("failed to download zip: %w", err)
308+
}
309+
defer resp.Body.Close()
310+
311+
// Create a temporary file to store the zip
312+
tempZipFile, err := os.Create(path.Join(downloadTo, getFilename(resp)))
313+
if err != nil {
314+
return nil, fmt.Errorf("failed to create zip file: %w", err)
315+
}
316+
defer tempZipFile.Close()
317+
318+
// Copy the downloaded content to the temporary file
319+
_, err = io.Copy(tempZipFile, resp.Body)
320+
if err != nil {
321+
return nil, fmt.Errorf("failed to save zip content: %w", err)
322+
}
323+
324+
// note that fileName include absolute path to the zip file
325+
fileName := tempZipFile.Name()
326+
return &fileName, nil
327+
328+
}
329+
190330
func NewBackendApi(hostName string, authToken string) Api {
191331
var backendApi Api
192332
if os.Getenv("NO_BACKEND") == "true" {

libs/backendapi/mocks.go

+8
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,11 @@ func (t MockBackendApi) ReportProjectRun(repo string, projectName string, starte
2020
func (t MockBackendApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *execution.DiggerExecutorPlanResult, PrCommentUrl string, terraformOutput string) (*scheduler.SerializedBatch, error) {
2121
return nil, nil
2222
}
23+
24+
func (t MockBackendApi) UploadJobArtefact(zipLocation string) (*int, *string, error) {
25+
return nil, nil, nil
26+
}
27+
28+
func (t MockBackendApi) DownloadJobArtefact(downloadTo string) (*string, error) {
29+
return nil, nil
30+
}

0 commit comments

Comments
 (0)