diff --git a/pkg/converter/convert_unix.go b/pkg/converter/convert_unix.go index 4d0dda0d39..574f8b3f4b 100644 --- a/pkg/converter/convert_unix.go +++ b/pkg/converter/convert_unix.go @@ -315,7 +315,7 @@ func seekFile(ra content.ReaderAt, targetName string, handle func(io.Reader, *ta // provided by the caller. // // Important: the caller must check `io.WriteCloser.Close() == nil` to ensure -// the conversion workflow is finished. +// the conversion workflow is finished if `io.WriteCloser == nil`. func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser, error) { if opt.FsVersion == "" { opt.FsVersion = "6" @@ -348,14 +348,93 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser, return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6") } + if opt.FromDir != "" { + return nil, packFromDirectory(ctx, dest, opt, builderPath, opt.FromDir) + } + if opt.features.Contains(tool.FeatureTar2Rafs) { return packFromTar(ctx, dest, opt) } - return packFromDirectory(ctx, dest, opt, builderPath) + return packFromUnpackedTar(ctx, dest, opt, builderPath) +} + +func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath, sourceDir string) error { + if opt.ExternalBlobWriter == nil { + return fmt.Errorf("the 'ExternalBlobWriter' option requires the 'AttributesPath' option be specified") + } + + workDir, err := ensureWorkDir(opt.WorkDir) + if err != nil { + return errors.Wrap(err, "ensure work directory") + } + defer func() { + if err != nil { + os.RemoveAll(workDir) + } + }() + + blobPath := filepath.Join(workDir, "blob") + blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640) + if err != nil { + return errors.Wrapf(err, "create fifo file for blob") + } + defer blobFifo.Close() + + externalBlobPath := filepath.Join(workDir, "external-blob") + externalBlobFifo, err := fifo.OpenFifo(ctx, externalBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640) + if err != nil { + return errors.Wrapf(err, "create fifo file for external blob") + } + defer externalBlobFifo.Close() + + go func() { + err := tool.Pack(tool.PackOption{ + BuilderPath: builderPath, + + BlobPath: blobPath, + ExternalBlobPath: externalBlobPath, + FsVersion: opt.FsVersion, + SourcePath: sourceDir, + ChunkDictPath: opt.ChunkDictPath, + AttributesPath: opt.AttributesPath, + PrefetchPatterns: opt.PrefetchPatterns, + AlignedChunk: opt.AlignedChunk, + ChunkSize: opt.ChunkSize, + BatchSize: opt.BatchSize, + Compressor: opt.Compressor, + Timeout: opt.Timeout, + Encrypt: opt.Encrypt, + + Features: opt.features, + }) + if err != nil { + blobFifo.Close() + } + }() + + eg := errgroup.Group{} + eg.Go(func() error { + buffer := bufPool.Get().(*[]byte) + defer bufPool.Put(buffer) + if _, err := io.CopyBuffer(dest, blobFifo, *buffer); err != nil { + return errors.Wrap(err, "pack to nydus blob") + } + return nil + }) + eg.Go(func() error { + buffer := bufPool.Get().(*[]byte) + defer bufPool.Put(buffer) + if _, err := io.CopyBuffer(opt.ExternalBlobWriter, externalBlobFifo, *buffer); err != nil { + return errors.Wrap(err, "pack to nydus external blob") + } + return nil + }) + + return eg.Wait() } -func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) { +func packFromUnpackedTar(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) { workDir, err := ensureWorkDir(opt.WorkDir) if err != nil { return nil, errors.Wrap(err, "ensure work directory") diff --git a/pkg/converter/tool/builder.go b/pkg/converter/tool/builder.go index 7a789b1a3a..7f84567e95 100644 --- a/pkg/converter/tool/builder.go +++ b/pkg/converter/tool/builder.go @@ -29,11 +29,12 @@ func isSignalKilled(err error) bool { type PackOption struct { BuilderPath string - BootstrapPath string BlobPath string + ExternalBlobPath string FsVersion string SourcePath string ChunkDictPath string + AttributesPath string PrefetchPatterns string Compressor string OCIRef bool @@ -97,10 +98,21 @@ func buildPackArgs(option PackOption) []string { if option.Features.Contains(FeatureTar2Rafs) { args = append( args, - "--type", - "tar-rafs", "--blob-inline-meta", ) + if option.AttributesPath != "" { + args = append( + args, + "--type", + "dir-rafs", + ) + } else { + args = append( + args, + "--type", + "tar-rafs", + ) + } if option.FsVersion == "6" { args = append( args, @@ -140,6 +152,12 @@ func buildPackArgs(option PackOption) []string { if option.Encrypt { args = append(args, "--encrypt") } + if option.AttributesPath != "" { + args = append(args, "--attributes", option.AttributesPath) + } + if option.ExternalBlobPath != "" { + args = append(args, "--external-blob", option.ExternalBlobPath) + } args = append(args, option.SourcePath) return args diff --git a/pkg/converter/types.go b/pkg/converter/types.go index 9827d0a377..4654fd0bcd 100644 --- a/pkg/converter/types.go +++ b/pkg/converter/types.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "io" "strings" "time" @@ -83,6 +84,12 @@ type PackOption struct { Timeout *time.Duration // Whether the generated Nydus blobs should be encrypted. Encrypt bool + // Pack to nydus blob from a directory. + FromDir string + // Path to specified nydus attributes configuration file. + AttributesPath string + // Path to outputted nydus external blob file. + ExternalBlobWriter io.Writer // Features keeps a feature list supported by newer version of builder, // It is detected automatically, so don't export it.