commit after sealed & refactored command execution

Signed-off-by: Haoqi Miao <miaohaoqi.mhq@alibaba-inc.com>

Author: HileQAQ

.github/workflows/ci.yml

@@ -148,8 +148,8 @@ jobs:
           echo "[ init env ]"
           sudo apt update && sudo apt install -y containerd.io libnl-3-200 libnl-genl-3-200
           sudo modprobe -a target_core_user tcm_loop
-          wget https://github.com/containerd/overlaybd/releases/download/v0.6.1/overlaybd-0.6.1-1.x86_64.deb &&
-          sudo dpkg -i overlaybd-0.6.1-1.x86_64.deb
+          wget https://github.com/containerd/overlaybd/releases/download/v0.6.4/overlaybd-0.6.4-0ubuntu1.22.04.x86_64.deb &&
+          sudo dpkg -i overlaybd-0.6.4-0ubuntu1.22.04.x86_64.deb
           sudo systemctl enable /opt/overlaybd/overlaybd-tcmu.service
           sudo systemctl start overlaybd-tcmu
           sudo systemctl status overlaybd-tcmu

cmd/convertor/builder/fastoci_builder.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/accelerated-container-image/pkg/snapshot"
+	"github.com/containerd/accelerated-container-image/pkg/utils"
 	"github.com/containerd/containerd/archive/compression"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
@@ -225,17 +226,7 @@ func (e *fastOCIBuilderEngine) createIdentifier(idx int) error {
 }
 
 func (e *fastOCIBuilderEngine) create(ctx context.Context, dir string) error {
-	binpath := filepath.Join("/opt/overlaybd/bin", "overlaybd-create")
-	dataPath := path.Join(dir, "writable_data")
-	indexPath := path.Join(dir, "writable_index")
-	os.RemoveAll(dataPath)
-	os.RemoveAll(indexPath)
-	out, err := exec.CommandContext(ctx, binpath, "-s",
-		dataPath, indexPath, "64", "--fastoci").CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "failed to overlaybd-create: %s", out)
-	}
-	return nil
+	return utils.Create(ctx, dir, "-s", "64", "--fastoci")
 }
 
 func (e *fastOCIBuilderEngine) apply(ctx context.Context, dir string) error {
@@ -253,16 +244,5 @@ func (e *fastOCIBuilderEngine) apply(ctx context.Context, dir string) error {
 }
 
 func (e *fastOCIBuilderEngine) commit(ctx context.Context, dir string) error {
-	binpath := filepath.Join("/opt/overlaybd/bin", "overlaybd-commit")
-
-	out, err := exec.CommandContext(ctx, binpath, "-z",
-		path.Join(dir, "writable_data"),
-		path.Join(dir, "writable_index"),
-		path.Join(dir, fsMetaFile),
-		"--fastoci",
-	).CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "failed to overlaybd-commit: %s", out)
-	}
-	return nil
+	return utils.Commit(ctx, dir, dir, false, "-z", "--fastoci")
 }

cmd/convertor/builder/overlaybd_builder.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/accelerated-container-image/pkg/snapshot"
+	"github.com/containerd/accelerated-container-image/pkg/utils"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
@@ -287,17 +288,7 @@ func (e *overlaybdBuilderEngine) getLayerDir(idx int) string {
 }
 
 func (e *overlaybdBuilderEngine) create(ctx context.Context, dir string) error {
-	binpath := filepath.Join("/opt/overlaybd/bin", "overlaybd-create")
-	dataPath := path.Join(dir, "writable_data")
-	indexPath := path.Join(dir, "writable_index")
-	os.RemoveAll(dataPath)
-	os.RemoveAll(indexPath)
-	out, err := exec.CommandContext(ctx, binpath, "-s",
-		dataPath, indexPath, "64").CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "failed to overlaybd-create: %s", out)
-	}
-	return nil
+	return utils.Create(ctx, dir, "-s", "64")
 }
 
 func (e *overlaybdBuilderEngine) apply(ctx context.Context, dir string) error {
@@ -314,7 +305,6 @@ func (e *overlaybdBuilderEngine) apply(ctx context.Context, dir string) error {
 }
 
 func (e *overlaybdBuilderEngine) commit(ctx context.Context, dir string, idx int) error {
-	binpath := filepath.Join("/opt/overlaybd/bin", "overlaybd-commit")
 	var parentUUID string
 	if idx > 0 {
 		parentUUID = chainIDtoUUID(e.overlaybdLayers[idx-1].chainID)
@@ -323,20 +313,8 @@ func (e *overlaybdBuilderEngine) commit(ctx context.Context, dir string, idx int
 	}
 	curUUID := chainIDtoUUID(e.overlaybdLayers[idx].chainID)
 
-	opts := []string{
-		"-z", "-t",
-		path.Join(dir, "writable_data"),
-		path.Join(dir, "writable_index"),
-		path.Join(dir, commitFile),
-		"--uuid", curUUID,
-	}
-	if parentUUID != "" {
-		opts = append(opts, "--parent-uuid", parentUUID)
-	}
-
-	out, err := exec.CommandContext(ctx, binpath, opts...).CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "failed to overlaybd-commit: %s", out)
+	if err := utils.Commit(ctx, dir, dir, false, "-z", "-t", "--uuid", curUUID); err != nil {
+		return err
 	}
 	logrus.Infof("layer %d committed, uuid: %s, parent uuid: %s", idx, curUUID, parentUUID)
 	return nil

docs/QUICKSTART.md

@@ -37,7 +37,7 @@ Users can compile the latest code to install or download the [release](https://g
 #### Compile from source
 
 Install dependencies:
-- golang 1.16+
+- golang 1.19+
 
 Run the following commands to build:
 ```bash

pkg/convertor/convertor.go

@@ -17,7 +17,6 @@
 package convertor
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
 	"context"
@@ -28,10 +27,13 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
+	"github.com/containerd/accelerated-container-image/pkg/utils"
+	"github.com/sirupsen/logrus"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/archive"
@@ -95,8 +97,8 @@ func NewContentLoaderWithFsType(isAccelLayer bool, fsType string, files ...Conte
 }
 
 type ContentFile struct {
-	SrcFilePath string
-	DstFileName string
+	SrcFilePath string // .../{ID}/fs/overlaybd.sealed
+	DstFileName string // overlaybd.commit
 }
 
 type contentLoader struct {
@@ -116,7 +118,6 @@ func (loader *contentLoader) Load(ctx context.Context, cs content.Store) (l Laye
 	srcPathList := make([]string, 0)
 	digester := digest.Canonical.Digester()
 	countWriter := &writeCountWrapper{w: io.MultiWriter(contentWriter, digester.Hash())}
-	tarWriter := tar.NewWriter(countWriter)
 
 	openedSrcFile := make([]*os.File, 0)
 	defer func() {
@@ -126,36 +127,26 @@ func (loader *contentLoader) Load(ctx context.Context, cs content.Store) (l Laye
 	}()
 
 	for _, loader := range loader.files {
-		srcPathList = append(srcPathList, loader.SrcFilePath)
-		srcFile, err := os.Open(loader.SrcFilePath)
-		if err != nil {
-			return emptyLayer, errors.Wrapf(err, "failed to open src file of %s", loader.SrcFilePath)
-		}
-		openedSrcFile = append(openedSrcFile, srcFile)
+		commitPath := filepath.Dir(loader.SrcFilePath)
+		commitFile := filepath.Join(commitPath, "overlaybd.commit")
+		srcPathList = append(srcPathList, commitFile)
 
-		fi, err := os.Stat(loader.SrcFilePath)
+		err := utils.Commit(ctx, commitPath, commitPath, true, "-z", "-t")
 		if err != nil {
-			return emptyLayer, errors.Wrapf(err, "failed to get info of %s", loader.SrcFilePath)
+			return emptyLayer, errors.Wrapf(err, "failed to overlaybd-commit for sealed file")
 		}
 
-		if err := tarWriter.WriteHeader(&tar.Header{
-			Name:     loader.DstFileName,
-			Mode:     0444,
-			Size:     fi.Size(),
-			Typeflag: tar.TypeReg,
-		}); err != nil {
-			return emptyLayer, errors.Wrapf(err, "failed to write tar header")
+		srcFile, err := os.Open(commitFile)
+		if err != nil {
+			return emptyLayer, errors.Wrapf(err, "failed to open src file of %s", loader.SrcFilePath)
 		}
-
-		if _, err := io.Copy(tarWriter, bufio.NewReader(srcFile)); err != nil {
-			return emptyLayer, errors.Wrapf(err, "failed to copy IO")
+		openedSrcFile = append(openedSrcFile, srcFile)
+		_, err = io.Copy(countWriter, bufio.NewReader(srcFile))
+		if err != nil {
+			logrus.Errorf("failed to do io.Copy(), error: %v", err)
+			return emptyLayer, err
 		}
 	}
-
-	if err = tarWriter.Close(); err != nil {
-		return emptyLayer, errors.Wrapf(err, "failed to close tar file")
-	}
-
 	labels := map[string]string{
 		labelBuildLayerFrom: strings.Join(srcPathList, ","),
 	}
@@ -516,7 +507,7 @@ func (c *overlaybdConvertor) convertLayers(ctx context.Context, srcDescs []ocisp
 		opts = append(opts, snapshots.WithLabels(map[string]string{
 			label.ZFileConfig: string(cfgStr),
 		}))
-		lastParentID, err = c.applyOCIV1LayerInZfile(ctx, lastParentID, desc, opts, nil)
+		lastParentID, err = c.applyOCIV1LayerInObd(ctx, lastParentID, desc, opts, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -548,8 +539,8 @@ func (c *overlaybdConvertor) convertLayers(ctx context.Context, srcDescs []ocisp
 	return commitLayers, nil
 }
 
-// applyOCIV1LayerInZfile applys the OCIv1 tarfile in zfile format and commit it.
-func (c *overlaybdConvertor) applyOCIV1LayerInZfile(
+// applyOCIV1LayerInObd applys the OCIv1 tarfile in overlaybd format and commit it.
+func (c *overlaybdConvertor) applyOCIV1LayerInObd(
 	ctx context.Context,
 	parentID string, // the ID of parent snapshot
 	desc ocispec.Descriptor, // the descriptor of layer

pkg/snapshot/overlay.go

@@ -18,7 +18,6 @@ package snapshot
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -507,7 +506,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 			}
 			fsType, ok := obdInfo.Labels[label.OverlayBDBlobFsType]
 			if !ok {
-				log.G(ctx).Warnf("cannot get fs type from label, %v", obdInfo.Labels)
+				log.G(ctx).Warnf("cannot get fs type from label, %v, using ext4", obdInfo.Labels)
 				fsType = "ext4"
 			}
 			log.G(ctx).Debugf("attachAndMountBlockDevice (obdID: %s, writeType: %s, fsType %s, targetPath: %s)",
@@ -686,28 +685,11 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
 				return errors.Wrapf(err, "failed to destroy target device for snapshot %s", key)
 			}
 
-			var zfileCfg ZFileConfig
-			if cfgStr, ok := oinfo.Labels[label.ZFileConfig]; ok {
-				if err := json.Unmarshal([]byte(cfgStr), &zfileCfg); err != nil {
-					return err
-				}
-			}
-
-			if err := o.commitWritableOverlaybd(ctx, id, zfileCfg); err != nil {
+			if err := o.sealWritableOverlaybd(ctx, id); err != nil {
 				return err
 			}
 
-			defer func() {
-				if retErr != nil {
-					return
-				}
-
-				// clean up the temporary data
-				os.Remove(o.overlaybdWritableDataPath(id))
-				os.Remove(o.overlaybdWritableIndexPath(id))
-			}()
-
-			opts = append(opts, snapshots.WithLabels(map[string]string{label.LocalOverlayBDPath: o.magicFilePath(id)}))
+			opts = append(opts, snapshots.WithLabels(map[string]string{label.LocalOverlayBDPath: o.overlaybdSealedFilePath(id)}))
 		}
 	}
 
@@ -732,7 +714,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
 			info.Labels = make(map[string]string)
 		}
 
-		info.Labels[label.LocalOverlayBDPath] = o.magicFilePath(id)
+		info.Labels[label.LocalOverlayBDPath] = o.overlaybdSealedFilePath(id)
 		info, err = storage.UpdateInfo(ctx, info, fmt.Sprintf("labels.%s", label.LocalOverlayBDPath))
 		if err != nil {
 			return err
@@ -1118,6 +1100,13 @@ func (o *snapshotter) identifySnapshotStorageType(ctx context.Context, id string
 	}
 	log.G(ctx).Debugf("failed to identify by %s, error %v, try to identify by writable_data", filePath, err)
 
+	// check sealed data file
+	filePath = o.overlaybdSealedFilePath(id)
+	st, err = o.identifyLocalStorageType(filePath)
+	if err == nil {
+		return st, nil
+	}
+
 	// check writable data file
 	filePath = o.overlaybdWritableDataPath(id)
 	st, err = o.identifyLocalStorageType(filePath)
@@ -1151,10 +1140,18 @@ func (o *snapshotter) workPath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "work")
 }
 
+func (o *snapshotter) blockPath(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "block")
+}
+
 func (o *snapshotter) magicFilePath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "fs", "overlaybd.commit")
 }
 
+func (o *snapshotter) overlaybdSealedFilePath(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "fs", "overlaybd.sealed")
+}
+
 func (o *snapshotter) overlaybdMountpoint(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "block", "mountpoint")
 }

pkg/snapshot/storage.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
+	"github.com/containerd/accelerated-container-image/pkg/utils"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
@@ -653,47 +654,16 @@ func (o *snapshotter) atomicWriteOverlaybdTargetConfig(snID string, configJSON *
 
 // prepareWritableOverlaybd
 func (o *snapshotter) prepareWritableOverlaybd(ctx context.Context, snID string) error {
-
-	binpath := filepath.Join(o.config.OverlayBDUtilBinDir, "overlaybd-create")
-
 	// TODO(fuweid): 256GB can be configurable?
-	args := []string{
-		o.overlaybdWritableDataPath(snID),
-		o.overlaybdWritableIndexPath(snID),
-		"64",
-	}
+	args := []string{"64"}
 	if o.writableLayerType == "sparse" {
 		args = append(args, "-s")
 	}
-	out, err := exec.CommandContext(ctx, binpath, args...).CombinedOutput()
-	if err != nil {
-		err := errors.Wrapf(err, "failed to prepare writable overlaybd: %s", out)
-		log.G(ctx).Errorln(err)
-		return err
-	}
-	return nil
+	return utils.Create(ctx, o.blockPath(snID), args...)
 }
 
-// commitWritableOverlaybd
-func (o *snapshotter) commitWritableOverlaybd(ctx context.Context, snID string, cfg ZFileConfig) (retErr error) {
-	binpath := filepath.Join(o.config.OverlayBDUtilBinDir, "overlaybd-commit")
-	opts := []string{
-		"-z",
-		o.overlaybdWritableDataPath(snID),
-		o.overlaybdWritableIndexPath(snID),
-		o.magicFilePath(snID),
-	}
-	if cfg.BlockSize != 0 {
-		opts = append(opts, "--bs", strconv.Itoa(cfg.BlockSize))
-	}
-	if cfg.Algorithm != "" {
-		opts = append(opts, "--algorithm", cfg.Algorithm)
-	}
-	out, err := exec.CommandContext(ctx, binpath, opts...).CombinedOutput()
-	if err != nil {
-		return errors.Wrapf(err, "failed to commit writable overlaybd: %s", out)
-	}
-	return nil
+func (o *snapshotter) sealWritableOverlaybd(ctx context.Context, snID string) (retErr error) {
+	return utils.Seal(ctx, o.blockPath(snID), o.upperPath(snID))
 }
 
 func (o *snapshotter) overlaybdTargetPath(id string) string {