Temp gid

Dockerfile

@@ -49,7 +49,9 @@ RUN cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci /usr/local/bin/
     cp $GOPATH/src/github.com/awslabs/soci-snapshotter/soci-snapshotter.socket /etc/systemd/system
 RUN curl -sSL --output /tmp/containerd.tgz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz && \
     tar zxvf /tmp/containerd.tgz -C /usr/local/ && \
-    rm -f /tmp/containerd.tgz
+    rm -f /tmp/containerd.tgz && \
+    cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/ctr-with-idmapping /usr/local/bin && \
+    chmod +x /usr/local/bin/ctr-with-idmapping
 RUN curl -sSL --output /tmp/runc https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.${TARGETARCH:-amd64} && \
     cp /tmp/runc /usr/local/bin/ && \
     chmod +x /usr/local/bin/runc && \

Makefile

@@ -65,7 +65,7 @@ SOCI_GRPC_PACKAGE_LIST=$(shell echo $(SOCI_LIBRARY_PACKAGE_LIST),$(shell cd $(SO
 GO_BENCHMARK_TESTS?=.
 
 .PHONY: all build check flatc add-ltag install uninstall tidy vendor clean clean-coverage \
-	clean-integration test test-with-coverage show-test-coverage show-test-coverage-html \
+	clean-integration test test-with-coverage show-test-coverage show-test-coverage-html ctr-with-idmapping \
 	integration integration-with-coverage show-integration-coverage show-integration-coverage-html \
 	release benchmarks build-benchmarks benchmarks-perf-test benchmarks-comparison-test
 
@@ -162,7 +162,7 @@ $(COVDIR)/unit: $(COVDIR)
 	GO_BUILD_FLAGS="$(GO_BUILD_FLAGS) -coverpkg=$(SOCI_LIBRARY_PACKAGE_LIST)"\
 		$(MAKE) test
 
-integration: build
+integration: build ctr-with-idmapping
 	@echo "$@"
 	@echo "SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT)"
 	@GO111MODULE=$(GO111MODULE_VALUE) SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT) ENABLE_INTEGRATION_TEST=true go test $(GO_TEST_FLAGS) -v -timeout=0 ./integration
@@ -182,6 +182,19 @@ $(COVDIR)/integration: $(COVDIR)
 	GO_BUILD_FLAGS="$(GO_BUILD_FLAGS) -coverpkg=$(SOCI_CLI_PACKAGE_LIST),$(SOCI_GRPC_PACKAGE_LIST)" \
 		$(MAKE) integration
 
+ctr-with-idmapping: $(OUTDIR)/ctr-with-idmapping
+
+$(OUTDIR)/ctr-with-idmapping:
+    # Use a custom fork for testing ID-mapping as containerd doesn't fully support this yet.
+	git clone https://github.com/containerd/containerd.git tempfolder
+	cd tempfolder && \
+	git checkout v1.7.22 && \
+	git apply $(SOCI_SNAPSHOTTER_PROJECT_ROOT)/integration/config/ctr.patch && \
+	make bin/ctr && \
+	cp bin/ctr $(OUTDIR)/ctr-with-idmapping && \
+	cd ../
+	rm -rf tempfolder
+
 release:
 	@echo "$@"
 	@$(SOCI_SNAPSHOTTER_PROJECT_ROOT)/scripts/create-releases.sh $(RELEASE_TAG)

config/config.toml

@@ -16,6 +16,7 @@ filesystem_cache_type=""
 resolve_result_entry=0
 debug=false
 allow_no_verification=true
+allow_idmap=true
 # disable_verification=false
 # Causes TestRunWithDefaultConfig to break, but
 # fine to use in /etc/soci-snapshotter-grpc-config.toml

config/fs.go

@@ -56,6 +56,7 @@ type FSConfig struct {
 	NoPrometheus                   bool   `toml:"no_prometheus"`
 	MountTimeoutSec                int64  `toml:"mount_timeout_sec"`
 	FuseMetricsEmitWaitDurationSec int64  `toml:"fuse_metrics_emit_wait_duration_sec"`
+	AllowIDMap                     bool   `toml:"allow_idmap" default:"true"`
 
 	RetryableHTTPClientConfig `toml:"http"`
 	BlobConfig                `toml:"blob"`

fs/fs.go

@@ -45,9 +45,12 @@ package fs
 import (
 	"context"
 	"fmt"
+	"io"
 	golog "log"
 	"net/http"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"sync"
 	"syscall"
 	"time"
@@ -59,6 +62,7 @@ import (
 	layermetrics "github.com/awslabs/soci-snapshotter/fs/metrics/layer"
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 	"github.com/awslabs/soci-snapshotter/fs/source"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/awslabs/soci-snapshotter/snapshot"
 	"github.com/awslabs/soci-snapshotter/soci"
@@ -67,6 +71,7 @@ import (
 	ctdsnapshotters "github.com/containerd/containerd/pkg/snapshotters"
 	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	metrics "github.com/docker/go-metrics"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
@@ -455,6 +460,58 @@ func (fs *filesystem) getSociContext(ctx context.Context, imageRef, indexDigest,
 	return c, err
 }
 
+func getIDMappedMountpoint(mountpoint, activeLayerKey string) string {
+	d := filepath.Dir(mountpoint)
+	return filepath.Join(fmt.Sprintf("%s_%s", d, activeLayerKey), "fs")
+}
+
+func (fs *filesystem) IDMapMount(ctx context.Context, mountpoint, activeLayerKey string, idmapper idtools.IDMap) (string, error) {
+	newMountpoint := getIDMappedMountpoint(mountpoint, activeLayerKey)
+	logger := log.G(ctx).WithField("mountpoint", newMountpoint)
+
+	logger.Debug("creating remote id-mapped mount")
+	if err := os.Mkdir(filepath.Dir(newMountpoint), 0700); err != nil {
+		return "", err
+	}
+	if err := os.Mkdir(newMountpoint, 0755); err != nil {
+		return "", err
+	}
+
+	fs.layerMu.Lock()
+	l := fs.layer[mountpoint]
+	if l == nil {
+		fs.layerMu.Unlock()
+		logger.Debug("failed to create remote id-mapped mount")
+		return "", errdefs.ErrNotFound
+	}
+	fs.layer[newMountpoint] = l
+	fs.layerMu.Unlock()
+	node, err := l.RootNode(0, idmapper)
+	if err != nil {
+		return "", err
+	}
+
+	fuseLogger := log.L.
+		WithField("mountpoint", mountpoint).
+		WriterLevel(logrus.TraceLevel)
+
+	return newMountpoint, fs.setupFuseServer(ctx, newMountpoint, node, l, fuseLogger, nil)
+}
+
+func (fs *filesystem) IDMapMountLocal(ctx context.Context, mountpoint, activeLayerKey string, idmapper idtools.IDMap) (string, error) {
+	newMountpoint := getIDMappedMountpoint(mountpoint, activeLayerKey)
+	logger := log.G(ctx).WithField("mountpoint", newMountpoint)
+
+	logger.Debug("creating local id-mapped mount")
+	if err := idtools.RemapDir(ctx, mountpoint, newMountpoint, idmapper); err != nil {
+		logger.Errorf("failed to create local mount: %v", err)
+		return "", err
+	}
+
+	logger.Debug("successfully created local mountpoint")
+	return newMountpoint, nil
+}
+
 func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[string]string) (retErr error) {
 	// Setting the start time to measure the Mount operation duration.
 	start := time.Now()
@@ -566,7 +623,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 	// Maybe we should reword the log here or remove it entirely,
 	// since the old Verify() function no longer serves any purpose.
 
-	node, err := l.RootNode(0)
+	node, err := l.RootNode(0, idtools.IDMap{})
 	if err != nil {
 		log.G(ctx).WithError(err).Warnf("Failed to get root node")
 		retErr = fmt.Errorf("failed to get root node: %w", err)
@@ -583,6 +640,16 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 	fs.layerMu.Unlock()
 	fs.metricsController.Add(mountpoint, l)
 
+	// Pass in a logger to go-fuse with the layer digest
+	// The go-fuse logs are useful for tracing exactly what's happening at the fuse level.
+	fuseLogger := log.L.
+		WithField("layerDigest", labels[ctdsnapshotters.TargetLayerDigestLabel]).
+		WriterLevel(logrus.TraceLevel)
+
+	return fs.setupFuseServer(ctx, mountpoint, node, l, fuseLogger, c)
+}
+
+func (fs *filesystem) setupFuseServer(ctx context.Context, mountpoint string, node fusefs.InodeEmbedder, l layer.Layer, logger *io.PipeWriter, c *sociContext) error {
 	// mount the node to the specified mountpoint
 	// TODO: bind mount the state directory as a read-only fs on snapshotter's side
 	rawFS := fusefs.NewNodeFS(node, &fusefs.Options{
@@ -591,40 +658,37 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 		NegativeTimeout: &fs.negativeTimeout,
 		NullPermissions: true,
 	})
-	// Pass in a logger to go-fuse with the layer digest
-	// The go-fuse logs are useful for tracing exactly what's happening at the fuse level.
-	logger := log.L.
-		WithField("layerDigest", labels[ctdsnapshotters.TargetLayerDigestLabel]).
-		WriterLevel(logrus.TraceLevel)
 	mountOpts := &fuse.MountOptions{
 		AllowOther:    true,   // allow users other than root&mounter to access fs
 		FsName:        "soci", // name this filesystem as "soci"
 		Debug:         fs.debug,
 		Logger:        golog.New(logger, "", 0),
 		DisableXAttrs: l.DisableXAttrs(),
+		Options:       []string{"default_permissions", "ro"},
 	}
 	if _, err := exec.LookPath(fusermountBin); err == nil {
 		mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container
 	} else {
-		log.G(ctx).WithError(err).Infof("%s not installed; trying direct mount", fusermountBin)
+		log.G(ctx).WithField("binary", fusermountBin).WithError(err).Info("fuse binary not installed; trying direct mount")
 		mountOpts.DirectMount = true
 	}
 	server, err := fuse.NewServer(rawFS, mountpoint, mountOpts)
 	if err != nil {
-		log.G(ctx).WithError(err).Debug("failed to make filesystem server")
-		retErr = err
-		return
+		log.G(ctx).WithError(err).Error("failed to make filesystem server")
+		return err
 	}
 
 	go server.Serve()
 
-	// Send a signal to the background fetcher that a new image is being mounted
-	// and to pause all background fetches.
-	c.bgFetchPauseOnce.Do(func() {
-		if fs.bgFetcher != nil {
-			fs.bgFetcher.Pause()
-		}
-	})
+	if c != nil {
+		// Send a signal to the background fetcher that a new image is being mounted
+		// and to pause all background fetches.
+		c.bgFetchPauseOnce.Do(func() {
+			if fs.bgFetcher != nil {
+				fs.bgFetcher.Pause()
+			}
+		})
+	}
 
 	return server.WaitMount()
 }
@@ -692,7 +756,8 @@ func (fs *filesystem) Unmount(ctx context.Context, mountpoint string) error {
 	l, ok := fs.layer[mountpoint]
 	if !ok {
 		fs.layerMu.Unlock()
-		return fmt.Errorf("specified path %q isn't a mountpoint", mountpoint)
+		log.G(ctx).Errorf("specified path %q isn't a remote mount", mountpoint)
+		return errdefs.ErrNotFound
 	}
 	delete(fs.layer, mountpoint) // unregisters the corresponding layer
 	l.Done()
@@ -706,6 +771,10 @@ func (fs *filesystem) Unmount(ctx context.Context, mountpoint string) error {
 	return syscall.Unmount(mountpoint, syscall.MNT_FORCE)
 }
 
+func (fs *filesystem) UnmountLocal(ctx context.Context, mountpoint string) error {
+	return syscall.Unmount(mountpoint, syscall.MNT_FORCE)
+}
+
 // neighboringLayers returns layer descriptors except the `target` layer in the specified manifest.
 func neighboringLayers(manifest ocispec.Manifest, target ocispec.Descriptor) (descs []ocispec.Descriptor) {
 	for _, desc := range manifest.Layers {

fs/fs_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/awslabs/soci-snapshotter/fs/layer"
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 	"github.com/awslabs/soci-snapshotter/fs/source"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes/docker"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
@@ -83,10 +84,12 @@ func (l *breakableLayer) Info() layer.Info {
 		Size: 1,
 	}
 }
-func (l *breakableLayer) DisableXAttrs() bool                           { return false }
-func (l *breakableLayer) RootNode(uint32) (fusefs.InodeEmbedder, error) { return nil, nil }
-func (l *breakableLayer) Verify(tocDigest digest.Digest) error          { return nil }
-func (l *breakableLayer) SkipVerify()                                   {}
+func (l *breakableLayer) DisableXAttrs() bool { return false }
+func (l *breakableLayer) RootNode(uint32, idtools.IDMap) (fusefs.InodeEmbedder, error) {
+	return nil, nil
+}
+func (l *breakableLayer) Verify(tocDigest digest.Digest) error { return nil }
+func (l *breakableLayer) SkipVerify()                          {}
 func (l *breakableLayer) ReadAt([]byte, int64, ...remote.Option) (int, error) {
 	return 0, fmt.Errorf("fail")
 }

fs/layer/layer.go

@@ -58,6 +58,7 @@ import (
 	"github.com/awslabs/soci-snapshotter/fs/remote"
 
 	spanmanager "github.com/awslabs/soci-snapshotter/fs/span-manager"
+	"github.com/awslabs/soci-snapshotter/idtools"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/awslabs/soci-snapshotter/soci"
 	"github.com/awslabs/soci-snapshotter/util/lrucache"
@@ -86,7 +87,7 @@ type Layer interface {
 	Info() Info
 
 	// RootNode returns the root node of this layer.
-	RootNode(baseInode uint32) (fusefs.InodeEmbedder, error)
+	RootNode(baseInode uint32, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error)
 
 	// Check checks if the layer is still connectable.
 	Check() error
@@ -489,14 +490,14 @@ func (l *layerRef) Done() {
 	l.done()
 }
 
-func (l *layer) RootNode(baseInode uint32) (fusefs.InodeEmbedder, error) {
+func (l *layer) RootNode(baseInode uint32, idMapper idtools.IDMap) (fusefs.InodeEmbedder, error) {
 	if l.isClosed() {
 		return nil, fmt.Errorf("layer is already closed")
 	}
 	if l.r == nil {
 		return nil, fmt.Errorf("layer hasn't been verified yet")
 	}
-	return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.resolver.config.LogFuseOperations, l.fuseOperationCounter)
+	return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.resolver.config.LogFuseOperations, l.fuseOperationCounter, idMapper)
 }
 
 func (l *layer) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {