[POC] Import images from tar file

cmd/ignite/cmd/imgcmd/image.go

@@ -33,6 +33,7 @@ func NewCmdImage(out io.Writer) *cobra.Command {
 	}
 
 	cmd.AddCommand(NewCmdImport(out))
+	cmd.AddCommand(NewCmdTarImport(out))
 	cmd.AddCommand(NewCmdLs(out))
 	cmd.AddCommand(NewCmdRm(out))
 	return cmd

cmd/ignite/cmd/imgcmd/tar_import.go

@@ -0,0 +1,29 @@
+package imgcmd
+
+import (
+	"io"
+
+	"github.com/lithammer/dedent"
+	"github.com/spf13/cobra"
+	"github.com/weaveworks/ignite/cmd/ignite/cmd/cmdutil"
+	"github.com/weaveworks/ignite/cmd/ignite/run"
+)
+
+// NewCmdImport imports new VM images from a tar source.
+func NewCmdTarImport(out io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "tarimport <OCI image>",
+		Short: "Import new base images for VMs from a tar file",
+		Long: dedent.Dedent(`
+			Import OCI images as a base images for VMs from a tar file, takes in a file path to
+			a tar file.
+		`),
+		Args: cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdutil.CheckErr(func() error {
+				return run.ImportTarFile(args[0])
+			}())
+		},
+	}
+	return cmd
+}

cmd/ignite/run/import.go

@@ -8,6 +8,11 @@ import (
 	"github.com/weaveworks/ignite/pkg/providers"
 )
 
+// ImportTarFile imports an image from a tar file.
+func ImportTarFile(tarPath string) error {
+	return operations.ImportImageFromTar(providers.Client, tarPath)
+}
+
 func ImportImage(source string) (*api.Image, error) {
 	ociRef, err := meta.NewOCIImageRef(source)
 	if err != nil {

pkg/apis/meta/v1alpha1/image.go

@@ -111,6 +111,13 @@ var _ json.Marshaler = &OCIContentID{}
 var _ json.Unmarshaler = &OCIContentID{}
 
 func parseOCIString(s string) (*OCIContentID, error) {
+	// Check the OCI string ID prefix to determine if it's a local image without
+	// any repo name. Parsing local images that don't have repo name with
+	// url.Parse fails because there's no host in the string.
+	if strings.HasPrefix(s, ociSchemeLocal) {
+		return ParseOCIContentID(strings.TrimPrefix(s, ociSchemeLocal))
+	}
+
 	u, err := url.Parse(s)
 	if err != nil {
 		return nil, err

pkg/dmlegacy/image_format.go

@@ -34,7 +34,15 @@ func CreateImageFilesystem(img *api.Image, src source.Source) error {
 	// To accommodate space for the tar file contents and the ext4 journal + other metadata,
 	// make the base image a sparse file three times the size of the source contents. This
 	// will be shrunk to fit by resizeToMinimum later.
-	if err := imageFile.Truncate(int64(img.Status.OCISource.Size.Bytes()) * 3); err != nil {
+
+	var imgSize int64
+	if (img.Status.OCISource.Size.Bytes() * 3) < 1000000 {
+		imgSize = int64(1000000)
+	} else {
+		imgSize = int64(img.Status.OCISource.Size.Bytes()) * 3
+	}
+	// if err := imageFile.Truncate(int64(img.Status.OCISource.Size.Bytes()) * 3); err != nil {
+	if err := imageFile.Truncate(imgSize); err != nil {
 		return errors.Wrapf(err, "failed to allocate space for image %s", img.GetUID())
 	}
 
@@ -106,6 +114,10 @@ func setupResolvConf(tempDir string) error {
 		return nil
 	}
 
+	if err := os.MkdirAll(filepath.Dir(resolvConf), 0755); err != nil {
+		return err
+	}
+
 	return os.Symlink("../proc/net/pnp", resolvConf)
 }
 

pkg/operations/import.go

@@ -40,7 +40,42 @@ func FindOrImportImage(c *client.Client, ociRef meta.OCIImageRef) (*api.Image, e
 	}
 }
 
-// importKernel imports an image from an OCI image
+// ImportImageFromTar reads a tar file and imports all the images as VM images.
+func ImportImageFromTar(c *client.Client, imagePath string) error {
+	log.Debugf("Importing image from tar file %q", imagePath)
+	dockerSource := source.NewDockerSource()
+	img, err := dockerSource.Import(imagePath)
+	if err != nil {
+		return err
+	}
+
+	// Create ignite image for all the imports.
+	for imgRef, imgSrc := range img {
+		image := c.Images().New()
+		image.Name = imgRef.String()
+		image.Spec.OCI = imgRef
+		image.Status.OCISource = *imgSrc
+
+		// Generate UID automatically.
+		if err := metadata.SetNameAndUID(image, c); err != nil {
+			return err
+		}
+
+		dockerSource.SetRef(imgRef)
+		if err := dmlegacy.CreateImageFilesystem(image, dockerSource); err != nil {
+			return err
+		}
+
+		if err := c.Images().Set(image); err != nil {
+			return err
+		}
+
+		log.Infof("Imported OCI image %q (%s) to base image with UID %q", imgRef, image.Status.OCISource.Size, image.GetUID())
+	}
+	return nil
+}
+
+// importImage imports an image from an OCI image
 func importImage(c *client.Client, ociRef meta.OCIImageRef) (*api.Image, error) {
 	log.Debugf("Importing image with ociRef %q", ociRef)
 	// Parse the source

pkg/runtime/containerd/client.go

@@ -144,6 +144,40 @@ func (cc *ctdClient) PullImage(image meta.OCIImageRef) error {
 	return err
 }
 
+// ImportImage imports a local image from a given tar file and returns a list of
+// OCI image refs of all the imported images.
+func (cc *ctdClient) ImportImage(imageFilePath string) ([]meta.OCIImageRef, error) {
+	log.Debugf("containerd: Importing image from %s", imageFilePath)
+	r, err := os.Open(imageFilePath)
+	if err != nil {
+		return nil, err
+	}
+	imgs, err := cc.client.Import(cc.ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	if err := r.Close(); err != nil {
+		return nil, err
+	}
+
+	imgRefs := []meta.OCIImageRef{}
+	for _, img := range imgs {
+		image := containerd.NewImage(cc.client, img)
+
+		log.Debugf("unpacking %s (%s)...", img.Name, img.Target.Digest)
+		if err := image.Unpack(cc.ctx, ""); err != nil {
+			return nil, fmt.Errorf("failed to unpack image: %v", err)
+		}
+
+		imgRef, err := meta.NewOCIImageRef(img.Name)
+		if err != nil {
+			return nil, err
+		}
+		imgRefs = append(imgRefs, imgRef)
+	}
+	return imgRefs, nil
+}
+
 func (cc *ctdClient) InspectImage(image meta.OCIImageRef) (result *runtime.ImageInspectResult, err error) {
 	var img containerd.Image
 	var config imagespec.Descriptor

pkg/runtime/docker/client.go

@@ -1,17 +1,22 @@
 package docker
 
 import (
+	"bufio"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	cont "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
+	log "github.com/sirupsen/logrus"
 	meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
 	"github.com/weaveworks/ignite/pkg/preflight"
 	"github.com/weaveworks/ignite/pkg/preflight/checkers"
@@ -21,6 +26,10 @@ import (
 
 const (
 	dcSocket = "/var/run/docker.sock"
+
+	// streamLoadedImage is a substring of the stream logs from docker server
+	// response when importing a local docker image from tar file.
+	streamLoadedImage = "Loaded image"
 )
 
 // dockerClient is a runtime.Interface
@@ -29,6 +38,11 @@ type dockerClient struct {
 	client *client.Client
 }
 
+// dockerStreamOutput is the structure of a docker stream output.
+type dockerStreamOutput struct {
+	Stream string `json:"stream"`
+}
+
 var _ runtime.Interface = &dockerClient{}
 
 // GetDockerClient builds a client for talking to docker
@@ -54,6 +68,57 @@ func (dc *dockerClient) PullImage(image meta.OCIImageRef) (err error) {
 	return
 }
 
+// ImportImage imports a local image from a given tar file and returns a list of
+// OCI image refs of all the imported images.
+func (dc *dockerClient) ImportImage(imageFilePath string) ([]meta.OCIImageRef, error) {
+	log.Debugf("docker: Importing image from %s", imageFilePath)
+	r, err := os.Open(imageFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	resp, err := dc.client.ImageLoad(context.Background(), r, true)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	imgRefs := []meta.OCIImageRef{}
+
+	// Maybe open the tar file and read manifest.json to get the image refs?
+
+	// Response body content looks like:
+	// {"stream":"Loaded image: hello-world:latest\n"}
+	// {"stream":"Loaded image: busybox:latest\n"}
+	//
+	// Parse the docker steam output and get the image refs.
+	rd := bufio.NewScanner(resp.Body)
+	for rd.Scan() {
+		log.Debugf("docker: %s", rd.Text())
+
+		// Parse the json output to get the stream message.
+		var stream dockerStreamOutput
+		if err := json.Unmarshal(rd.Bytes(), &stream); err != nil {
+			return imgRefs, err
+		}
+
+		// Split the message at ":" and check if the first part contains "Loaded
+		// image".
+		message := strings.SplitN(stream.Stream, ":", 2)
+		if strings.Contains(message[0], streamLoadedImage) {
+			// Trim the second part of the string and form an OCI image ref.
+			image, err := meta.NewOCIImageRef(strings.TrimSpace(message[1]))
+			if err != nil {
+				return imgRefs, err
+			}
+			imgRefs = append(imgRefs, image)
+		}
+	}
+
+	return imgRefs, nil
+}
+
 func (dc *dockerClient) InspectImage(image meta.OCIImageRef) (*runtime.ImageInspectResult, error) {
 	res, _, err := dc.client.ImageInspectWithRaw(context.Background(), image.Normalized())
 	if err != nil {

pkg/runtime/types.go

@@ -51,6 +51,7 @@ type ContainerConfig struct {
 
 type Interface interface {
 	PullImage(image meta.OCIImageRef) error
+	ImportImage(imageFilePath string) ([]meta.OCIImageRef, error)
 	InspectImage(image meta.OCIImageRef) (*ImageInspectResult, error)
 	ExportImage(image meta.OCIImageRef) (io.ReadCloser, func() error, error)
 

pkg/source/docker.go

@@ -29,6 +29,10 @@ func (ds *DockerSource) Ref() meta.OCIImageRef {
 	return ds.imageRef
 }
 
+func (ds *DockerSource) SetRef(imageRef meta.OCIImageRef) {
+	ds.imageRef = imageRef
+}
+
 func (ds *DockerSource) Parse(ociRef meta.OCIImageRef) (*api.OCIImageSource, error) {
 	res, err := providers.Runtime.InspectImage(ociRef)
 	if err != nil {
@@ -54,6 +58,29 @@ func (ds *DockerSource) Parse(ociRef meta.OCIImageRef) (*api.OCIImageSource, err
 	}, nil
 }
 
+func (ds *DockerSource) Import(imagePath string) (map[meta.OCIImageRef]*api.OCIImageSource, error) {
+	imgRefs, err := providers.Runtime.ImportImage(imagePath)
+	if err != nil {
+		return nil, err
+	}
+
+	img := map[meta.OCIImageRef]*api.OCIImageSource{}
+	for _, imgRef := range imgRefs {
+		// Inspect image and get image ID from inspect result. Use ID to create
+		// image source. Image source can be used to create ignite image.
+		res, err := providers.Runtime.InspectImage(imgRef)
+		if err != nil {
+			return nil, err
+		}
+
+		img[imgRef] = &api.OCIImageSource{
+			ID:   res.ID,
+			Size: meta.NewSizeFromBytes(uint64(res.Size)),
+		}
+	}
+	return img, nil
+}
+
 func (ds *DockerSource) Reader() (rc io.ReadCloser, err error) {
 	// Export the image
 	rc, ds.cleanupFunc, err = providers.Runtime.ExportImage(ds.imageRef)

pkg/source/source.go

@@ -12,9 +12,15 @@ type Source interface {
 	// Ref returns the reference of the source
 	Ref() meta.OCIImageRef
 
+	// SetRef allows setting reference of the source.
+	SetRef(meta.OCIImageRef)
+
 	// Parse verifies the ImageSource, fills in any missing fields and prepares the reader
 	Parse(src meta.OCIImageRef) (*api.OCIImageSource, error)
 
+	// Import  an image from a file source and prepares the reader.
+	Import(path string) (map[meta.OCIImageRef]*api.OCIImageSource, error)
+
 	// Reader provides a tar stream reader
 	Reader() (io.ReadCloser, error)