diff --git a/Makefile b/Makefile index 80f467cd8..812b61fda 100644 --- a/Makefile +++ b/Makefile @@ -278,8 +278,8 @@ demo-network: install-cni-bins $(FCNET_CONFIG) # Firecracker submodule ########################## .PHONY: firecracker -firecracker: $(FIRECRACKER_BIN) $(JAILER_BIN) - +firecracker: + _submodules/firecracker/tools/devtool build --release .PHONY: install-firecracker install-firecracker: firecracker install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $(FIRECRACKER_BIN) diff --git a/_submodules/firecracker b/_submodules/firecracker index 1ee9e9365..9511525c8 160000 --- a/_submodules/firecracker +++ b/_submodules/firecracker @@ -1 +1 @@ -Subproject commit 1ee9e936537b950452eb75fc22a9c4301783b7f3 +Subproject commit 9511525c8d8abf836a45f922ca69ef18b04efa41 diff --git a/firecracker-control/local.go b/firecracker-control/local.go index 89dbd770b..973c5e0ed 100644 --- a/firecracker-control/local.go +++ b/firecracker-control/local.go @@ -71,6 +71,8 @@ type local struct { processesMu sync.Mutex processes map[string]int32 + + fcControlSocket *net.UnixListener } func newLocal(ic *plugin.InitContext) (*local, error) { @@ -243,7 +245,7 @@ func (s *local) StopVM(requestCtx context.Context, req *proto.StopVMRequest) (*e defer client.Close() resp, shimErr := client.StopVM(requestCtx, req) - waitErr := s.waitForShimToExit(requestCtx, req.VMID) + waitErr := s.waitForShimToExit(requestCtx, req.VMID, false) // Assuming the shim is returning containerd's error code, return the error as is if possible. if waitErr == nil { @@ -252,7 +254,7 @@ func (s *local) StopVM(requestCtx context.Context, req *proto.StopVMRequest) (*e return resp, multierror.Append(shimErr, waitErr).ErrorOrNil() } -func (s *local) waitForShimToExit(ctx context.Context, vmID string) error { +func (s *local) waitForShimToExit(ctx context.Context, vmID string, killShim bool) error { socketAddr, err := fcShim.SocketAddress(ctx, vmID) if err != nil { return err @@ -267,6 +269,17 @@ func (s *local) waitForShimToExit(ctx context.Context, vmID string) error { } defer delete(s.processes, socketAddr) + if killShim { + s.logger.Debug("Killing shim") + + if err := syscall.Kill(int(pid), syscall.SIGKILL); err != nil { + s.logger.WithError(err).Error("Failed to kill shim process") + return err + } + + return nil + } + return internal.WaitForPidToExit(ctx, stopVMInterval, pid) } @@ -429,10 +442,6 @@ func (s *local) newShim(ns, vmID, containerdAddress string, shimSocket *net.Unix if err := fcSocketFile.Close(); err != nil { logger.WithError(err).Errorf("failed to close %q", fcSocketFile.Name()) } - - if err := os.RemoveAll(shimDir.RootPath()); err != nil { - logger.WithError(err).Errorf("failed to remove %q", shimDir.RootPath()) - } }() err = setShimOOMScore(cmd.Process.Pid) @@ -463,3 +472,275 @@ func setShimOOMScore(shimPid int) error { return nil } + +func (s *local) loadShim(ctx context.Context, ns, vmID, containerdAddress string) (*exec.Cmd, error) { + logger := s.logger.WithField("vmID", vmID) + logger.Debug("Loading shim") + + shimSocketAddress, err := fcShim.SocketAddress(ctx, vmID) + if err != nil { + err = errors.Wrap(err, "failed to obtain shim socket address") + s.logger.WithError(err).Error() + return nil, err + } + + shimSocket, err := shim.NewSocket(shimSocketAddress) + if isEADDRINUSE(err) { + return nil, status.Errorf(codes.AlreadyExists, "VM with ID %q already exists (socket: %q)", vmID, shimSocketAddress) + } else if err != nil { + err = errors.Wrapf(err, "failed to open shim socket at address %q", shimSocketAddress) + s.logger.WithError(err).Error() + return nil, err + } + + defer shimSocket.Close() + + // If we're here, there is no pre-existing shim for this VMID, so we spawn a new one + if _, err := os.Stat(s.config.ShimBaseDir); os.IsNotExist(err) { + return nil, errors.Wrapf(err, "shim base dir does not exist: %s", s.config.ShimBaseDir) + } + + shimDir, err := vm.ShimDir(s.config.ShimBaseDir, ns, vmID) + if err != nil { + err = errors.Wrapf(err, "failed to build shim path") + s.logger.WithError(err).Error() + return nil, err + } + + fcSocketAddress, err := fcShim.FCControlSocketAddress(ctx, vmID) + if err != nil { + err = errors.Wrap(err, "failed to obtain shim socket address") + s.logger.WithError(err).Error() + return nil, err + } + + fcSocket, err := shim.NewSocket(fcSocketAddress) + if err != nil { + err = errors.Wrapf(err, "failed to open fccontrol socket at address %q", fcSocketAddress) + s.logger.WithError(err).Error() + return nil, err + } + + s.fcControlSocket = fcSocket + + args := []string{ + "-namespace", ns, + "-address", containerdAddress, + } + + cmd := exec.Command(internal.ShimBinaryName, args...) + + // note: The working dir of the shim has an effect on the length of the path + // needed to specify various unix sockets that the shim uses to communicate + // with the firecracker VMM and guest agent within. The length of that path + // has a relatively low limit (usually 108 chars), so modifying the working + // dir should be done with caution. See internal/vm/dir.go for the path + // definitions. + cmd.Dir = shimDir.RootPath() + + shimSocketFile, err := shimSocket.File() + if err != nil { + err = errors.Wrap(err, "failed to get shim socket fd") + logger.WithError(err).Error() + return nil, err + } + + fcSocketFile, err := fcSocket.File() + if err != nil { + err = errors.Wrap(err, "failed to get shim fccontrol socket fd") + logger.WithError(err).Error() + return nil, err + } + + cmd.ExtraFiles = append(cmd.ExtraFiles, shimSocketFile, fcSocketFile) + fcSocketFDNum := 2 + len(cmd.ExtraFiles) // "2 +" because ExtraFiles come after stderr (fd #2) + + ttrpc := containerdAddress + ".ttrpc" + cmd.Env = append(os.Environ(), + fmt.Sprintf("%s=%s", ttrpcAddressEnv, ttrpc), + fmt.Sprintf("%s=%s", internal.VMIDEnvVarKey, vmID), + fmt.Sprintf("%s=%s", internal.FCSocketFDEnvKey, strconv.Itoa(fcSocketFDNum))) // TODO remove after containerd is updated to expose ttrpc server to shim + + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + + // shim stderr is just raw text, so pass it through our logrus formatter first + cmd.Stderr = logger.WithField("shim_stream", "stderr").WriterLevel(logrus.ErrorLevel) + // shim stdout on the other hand is already formatted by logrus, so pass that transparently through to containerd logs + cmd.Stdout = logger.Logger.Out + + logger.Debugf("starting %s", internal.ShimBinaryName) + + err = cmd.Start() + if err != nil { + err = errors.Wrap(err, "failed to start shim child process") + logger.WithError(err).Error() + return nil, err + } + + // make sure to wait after start + go func() { + if err := cmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + // shim is usually terminated by cancelling the context + logger.WithError(exitErr).Debug("shim has been terminated") + } else { + logger.WithError(err).Error("shim has been unexpectedly terminated") + } + } + + // Close all Unix abstract sockets. + if err := shimSocketFile.Close(); err != nil { + logger.WithError(err).Errorf("failed to close %q", shimSocketFile.Name()) + } + if err := fcSocketFile.Close(); err != nil { + logger.WithError(err).Errorf("failed to close %q", fcSocketFile.Name()) + } + }() + + err = setShimOOMScore(cmd.Process.Pid) + if err != nil { + logger.WithError(err).Error() + return nil, err + } + + s.addShim(shimSocketAddress, cmd) + + return cmd, nil +} + +// PauseVM Pauses a VM +func (s *local) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.PauseVM(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// ResumeVM Resumes a VM +func (s *local) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.ResumeVM(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// CreateSnapshot Creates a snapshot of a VM +func (s *local) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.CreateSnapshot(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// LoadSnapshot Loads a snapshot of a VM +func (s *local) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + ns, err := namespaces.NamespaceRequired(ctx) + if err != nil { + err = errors.Wrap(err, "error retrieving namespace of request") + s.logger.WithError(err).Error() + return nil, err + } + + _, err = s.loadShim(ctx, ns, req.VMID, s.containerdAddress) + if err != nil { + return nil, err + } + + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.LoadSnapshot(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// Offload Shuts down a VM started through the firecracker go sdk and deletes +// the corresponding firecracker socket. All of the other resources (shim, other sockets) +// will persist. +func (s *local) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.Offload(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + s.fcControlSocket.Close() + + shimSocketAddress, err := fcShim.SocketAddress(ctx, req.VMID) + if err != nil { + err = errors.Wrap(err, "failed to obtain shim socket address") + s.logger.WithError(err).Error() + return nil, err + } + removeErr := os.RemoveAll(shimSocketAddress) + if removeErr != nil { + s.logger.Errorf("failed to remove shim socket addr file: %v", removeErr) + return nil, err + } + + fcSocketAddress, err := fcShim.FCControlSocketAddress(ctx, req.VMID) + if err != nil { + s.logger.Error("failed to get FC socket address") + return nil, err + } + removeErr = os.RemoveAll(fcSocketAddress) + if removeErr != nil { + s.logger.Errorf("failed to remove fc socket addr file: %v", removeErr) + return nil, err + } + + waitErr := s.waitForShimToExit(ctx, req.VMID, true) + if waitErr != nil { + s.logger.Error("failed to wait for shim to exit on offload") + return nil, waitErr + } + + return resp, nil +} diff --git a/firecracker-control/service.go b/firecracker-control/service.go index 5c931aba5..a447c6396 100644 --- a/firecracker-control/service.go +++ b/firecracker-control/service.go @@ -96,3 +96,28 @@ func (s *service) GetVMMetadata(ctx context.Context, req *proto.GetVMMetadataReq log.G(ctx).Debug("Getting vm metadata") return s.local.GetVMMetadata(ctx, req) } + +func (s *service) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("pause VM request: %+v", req) + return s.local.PauseVM(ctx, req) +} + +func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("resume VM request: %+v", req) + return s.local.ResumeVM(ctx, req) +} + +func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("load snapshot request: %+v", req) + return s.local.LoadSnapshot(ctx, req) +} + +func (s *service) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("create snapshot request: %+v", req) + return s.local.CreateSnapshot(ctx, req) +} + +func (s *service) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("offload request: %+v", req) + return s.local.Offload(ctx, req) +} diff --git a/go.mod b/go.mod index 4ed72e0e8..854f41336 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/firecracker-microvm/firecracker-containerd +replace github.com/firecracker-microvm/firecracker-go-sdk => github.com/ustiugov/firecracker-go-sdk v0.20.1-0.20200625102438-8edf287b0123 + require ( github.com/Microsoft/go-winio v0.4.14 // indirect github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect @@ -17,7 +19,8 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect - github.com/firecracker-microvm/firecracker-go-sdk v0.21.1-0.20200811001213-ee1e7c41b7bd + github.com/docker/go-units v0.4.0 // indirect + github.com/firecracker-microvm/firecracker-go-sdk v0.0.0-00010101000000-000000000000 github.com/go-ole/go-ole v1.2.4 // indirect github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384 // indirect github.com/gofrs/uuid v3.3.0+incompatible @@ -40,11 +43,13 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect + github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c github.com/urfave/cli v1.20.0 // indirect github.com/vishvananda/netlink v1.1.0 go.etcd.io/bbolt v1.3.1-etcd.8 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd + golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1 // indirect google.golang.org/genproto v0.0.0-20181109154231-b5d43981345b // indirect google.golang.org/grpc v1.21.0 gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index 56e869abc..0f8baf399 100644 --- a/go.sum +++ b/go.sum @@ -320,8 +320,13 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/ustiugov/firecracker-go-sdk v0.20.1-0.20200625102438-8edf287b0123 h1:RmVkdn3XSJSVUy/yStvqVJGQ8yw93mUtJn4N+8Aytlw= +github.com/ustiugov/firecracker-go-sdk v0.20.1-0.20200625102438-8edf287b0123/go.mod h1:zyc9BrKGePpNLbQ5y2ZtdzXEfpMJeHPeFNVpyo0S1WQ= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf h1:3J37+NPjNyGW/dbfXtj3yWuF9OEepIdGOXRaJGbORV8= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= @@ -350,8 +355,19 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -364,6 +380,7 @@ golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1 golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -377,6 +394,10 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -415,7 +436,20 @@ golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774 h1:CQVOmarCBFzTx0kbOU0ru54Cvot8SdSrNYjZPhQl+gk= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixaA66WGWmrijSlxaM+JM/g= +golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200708183856-df98bc6d456c h1:Jt8nybBNSGn80qEV8fQLwCam6RQeX4dsxit8if67Sfc= +golang.org/x/tools v0.0.0-20200708183856-df98bc6d456c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1 h1:rD1FcWVsRaMY+l8biE9jbWP5MS/CJJ/90a9TMkMgNrM= +golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181109154231-b5d43981345b h1:WkFtVmaZoTRVoRYr0LTC9SYNhlw0X0HrVPz2OVssVm4= diff --git a/internal/common.go b/internal/common.go index 9e7a78b48..cb8cfb7a9 100644 --- a/internal/common.go +++ b/internal/common.go @@ -50,6 +50,12 @@ const ( // ShimLogFifoName is the name of the FIFO created by containerd for a shim to write its logs to ShimLogFifoName = "log" + // LogPathNameStart is the name of the FIFO created by containerd for a shim to write its logs to + LogPathNameStart = "log_start" + + // LogPathNameLoad is the name of the FIFO created by containerd for a shim to write its logs to + LogPathNameLoad = "log_load" + // OCIConfigName is the name of the OCI bundle's config field OCIConfigName = "config.json" diff --git a/internal/vm/dir.go b/internal/vm/dir.go index 228350a35..6c3fdece7 100644 --- a/internal/vm/dir.go +++ b/internal/vm/dir.go @@ -83,6 +83,19 @@ func (d Dir) OpenLogFifo(requestCtx context.Context) (io.ReadWriteCloser, error) return fifo.OpenFifo(requestCtx, d.LogFifoPath(), unix.O_WRONLY|unix.O_NONBLOCK, 0200) } +// LogStartPath returns the path to the file for storing +// firecracker logs after the microVM is started and until +// it is Offloaded +func (d Dir) LogStartPath() string { + return filepath.Join(d.RootPath(), internal.LogPathNameStart) +} + +// LogLoadPath returns the path to the file for storing +// firecracker logs after the microVM is loaded from a snapshot +func (d Dir) LogLoadPath() string { + return filepath.Join(d.RootPath(), internal.LogPathNameLoad) +} + // FirecrackerSockPath returns the path to the unix socket at which the firecracker VMM // services its API func (d Dir) FirecrackerSockPath() string { diff --git a/proto/firecracker.pb.go b/proto/firecracker.pb.go index 2d9697e42..7f0ab3f74 100644 --- a/proto/firecracker.pb.go +++ b/proto/firecracker.pb.go @@ -569,6 +569,228 @@ func (m *GetVMMetadataResponse) GetMetadata() string { return "" } +type PauseVMRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PauseVMRequest) Reset() { *m = PauseVMRequest{} } +func (m *PauseVMRequest) String() string { return proto.CompactTextString(m) } +func (*PauseVMRequest) ProtoMessage() {} +func (*PauseVMRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{9} +} +func (m *PauseVMRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PauseVMRequest.Unmarshal(m, b) +} +func (m *PauseVMRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PauseVMRequest.Marshal(b, m, deterministic) +} +func (m *PauseVMRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PauseVMRequest.Merge(m, src) +} +func (m *PauseVMRequest) XXX_Size() int { + return xxx_messageInfo_PauseVMRequest.Size(m) +} +func (m *PauseVMRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PauseVMRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PauseVMRequest proto.InternalMessageInfo + +func (m *PauseVMRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +type ResumeVMRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ResumeVMRequest) Reset() { *m = ResumeVMRequest{} } +func (m *ResumeVMRequest) String() string { return proto.CompactTextString(m) } +func (*ResumeVMRequest) ProtoMessage() {} +func (*ResumeVMRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{10} +} +func (m *ResumeVMRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ResumeVMRequest.Unmarshal(m, b) +} +func (m *ResumeVMRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ResumeVMRequest.Marshal(b, m, deterministic) +} +func (m *ResumeVMRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResumeVMRequest.Merge(m, src) +} +func (m *ResumeVMRequest) XXX_Size() int { + return xxx_messageInfo_ResumeVMRequest.Size(m) +} +func (m *ResumeVMRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ResumeVMRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ResumeVMRequest proto.InternalMessageInfo + +func (m *ResumeVMRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +type CreateSnapshotRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + SnapshotFilePath string `protobuf:"bytes,2,opt,name=SnapshotFilePath,json=snapshotFilePath,proto3" json:"SnapshotFilePath,omitempty"` + MemFilePath string `protobuf:"bytes,3,opt,name=MemFilePath,json=memFilePath,proto3" json:"MemFilePath,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } +func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*CreateSnapshotRequest) ProtoMessage() {} +func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{11} +} +func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateSnapshotRequest.Unmarshal(m, b) +} +func (m *CreateSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *CreateSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateSnapshotRequest.Merge(m, src) +} +func (m *CreateSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_CreateSnapshotRequest.Size(m) +} +func (m *CreateSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateSnapshotRequest proto.InternalMessageInfo + +func (m *CreateSnapshotRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +func (m *CreateSnapshotRequest) GetSnapshotFilePath() string { + if m != nil { + return m.SnapshotFilePath + } + return "" +} + +func (m *CreateSnapshotRequest) GetMemFilePath() string { + if m != nil { + return m.MemFilePath + } + return "" +} + +type LoadSnapshotRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + SnapshotFilePath string `protobuf:"bytes,2,opt,name=SnapshotFilePath,json=snapshotFilePath,proto3" json:"SnapshotFilePath,omitempty"` + MemFilePath string `protobuf:"bytes,3,opt,name=MemFilePath,json=memFilePath,proto3" json:"MemFilePath,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadSnapshotRequest) Reset() { *m = LoadSnapshotRequest{} } +func (m *LoadSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*LoadSnapshotRequest) ProtoMessage() {} +func (*LoadSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{12} +} +func (m *LoadSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadSnapshotRequest.Unmarshal(m, b) +} +func (m *LoadSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *LoadSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadSnapshotRequest.Merge(m, src) +} +func (m *LoadSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_LoadSnapshotRequest.Size(m) +} +func (m *LoadSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoadSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadSnapshotRequest proto.InternalMessageInfo + +func (m *LoadSnapshotRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +func (m *LoadSnapshotRequest) GetSnapshotFilePath() string { + if m != nil { + return m.SnapshotFilePath + } + return "" +} + +func (m *LoadSnapshotRequest) GetMemFilePath() string { + if m != nil { + return m.MemFilePath + } + return "" +} + +type OffloadRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *OffloadRequest) Reset() { *m = OffloadRequest{} } +func (m *OffloadRequest) String() string { return proto.CompactTextString(m) } +func (*OffloadRequest) ProtoMessage() {} +func (*OffloadRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{13} +} +func (m *OffloadRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_OffloadRequest.Unmarshal(m, b) +} +func (m *OffloadRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_OffloadRequest.Marshal(b, m, deterministic) +} +func (m *OffloadRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_OffloadRequest.Merge(m, src) +} +func (m *OffloadRequest) XXX_Size() int { + return xxx_messageInfo_OffloadRequest.Size(m) +} +func (m *OffloadRequest) XXX_DiscardUnknown() { + xxx_messageInfo_OffloadRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_OffloadRequest proto.InternalMessageInfo + +func (m *OffloadRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + type JailerConfig struct { NetNS string `protobuf:"bytes,1,opt,name=NetNS,json=netNS,proto3" json:"NetNS,omitempty"` // List of the physical numbers of the CPUs on which processes in that @@ -609,7 +831,7 @@ func (m *JailerConfig) Reset() { *m = JailerConfig{} } func (m *JailerConfig) String() string { return proto.CompactTextString(m) } func (*JailerConfig) ProtoMessage() {} func (*JailerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_a73317e9fb8da571, []int{9} + return fileDescriptor_a73317e9fb8da571, []int{14} } func (m *JailerConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JailerConfig.Unmarshal(m, b) @@ -689,6 +911,11 @@ func init() { proto.RegisterType((*UpdateVMMetadataRequest)(nil), "UpdateVMMetadataRequest") proto.RegisterType((*GetVMMetadataRequest)(nil), "GetVMMetadataRequest") proto.RegisterType((*GetVMMetadataResponse)(nil), "GetVMMetadataResponse") + proto.RegisterType((*PauseVMRequest)(nil), "PauseVMRequest") + proto.RegisterType((*ResumeVMRequest)(nil), "ResumeVMRequest") + proto.RegisterType((*CreateSnapshotRequest)(nil), "CreateSnapshotRequest") + proto.RegisterType((*LoadSnapshotRequest)(nil), "LoadSnapshotRequest") + proto.RegisterType((*OffloadRequest)(nil), "OffloadRequest") proto.RegisterType((*JailerConfig)(nil), "JailerConfig") } diff --git a/proto/firecracker.proto b/proto/firecracker.proto index 6a9956e0d..aa80798d5 100644 --- a/proto/firecracker.proto +++ b/proto/firecracker.proto @@ -90,6 +90,23 @@ enum DriveExposePolicy { BIND = 1; } +message CreateSnapshotRequest { + string VMID = 1; + string SnapshotFilePath = 2; + string MemFilePath = 3; +} + +message LoadSnapshotRequest { + string VMID = 1; + string SnapshotFilePath = 2; + string MemFilePath = 3; +} + +message OffloadRequest { + string VMID = 1; +} + + message JailerConfig { string NetNS = 1; // List of the physical numbers of the CPUs on which processes in that diff --git a/proto/service/fccontrol/fccontrol.proto b/proto/service/fccontrol/fccontrol.proto index a55ff09f6..5c6e085d0 100644 --- a/proto/service/fccontrol/fccontrol.proto +++ b/proto/service/fccontrol/fccontrol.proto @@ -24,4 +24,19 @@ service Firecracker { // Get Vm's instance metadata rpc GetVMMetadata(GetVMMetadataRequest) returns (GetVMMetadataResponse); + + // Pauses a VM + rpc PauseVM(PauseVMRequest) returns (google.protobuf.Empty); + + // Resumes a VM + rpc ResumeVM(ResumeVMRequest) returns (google.protobuf.Empty); + + // Loads VM from snapshot + rpc LoadSnapshot(LoadSnapshotRequest) returns (google.protobuf.Empty); + + // Make a snapshot of a VM + rpc CreateSnapshot(CreateSnapshotRequest) returns (google.protobuf.Empty); + + // Offload a snapshotted VM + rpc Offload(OffloadRequest) returns (google.protobuf.Empty); } diff --git a/proto/service/fccontrol/ttrpc/fccontrol.pb.go b/proto/service/fccontrol/ttrpc/fccontrol.pb.go index 9ffb89bb3..f97fc42e7 100644 --- a/proto/service/fccontrol/ttrpc/fccontrol.pb.go +++ b/proto/service/fccontrol/ttrpc/fccontrol.pb.go @@ -27,24 +27,28 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("fccontrol.proto", fileDescriptor_b99f53e2bf82c5ef) } var fileDescriptor_b99f53e2bf82c5ef = []byte{ - // 261 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4f, 0x4b, 0x4e, 0xce, - 0xcf, 0x2b, 0x29, 0xca, 0xcf, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x97, 0x92, 0x4e, 0xcf, 0xcf, - 0x4f, 0xcf, 0x49, 0xd5, 0x07, 0xf3, 0x92, 0x4a, 0xd3, 0xf4, 0x53, 0x73, 0x0b, 0x4a, 0x2a, 0xa1, - 0x92, 0x82, 0x69, 0x99, 0x45, 0xa9, 0xc9, 0x45, 0x89, 0xc9, 0xd9, 0xa9, 0x45, 0x10, 0x21, 0xa3, - 0x57, 0x4c, 0x5c, 0xdc, 0x6e, 0x08, 0x51, 0x21, 0x7d, 0x2e, 0x0e, 0xe7, 0xa2, 0xd4, 0xc4, 0x92, - 0xd4, 0x30, 0x5f, 0x21, 0x01, 0x3d, 0x18, 0x33, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x4a, - 0x10, 0x49, 0xa4, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, 0x80, 0x8b, 0x2d, 0xb8, 0x24, 0xbf, - 0x20, 0xcc, 0x57, 0x88, 0x4f, 0x0f, 0xc2, 0x80, 0x29, 0x16, 0xd3, 0x83, 0xb8, 0x45, 0x0f, 0xe6, - 0x16, 0x3d, 0x57, 0x90, 0x5b, 0x84, 0x8c, 0xb8, 0x38, 0xdd, 0x53, 0x4b, 0xc2, 0x7c, 0x3d, 0xf3, - 0xd2, 0xf2, 0x85, 0x04, 0xf5, 0xe0, 0x6c, 0x98, 0x3e, 0x21, 0x64, 0x21, 0xa8, 0x2d, 0x76, 0x5c, - 0xbc, 0xc1, 0x20, 0x41, 0xdf, 0xd4, 0x92, 0xc4, 0x94, 0xc4, 0x92, 0x44, 0x21, 0x51, 0x3d, 0x14, - 0x3e, 0x21, 0x3b, 0x5d, 0xb8, 0x04, 0x42, 0x0b, 0x52, 0xc0, 0x2e, 0x87, 0x1b, 0x21, 0xa1, 0x87, - 0x2e, 0x44, 0xc8, 0x14, 0x3b, 0x2e, 0x5e, 0x77, 0x34, 0x57, 0xb8, 0x63, 0x77, 0x05, 0x9a, 0x30, - 0xc4, 0x17, 0x4e, 0xca, 0x27, 0x1e, 0xca, 0x31, 0xdc, 0x78, 0x28, 0xc7, 0xd0, 0xf0, 0x48, 0x8e, - 0xf1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x8c, 0xe2, 0x84, - 0xc7, 0x63, 0x12, 0x1b, 0xd8, 0x52, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x15, 0x3e, - 0x05, 0xdb, 0x01, 0x00, 0x00, + // 336 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xbb, 0x4e, 0xf3, 0x40, + 0x10, 0x85, 0xe3, 0xe2, 0xcf, 0x65, 0x7e, 0x72, 0x5b, 0x41, 0x84, 0x82, 0xe4, 0x86, 0x7e, 0x82, + 0x02, 0x25, 0x8a, 0x10, 0xb7, 0x08, 0x09, 0x0b, 0x94, 0x88, 0x14, 0x74, 0x1b, 0x7b, 0x1c, 0x10, + 0x89, 0xc7, 0xd8, 0xeb, 0x82, 0x8e, 0x92, 0x47, 0x4b, 0x49, 0x49, 0x49, 0xfc, 0x24, 0x28, 0xf1, + 0x85, 0x24, 0x42, 0xda, 0xee, 0xcc, 0x67, 0x9f, 0x99, 0xb3, 0x3b, 0x0b, 0x75, 0xd7, 0xb6, 0xd9, + 0x53, 0x01, 0x4f, 0xd1, 0x0f, 0x58, 0x71, 0xfb, 0x60, 0xc2, 0x3c, 0x99, 0x52, 0x67, 0x55, 0x8d, + 0x23, 0xb7, 0x43, 0x33, 0x5f, 0xbd, 0xa5, 0x1f, 0x9b, 0xee, 0x73, 0x40, 0x76, 0x20, 0xed, 0x17, + 0x0a, 0x12, 0xd4, 0xfd, 0xf8, 0x07, 0xff, 0xaf, 0x7f, 0xa9, 0xe8, 0x40, 0xf9, 0x22, 0x20, 0xa9, + 0x68, 0x64, 0x89, 0x06, 0x66, 0x72, 0x40, 0xaf, 0x11, 0x85, 0xaa, 0xdd, 0x5c, 0x23, 0xa1, 0xcf, + 0x5e, 0x48, 0xe2, 0x08, 0x8a, 0x43, 0xc5, 0xfe, 0xc8, 0x12, 0x35, 0x4c, 0x44, 0xf6, 0x73, 0x0b, + 0x93, 0x2c, 0x98, 0x65, 0xc1, 0xab, 0x65, 0x16, 0xd1, 0x85, 0x4a, 0x9f, 0xd4, 0xc8, 0xba, 0xf1, + 0x5c, 0x16, 0x4d, 0xcc, 0x75, 0xe6, 0x13, 0xeb, 0x28, 0x9d, 0xd2, 0x83, 0xea, 0x70, 0x09, 0x2d, + 0x52, 0xd2, 0x91, 0x4a, 0x8a, 0x3d, 0xdc, 0xa8, 0x75, 0x33, 0x2f, 0xa1, 0xf1, 0xe0, 0x3b, 0xab, + 0xe4, 0x79, 0x8b, 0x7d, 0xdc, 0x46, 0xba, 0x2e, 0x3d, 0xa8, 0xf6, 0xb7, 0x52, 0xf4, 0xff, 0x4e, + 0xb1, 0x85, 0xd3, 0x53, 0x74, 0xa1, 0x74, 0x2f, 0xa3, 0x70, 0x79, 0xb7, 0x75, 0x4c, 0x95, 0x6e, + 0xe6, 0x09, 0x94, 0x07, 0x14, 0x46, 0xb3, 0x64, 0x21, 0x99, 0xd4, 0xb9, 0x4e, 0x61, 0xe7, 0x96, + 0xa5, 0x33, 0xf4, 0xa4, 0x1f, 0x3e, 0xb1, 0x12, 0xbb, 0xb8, 0x5e, 0xea, 0xdc, 0x67, 0x50, 0x4b, + 0xf6, 0x9c, 0xfb, 0x5b, 0xb8, 0x09, 0xf4, 0x3b, 0x2e, 0xdd, 0xb9, 0xee, 0x94, 0xa5, 0x23, 0xea, + 0x98, 0x2a, 0x8d, 0xe7, 0xfc, 0x70, 0xbe, 0x30, 0x0b, 0x5f, 0x0b, 0xb3, 0xf0, 0x1e, 0x9b, 0xc6, + 0x3c, 0x36, 0x8d, 0xcf, 0xd8, 0x34, 0xbe, 0x63, 0xd3, 0x78, 0xac, 0xe4, 0xaf, 0x7c, 0x5c, 0x5c, + 0x99, 0x8e, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xfb, 0xe9, 0x16, 0xe0, 0xf9, 0x02, 0x00, 0x00, } type FirecrackerService interface { @@ -54,6 +58,11 @@ type FirecrackerService interface { SetVMMetadata(ctx context.Context, req *proto1.SetVMMetadataRequest) (*empty.Empty, error) UpdateVMMetadata(ctx context.Context, req *proto1.UpdateVMMetadataRequest) (*empty.Empty, error) GetVMMetadata(ctx context.Context, req *proto1.GetVMMetadataRequest) (*proto1.GetVMMetadataResponse, error) + PauseVM(ctx context.Context, req *proto1.PauseVMRequest) (*empty.Empty, error) + ResumeVM(ctx context.Context, req *proto1.ResumeVMRequest) (*empty.Empty, error) + LoadSnapshot(ctx context.Context, req *proto1.LoadSnapshotRequest) (*empty.Empty, error) + CreateSnapshot(ctx context.Context, req *proto1.CreateSnapshotRequest) (*empty.Empty, error) + Offload(ctx context.Context, req *proto1.OffloadRequest) (*empty.Empty, error) } func RegisterFirecrackerService(srv *github_com_containerd_ttrpc.Server, svc FirecrackerService) { @@ -100,6 +109,41 @@ func RegisterFirecrackerService(srv *github_com_containerd_ttrpc.Server, svc Fir } return svc.GetVMMetadata(ctx, &req) }, + "PauseVM": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.PauseVMRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.PauseVM(ctx, &req) + }, + "ResumeVM": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.ResumeVMRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.ResumeVM(ctx, &req) + }, + "LoadSnapshot": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.LoadSnapshotRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.LoadSnapshot(ctx, &req) + }, + "CreateSnapshot": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.CreateSnapshotRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.CreateSnapshot(ctx, &req) + }, + "Offload": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.OffloadRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.Offload(ctx, &req) + }, }) } @@ -160,3 +204,43 @@ func (c *firecrackerClient) GetVMMetadata(ctx context.Context, req *proto1.GetVM } return &resp, nil } + +func (c *firecrackerClient) PauseVM(ctx context.Context, req *proto1.PauseVMRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "PauseVM", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) ResumeVM(ctx context.Context, req *proto1.ResumeVMRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "ResumeVM", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) LoadSnapshot(ctx context.Context, req *proto1.LoadSnapshotRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "LoadSnapshot", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) CreateSnapshot(ctx context.Context, req *proto1.CreateSnapshotRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "CreateSnapshot", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) Offload(ctx context.Context, req *proto1.OffloadRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "Offload", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} diff --git a/runtime/service.go b/runtime/service.go index 2b45809b5..b7c456696 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -14,16 +14,20 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" "math" "net" + "net/http" "os" + "os/exec" "runtime/debug" "strconv" "strings" "sync" + "syscall" "time" // disable gosec check for math/rand. We just need a random starting @@ -61,6 +65,8 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/proto" drivemount "github.com/firecracker-microvm/firecracker-containerd/proto/service/drivemount/ttrpc" fccontrolTtrpc "github.com/firecracker-microvm/firecracker-containerd/proto/service/fccontrol/ttrpc" + + "github.com/tv42/httpunix" ) func init() { @@ -144,6 +150,10 @@ type service struct { machineConfig *firecracker.Config vsockIOPortCount uint32 vsockPortMu sync.Mutex + + // httpControlClient is to send pause/resume/snapshot commands to the microVM + httpControlClient *http.Client + firecrackerPid int } func shimOpts(shimCtx context.Context) (*shim.Opts, error) { @@ -478,7 +488,10 @@ func (s *service) CreateVM(requestCtx context.Context, request *proto.CreateVMRe s.logger.WithError(err).Error("failed to publish start VM event") } - go s.monitorVMExit() + // Commented out because its execution cancels the shim, and + // it would get executed on Offload if we leave it, killing the shim, + // and making snapshots impossible. + //go s.monitorVMExit() // let all the other methods know that the VM is ready for tasks close(s.vmReady) @@ -579,7 +592,18 @@ func (s *service) createVM(requestCtx context.Context, request *proto.CreateVMRe return err } + s.createHTTPControlClient() + + pid, err := s.machine.PID() + if err != nil { + s.logger.WithError(err).Error("Failed to get PID of firecracker process") + return err + } + + s.firecrackerPid = pid + s.logger.Info("successfully started the VM") + return nil } @@ -711,6 +735,126 @@ func (s *service) GetVMMetadata(requestCtx context.Context, request *proto.GetVM return &proto.GetVMMetadataResponse{Metadata: string(metadata)}, nil } +// PauseVM Pauses a VM +func (s *service) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { + pauseReq, err := formPauseReq() + if err != nil { + s.logger.WithError(err).Error("Failed to create pause vm request") + return nil, err + } + + err = s.waitVMReady() + if err != nil { + return nil, err + } + + resp, err := s.httpControlClient.Do(pauseReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send pause VM request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to pause VM") + return nil, err + } + + return &empty.Empty{}, nil +} + +// ResumeVM Resumes a VM +func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empty.Empty, error) { + resumeReq, err := formResumeReq() + if err != nil { + s.logger.WithError(err).Error("Failed to create resume vm request") + return nil, err + } + + resp, err := s.httpControlClient.Do(resumeReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send resume VM request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to resume VM") + return nil, err + } + return &empty.Empty{}, nil +} + +// LoadSnapshot Loads a VM from a snapshot +func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + if err := s.startFirecrackerProcess(); err != nil { + s.logger.WithError(err).Error("startFirecrackerProcess returned an error") + return nil, err + } + + if err := s.dialFirecrackerSocket(); err != nil { + s.logger.WithError(err).Error("Failed to wait for firecracker socket") + } + s.createHTTPControlClient() + + loadSnapReq, err := formLoadSnapReq(req.SnapshotFilePath, req.MemFilePath) + if err != nil { + s.logger.WithError(err).Error("Failed to create load snapshot request") + return nil, err + } + + resp, err := s.httpControlClient.Do(loadSnapReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send load snapshot request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to load VM from snapshot") + s.logger.WithError(err).Errorf("Status of request: %s", resp.Status) + return nil, err + } + + return &empty.Empty{}, nil +} + +// CreateSnapshot Creates a snapshot of a VM +func (s *service) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*empty.Empty, error) { + createSnapReq, err := formCreateSnapReq(req.SnapshotFilePath, req.MemFilePath) + if err != nil { + s.logger.WithError(err).Error("Failed to create make snapshot request") + return nil, err + } + + resp, err := s.httpControlClient.Do(createSnapReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send make snapshot request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to make snapshot of VM") + return nil, err + } + + return &empty.Empty{}, nil +} + +// Offload Shuts down a VM and deletes the corresponding firecracker socket +// and vsock. All of the other resources will persist +func (s *service) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { + if err := syscall.Kill(s.firecrackerPid, syscall.SIGKILL); err != nil { + s.logger.WithError(err).Error("Failed to kill firecracker process") + return nil, err + } + + if err := os.RemoveAll(s.shimDir.FirecrackerSockPath()); err != nil { + s.logger.WithError(err).Error("Failed to delete firecracker socket") + return nil, err + } + + if err := os.RemoveAll(s.shimDir.FirecrackerVSockPath()); err != nil { + s.logger.WithError(err).Error("Failed to delete firecracker vsock") + return nil, err + } + + return &empty.Empty{}, nil +} + func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker.Config, error) { for _, driveMount := range req.DriveMounts { // Verify the request specified an absolute path for the source/dest of drives. @@ -730,17 +874,23 @@ func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker return nil, errors.Wrapf(err, "failed to get relative path to firecracker vsock") } + // TODO: Remove hardcoding and make a parameter + if _, err := os.OpenFile(s.shimDir.LogStartPath(), os.O_RDONLY|os.O_CREATE, 0600); err != nil { + s.logger.WithError(err).Errorf("Failed to create %s", s.shimDir.LogStartPath()) + return nil, err + } + cfg := firecracker.Config{ SocketPath: relSockPath, VsockDevices: []firecracker.VsockDevice{{ Path: relVSockPath, ID: "agent_api", }}, - LogFifo: s.shimDir.FirecrackerLogFifoPath(), - MetricsFifo: s.shimDir.FirecrackerMetricsFifoPath(), - MachineCfg: machineConfigurationFromProto(s.config, req.MachineCfg), - LogLevel: s.config.DebugHelper.GetFirecrackerLogLevel(), - VMID: s.vmID, + // Put LogPath insteadof LogFifo here to comply with the new Firecracker logging + LogPath: s.shimDir.LogStartPath(), + MachineCfg: machineConfigurationFromProto(s.config, req.MachineCfg), + LogLevel: s.config.DebugHelper.GetFirecrackerLogLevel(), + VMID: s.vmID, } if req.JailerConfig != nil { @@ -1347,6 +1497,8 @@ func (s *service) cleanup() error { } // monitorVMExit watches the VM and cleanup resources when it terminates. +// Comment out because unused +/* func (s *service) monitorVMExit() { // Block until the VM exits if err := s.machine.Wait(s.shimCtx); err != nil && err != context.Canceled { @@ -1357,3 +1509,181 @@ func (s *service) monitorVMExit() { s.logger.WithError(err).Error("failed to clean up the VM") } } +*/ + +func (s *service) createHTTPControlClient() { + u := &httpunix.Transport{ + DialTimeout: 100 * time.Millisecond, + RequestTimeout: 10 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + } + u.RegisterLocation("firecracker", s.shimDir.FirecrackerSockPath()) + + t := &http.Transport{} + t.RegisterProtocol(httpunix.Scheme, u) + + var client = http.Client{ + Transport: t, + } + + s.httpControlClient = &client +} + +func formResumeReq() (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "state": "Resumed", + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PATCH", "http+unix://firecracker/vm", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formResumeReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func formPauseReq() (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "state": "Paused", + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PATCH", "http+unix://firecracker/vm", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formPauseReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func formLoadSnapReq(snapshotPath, memPath string) (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "snapshot_path": snapshotPath, + "mem_file_path": memPath, + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PUT", "http+unix://firecracker/snapshot/load", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formLoadSnapReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func formCreateSnapReq(snapshotPath, memPath string) (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "snapshot_type": "Full", + "snapshot_path": snapshotPath, + "mem_file_path": memPath, + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PUT", "http+unix://firecracker/snapshot/create", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formCreateSnapReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func (s *service) startFirecrackerProcess() error { + firecPath, err := exec.LookPath("firecracker") + if err != nil { + logrus.WithError(err).Error("failed to look up firecracker binary") + return err + } + + logFilePath := s.shimDir.LogLoadPath() + if err := os.RemoveAll(logFilePath); err != nil { + s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) + return err + } + if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0600); err != nil { + s.logger.WithError(err).Errorf("Failed to create %s", logFilePath) + return err + } + + args := []string{ + "--api-sock", s.shimDir.FirecrackerSockPath(), + "--log-path", logFilePath, + "--level", s.config.DebugHelper.GetFirecrackerLogLevel(), + "--show-level", + "--show-log-origin", + } + + firecrackerCmd := exec.Command(firecPath, args...) + firecrackerCmd.Dir = s.shimDir.RootPath() + + if err := firecrackerCmd.Start(); err != nil { + logrus.WithError(err).Error("Failed to start firecracker process") + } + + go firecrackerCmd.Wait() + + s.firecrackerPid = firecrackerCmd.Process.Pid + + return nil +} + +func (s *service) dialFirecrackerSocket() error { + for { + var d net.Dialer + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + c, err := d.DialContext(ctx, "unix", s.shimDir.FirecrackerSockPath()) + if err != nil { + if ctx.Err() != nil { + s.logger.WithError(ctx.Err()).Error("timed out while waiting for firecracker socket") + return ctx.Err() + } + + time.Sleep(1 * time.Millisecond) + continue + } + + c.Close() + + break + } + + return nil +} diff --git a/runtime/service_test.go b/runtime/service_test.go index 7cc842a1e..b10417dd3 100644 --- a/runtime/service_test.go +++ b/runtime/service_test.go @@ -253,8 +253,14 @@ func TestBuildVMConfiguration(t *testing.T) { Path: relVSockPath, ID: "agent_api", }} - tc.expectedCfg.LogFifo = svc.shimDir.FirecrackerLogFifoPath() - tc.expectedCfg.MetricsFifo = svc.shimDir.FirecrackerMetricsFifoPath() + + // Remove LogFifo and MetricsInfo in order to comply + // with new Firecracker logging. + // Include hard coded LogPath + // TODO: FIX TEST WHEN LogPath is no longer hardcoded + //tc.expectedCfg.LogFifo = svc.shimDir.FirecrackerLogFifoPath() + //tc.expectedCfg.MetricsFifo = svc.shimDir.FirecrackerMetricsFifoPath() + tc.expectedCfg.LogPath = svc.shimDir.LogStartPath() drives := make([]models.Drive, tc.expectedStubDriveCount) for i := 0; i < tc.expectedStubDriveCount; i++ {