vz: implement auto save/restore

cmd/limactl/main.go

@@ -155,6 +155,7 @@ func newApp() *cobra.Command {
 		newProtectCommand(),
 		newUnprotectCommand(),
 		newTunnelCommand(),
+		newSaveCommand(),
 	)
 	if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
 		rootCmd.AddCommand(startAtLoginCommand())

cmd/limactl/save.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"github.com/lima-vm/lima/pkg/instance"
+	networks "github.com/lima-vm/lima/pkg/networks/reconcile"
+	"github.com/lima-vm/lima/pkg/store"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func newSaveCommand() *cobra.Command {
+	saveCmd := &cobra.Command{
+		Use:   "save INSTANCE",
+		Short: "Save an instance",
+		PersistentPreRun: func(*cobra.Command, []string) {
+			logrus.Warn("`limactl save` is experimental")
+		},
+		Args:              WrapArgsError(cobra.MaximumNArgs(1)),
+		RunE:              saveAction,
+		ValidArgsFunction: saveBashComplete,
+		GroupID:           basicCommand,
+	}
+
+	return saveCmd
+}
+
+func saveAction(cmd *cobra.Command, args []string) error {
+	instName := DefaultInstanceName
+	if len(args) > 0 {
+		instName = args[0]
+	}
+
+	inst, err := store.Inspect(instName)
+	if err != nil {
+		return err
+	}
+
+	err = instance.StopGracefully(inst, true)
+	// TODO: should we also reconcile networks if graceful save returned an error?
+	if err == nil {
+		err = networks.Reconcile(cmd.Context(), "")
+	}
+	return err
+}
+
+func saveBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {
+	return bashCompleteInstanceNames(cmd)
+}

cmd/limactl/stop.go

@@ -39,7 +39,7 @@ func stopAction(cmd *cobra.Command, args []string) error {
 	if force {
 		instance.StopForcibly(inst)
 	} else {
-		err = instance.StopGracefully(inst)
+		err = instance.StopGracefully(inst, false)
 	}
 	// TODO: should we also reconcile networks if graceful stop returned an error?
 	if err == nil {

pkg/driver/driver.go

@@ -68,6 +68,9 @@ type Driver interface {
 
 	// GuestAgentConn returns the guest agent connection, or nil (if forwarded by ssh).
 	GuestAgentConn(_ context.Context) (net.Conn, error)
+
+	// RuntimeConfig accepts config containing changes to the runtime configuration, and returns the updated runtime configuration.
+	RuntimeConfig(_ context.Context, config interface{}) (interface{}, error)
 }
 
 type BaseDriver struct {
@@ -149,3 +152,7 @@ func (d *BaseDriver) GuestAgentConn(_ context.Context) (net.Conn, error) {
 	// use the unix socket forwarded by host agent
 	return nil, nil
 }
+
+func (d *BaseDriver) RuntimeConfig(_ context.Context, _ interface{}) (interface{}, error) {
+	return nil, fmt.Errorf("unimplemented")
+}

pkg/hostagent/api/client/client.go

@@ -4,9 +4,11 @@ package client
 // Apache License 2.0
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 
 	"github.com/lima-vm/lima/pkg/hostagent/api"
@@ -16,6 +18,7 @@ import (
 type HostAgentClient interface {
 	HTTPClient() *http.Client
 	Info(context.Context) (*api.Info, error)
+	DriverConfig(context.Context, interface{}) (interface{}, error)
 }
 
 // NewHostAgentClient creates a client.
@@ -62,3 +65,35 @@ func (c *client) Info(ctx context.Context) (*api.Info, error) {
 	}
 	return &info, nil
 }
+
+func (c *client) DriverConfig(ctx context.Context, config interface{}) (interface{}, error) {
+	u := fmt.Sprintf("http://%s/%s/driver/config", c.dummyHost, c.version)
+	method := "GET"
+	var body io.Reader
+	if config != nil {
+		method = "PATCH"
+		b, err := json.Marshal(config)
+		if err != nil {
+			return nil, err
+		}
+		body = bytes.NewBuffer(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, u, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if err := httpclientutil.Successful(resp); err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&config); err != nil {
+		return nil, err
+	}
+	return &config, nil
+}

pkg/hostagent/api/server/server.go

@@ -50,6 +50,36 @@ func (b *Backend) GetInfo(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write(m)
 }
 
+// DriverConfig is the handler for GET /v1/driver/config and PATCH /v1/driver/config.
+func (b *Backend) DriverConfig(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var config interface{}
+	if r.Method == http.MethodPatch {
+		defer r.Body.Close()
+		if err := json.NewDecoder(r.Body).Decode(&config); err != nil {
+			b.onError(w, err, http.StatusBadRequest)
+			return
+		}
+	}
+	config, err := b.Agent.DriverRuntimeConfig(ctx, config)
+	if err != nil {
+		b.onError(w, err, http.StatusInternalServerError)
+		return
+	}
+	m, err := json.Marshal(config)
+	if err != nil {
+		b.onError(w, err, http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write(m)
+}
+
 func AddRoutes(r *http.ServeMux, b *Backend) {
 	r.Handle("/v1/info", http.HandlerFunc(b.GetInfo))
+	r.Handle("GET /v1/driver/config", http.HandlerFunc(b.DriverConfig))
+	r.Handle("PATCH /v1/driver/config", http.HandlerFunc(b.DriverConfig))
 }

pkg/hostagent/hostagent.go

@@ -423,6 +423,10 @@ func (a *HostAgent) Info(_ context.Context) (*hostagentapi.Info, error) {
 	return info, nil
 }
 
+func (a *HostAgent) DriverRuntimeConfig(ctx context.Context, config interface{}) (interface{}, error) {
+	return a.driver.RuntimeConfig(ctx, config)
+}
+
 func (a *HostAgent) startHostAgentRoutines(ctx context.Context) error {
 	if *a.instConfig.Plain {
 		logrus.Info("Running in plain mode. Mounts, port forwarding, containerd, etc. will be ignored. Guest agent will not be running.")

pkg/instance/stop.go

@@ -9,18 +9,42 @@ import (
 	"strings"
 	"time"
 
+	hostagentclient "github.com/lima-vm/lima/pkg/hostagent/api/client"
 	hostagentevents "github.com/lima-vm/lima/pkg/hostagent/events"
+	"github.com/lima-vm/lima/pkg/limayaml"
 	"github.com/lima-vm/lima/pkg/osutil"
 	"github.com/lima-vm/lima/pkg/store"
 	"github.com/lima-vm/lima/pkg/store/filenames"
 	"github.com/sirupsen/logrus"
 )
 
-func StopGracefully(inst *store.Instance) error {
+func StopGracefully(inst *store.Instance, saveOnStop bool) error {
 	if inst.Status != store.StatusRunning {
 		return fmt.Errorf("expected status %q, got %q (maybe use `limactl stop -f`?)", store.StatusRunning, inst.Status)
 	}
 
+	if saveOnStop && inst.Saved {
+		logrus.Warn("saved VZ machine state is found. It will be overwritten by the new one.")
+	}
+
+	if inst.VMType == limayaml.VZ {
+		haSock := filepath.Join(inst.Dir, filenames.HostAgentSock)
+		haClient, err := hostagentclient.NewHostAgentClient(haSock)
+		if err != nil {
+			logrus.WithError(err).Error("Failed to create a host agent client")
+		}
+		ctx, cancel := context.WithTimeout(context.TODO(), 3*time.Second)
+		defer cancel()
+		disableSaveOnStopConfig := struct {
+			SaveOnStop bool `json:"saveOnStop"`
+		}{SaveOnStop: saveOnStop}
+		_, err = haClient.DriverConfig(ctx, disableSaveOnStopConfig)
+		if err != nil {
+			return fmt.Errorf("failed to disable saveOnStop: %w", err)
+		}
+	} else if saveOnStop {
+		return fmt.Errorf("save is not supported for %q", inst.VMType)
+	}
 	begin := time.Now() // used for logrus propagation
 	logrus.Infof("Sending SIGINT to hostagent process %d", inst.HostAgentPID)
 	if err := osutil.SysKill(inst.HostAgentPID, osutil.SigInt); err != nil {

pkg/limayaml/defaults.go

@@ -855,6 +855,32 @@ func FillDefault(y, d, o *LimaYAML, filePath string, warn bool) {
 		y.NestedVirtualization = ptr.Of(false)
 	}
 
+	if y.SaveOnStop == nil {
+		y.SaveOnStop = d.SaveOnStop
+	}
+	if o.SaveOnStop != nil {
+		y.SaveOnStop = o.SaveOnStop
+	}
+	if y.SaveOnStop == nil {
+		y.SaveOnStop = ptr.Of(false)
+	}
+	if *y.SaveOnStop {
+		y.SaveOnStop = ptr.Of(false)
+		if *y.VMType != VZ {
+			logrus.Warn("saveOnStop is only supported for VM type vz")
+		} else if runtime.GOARCH != "arm64" {
+			logrus.Warn("saveOnStop is only supported for arm64 VM type vz")
+		} else if runtime.GOOS != "darwin" {
+			logrus.Warn("saveOnStop is only supported on macOS")
+		} else if macOSProductVersion, err := osutil.ProductVersion(); err != nil {
+			logrus.WithError(err).Warn("Failed to get macOS product version")
+		} else if macOSProductVersion.LessThan(*semver.New("14.0.0")) {
+			logrus.Warn("saveOnStop is not supported on macOS prior to 14.0")
+		} else {
+			y.SaveOnStop = ptr.Of(true)
+		}
+	}
+
 	if y.Plain == nil {
 		y.Plain = d.Plain
 	}
@@ -878,6 +904,7 @@ func fixUpForPlainMode(y *LimaYAML) {
 	y.Containerd.User = ptr.Of(false)
 	y.Rosetta.BinFmt = ptr.Of(false)
 	y.Rosetta.Enabled = ptr.Of(false)
+	y.SaveOnStop = ptr.Of(false)
 	y.TimeZone = ptr.Of("")
 }
 

pkg/limayaml/defaults_test.go

@@ -112,6 +112,7 @@ func TestFillDefault(t *testing.T) {
 			RemoveDefaults: ptr.Of(false),
 		},
 		NestedVirtualization: ptr.Of(false),
+		SaveOnStop:           ptr.Of(false),
 		Plain:                ptr.Of(false),
 		User: User{
 			Name:    ptr.Of(user.Username),
@@ -437,6 +438,7 @@ func TestFillDefault(t *testing.T) {
 			BinFmt:  ptr.Of(true),
 		},
 		NestedVirtualization: ptr.Of(true),
+		SaveOnStop:           ptr.Of(true),
 		User: User{
 			Name:    ptr.Of("xxx"),
 			Comment: ptr.Of("Foo Bar"),
@@ -474,11 +476,13 @@ func TestFillDefault(t *testing.T) {
 			Enabled: ptr.Of(true),
 			BinFmt:  ptr.Of(true),
 		}
+		expect.SaveOnStop = ptr.Of(true)
 	} else {
 		expect.Rosetta = Rosetta{
 			Enabled: ptr.Of(false),
 			BinFmt:  ptr.Of(true),
 		}
+		expect.SaveOnStop = ptr.Of(false)
 	}
 	expect.Plain = ptr.Of(false)
 
@@ -660,6 +664,7 @@ func TestFillDefault(t *testing.T) {
 			BinFmt:  ptr.Of(false),
 		},
 		NestedVirtualization: ptr.Of(false),
+		SaveOnStop:           ptr.Of(false),
 		User: User{
 			Name:    ptr.Of("foo"),
 			Comment: ptr.Of("foo bar baz"),
@@ -723,6 +728,7 @@ func TestFillDefault(t *testing.T) {
 	expect.Plain = ptr.Of(false)
 
 	expect.NestedVirtualization = ptr.Of(false)
+	expect.SaveOnStop = ptr.Of(false)
 
 	FillDefault(&y, &d, &o, filePath, false)
 	assert.DeepEqual(t, &y, &expect, opts...)

pkg/limayaml/limayaml.go

@@ -48,6 +48,7 @@ type LimaYAML struct {
 	TimeZone             *string        `yaml:"timezone,omitempty" json:"timezone,omitempty" jsonschema:"nullable"`
 	NestedVirtualization *bool          `yaml:"nestedVirtualization,omitempty" json:"nestedVirtualization,omitempty" jsonschema:"nullable"`
 	User                 User           `yaml:"user,omitempty" json:"user,omitempty"`
+	SaveOnStop           *bool          `yaml:"saveOnStop,omitempty" json:"saveOnStop,omitempty" jsonschema:"nullable"`
 }
 
 type (

pkg/limayaml/validate.go

@@ -523,4 +523,7 @@ func warnExperimental(y *LimaYAML) {
 	if y.MountInotify != nil && *y.MountInotify {
 		logrus.Warn("`mountInotify` is experimental")
 	}
+	if y.SaveOnStop != nil && *y.SaveOnStop {
+		logrus.Warn("`saveOnStop` is experimental")
+	}
 }

pkg/store/filenames/filenames.go

@@ -56,6 +56,7 @@ const (
 	HostAgentStderrLog   = "ha.stderr.log"
 	VzIdentifier         = "vz-identifier"
 	VzEfi                = "vz-efi"           // efi variable store
+	VzMachineState       = "vz-machine-state" // machine state file
 	QemuEfiCodeFD        = "qemu-efi-code.fd" // efi code; not always created
 	AnsibleInventoryYAML = "ansible-inventory.yaml"
 

pkg/store/instance.go

@@ -43,6 +43,7 @@ type Instance struct {
 	// Hostname, not HostName (corresponds to SSH's naming convention)
 	Hostname        string             `json:"hostname"`
 	Status          Status             `json:"status"`
+	Saved           bool               `json:"saved"`
 	Dir             string             `json:"dir"`
 	VMType          limayaml.VMType    `json:"vmType"`
 	Arch            limayaml.Arch      `json:"arch"`
@@ -147,6 +148,14 @@ func Inspect(instName string) (*Instance, error) {
 	}
 
 	inspectStatus(instDir, inst, y)
+	_, err = os.Stat(filepath.Join(instDir, filenames.VzMachineState))
+	if err == nil {
+		inst.Saved = true
+	} else if errors.Is(err, os.ErrNotExist) {
+		inst.Saved = false
+	} else {
+		inst.Errors = append(inst.Errors, fmt.Errorf("cannot determine whether the instance is saved: %w", err))
+	}
 
 	tmpl, err := template.New("format").Parse(y.Message)
 	if err != nil {

pkg/vz/save_restore.go

@@ -0,0 +1,18 @@
+//go:build darwin && !arm64 && !no_vz
+
+package vz
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/Code-Hex/vz/v3"
+)
+
+func saveVM(vm *vz.VirtualMachine, machineStatePath string) error {
+	return fmt.Errorf("save is not supported on the vz driver for the architecture %s", runtime.GOARCH)
+}
+
+func restoreVM(vm *vz.VirtualMachine, machineStatePath string) error {
+	return fmt.Errorf("restore is not supported on the vz driver for the architecture %s", runtime.GOARCH)
+}