dxvm: rosetta, silent boot and other enhancements

A bunch of new miscellaneous new features and improvements:

- Rosetta is enabled on `aarch64-darwin`. This allows VMs to
  transparently run aarch64 and x86_64 Linux binaries simultaneously.
	- Install x86_64 binaries with Nix using the `--system` flag.
	  For example, `nix profile install --system x86_64 <pkg>`.
- The current Devbox project directory is now automatically detected and
  mounted. It can be found in `~/devbox` in the VM.
- Devbox is automatically installed when the VM is created using the
  `github:jetpack-io/devbox?ref=gcurtis/flake` flake.
- Starting a VM is now completely silent. The kernel's console is sent
  to `.devbox/vm/console` instead of stdout so that the first thing the
  user sees is their own shell prompt.
	- This work really well when launching a paused VM. If the VM
	  resumes fast enough, it feels like using a normal shell.
- Bootstrapping from an NixOS installer ISO with the `-install` flag is
  now fully automated. This is done by reading/writing to the VM's
  console using a pipe (similar to a program like `expect`).
- Fixed a bug where stopping the VM would always return an error.

Author: gcurtis

.gitignore

@@ -1,2 +1,3 @@
 # ignore files by Jetbrains IDEs
 *.idea
+bin/

pkg/sandbox/vm/README.md

@@ -7,20 +7,16 @@ Experimental support for Devbox virtual machines on macOS.
 The `dxvm` command acts like `devbox shell` except that it launches the Devbox
 environment in a VM.
 
-To create a new VM, run the following:
+To create a new VM, run with the `-install` flag inside a Devbox project
+directory:
 
 	cd ~/my/project
 	dxvm -install
-	# Wait for the prompt. This might appear to hang the first time it's run
-	# while downloading the NixOS installer.
 
-	mkdir bootstrap
-	sudo mount -t virtiofs bootstrap bootstrap
-	sudo bootstrap/install.sh
-	sudo shutdown now
-	# ^C to exit
+The VM will start, install NixOS, and then reboot into a shell. This step might
+appear to hang at times while it downloads NixOS.
 
-Now that the VM is bootstrapped, you can launch it any time with:
+After the VM is created, you no longer need the `-install` flag:
 
 	cd ~/my/project
 	dxvm
@@ -33,6 +29,7 @@ The first time `dxvm` is run in a Devbox project, it creates a `.devbox/vm`
 directory that contains the VM's state and configuration files:
 
 - `log` - error and debug log messages
+- `console` - the Linux kernel console output
 - `disk.img` - main disk image, typically mounted as root
 - `id` - an opaque Virtualization Framework machine ID
 
@@ -42,6 +39,13 @@ VM's resources:
 - `mem` - the amount of memory (in bytes) to allocate to the VM
 - `cpu` - the number of CPUs to allocate to the VM
 
+There are two directories shared between the host and guest machines:
+
+- `boot -> /boot` - gives the host access to the NixOS kernel and initrd so it
+  can create a bootloader
+- `bootstrap -> ~/bootstrap` - contains a script for bootstrapping a new VM from
+  a vanilla NixOS installer ISO (only mounted with `-install`)
+
 ## Building
 
 This package uses the macOS Virtualization Framework, and therefore needs CGO.
@@ -55,16 +59,16 @@ To compile and sign `dxvm` run:
 
 	devbox run build
 
+It's okay if it prints a couple of warnings about duplicate libraries and
+replacing the code-signing signature.
+
 The `devbox run build` script uses `./cmd/dxvmsign` to sign the Go binary, which
 allows it to use the Virtualization Framework. It's a small wrapper around
 Apple's `codesign` utility.
 
 ## Limitations
 
-- Mounting the Devbox project directory was temporarily removed while cleaning
-things up. Needs to be brought back.
-- Only aarch64-linux is implemented right now. Other systems have been tested,
-but they aren't an option in the dxvm command.
+- Intel macOS hasn't been tested yet.
 - Using ctrl-c to exit has the unfortunate side-effect of making it impossible
 to interrupt a program in the VM.
 - The host terminal has no way of telling the guest when it has resized (usually

pkg/sandbox/vm/bin/dxvm

@@ -20,15 +20,29 @@
   };
 
   environment = {
-    # defaultPackages = [ ];
-    systemPackages = with pkgs; [ curl vim ];
+    defaultPackages = [ ];
+    systemPackages = with pkgs; [
+      curl
+      git
+      vim
+      ((builtins.getFlake "github:jetpack-io/devbox?ref=gcurtis/flake").packages."{{.System}}".default)
+    ];
   };
 
   fileSystems = {
     "/" = {
       device = "/dev/vda";
       fsType = "ext4";
     };
+    "/boot" = {
+      device = "boot";
+      fsType = "virtiofs";
+    };
+    "/home/{{.User.Username}}/devbox" = {
+      device = "home";
+      fsType = "virtiofs";
+      options = [ "nofail" ];
+    };
   };
 
   nix = {
@@ -46,14 +60,18 @@
     hostPlatform = lib.mkDefault "{{.System}}";
   };
 
+  programs.bash.promptInit = "PS1='dxvm\$ '";
+
   security.sudo = {
     extraConfig = "Defaults lecture = never";
     wheelNeedsPassword = false;
   };
 
   services.getty = {
     autologinUser = "{{.User.Username}}";
-    greetingLine = "";
+    greetingLine = lib.mkForce "";
+    helpLine = lib.mkForce "";
+    extraArgs = [ "--skip-login" "--nohostname" "--noissue" "--noclear" "--nonewline" "--8bits" ];
   };
 
   system.stateVersion = "23.05";
@@ -71,4 +89,6 @@
       extraGroups = [ "wheel" ];
     };
   };
+
+  virtualisation.rosetta.enable = {{.Rosetta}};
 }

pkg/sandbox/vm/bootstrap/configuration.nix

@@ -7,4 +7,4 @@ mount -t virtiofs boot /mnt/boot
 cat << 'EOF' > /mnt/etc/nixos/configuration.nix
 {{ template "configuration.nix" . -}}
 EOF
-nixos-install --no-root-password --show-trace --root /mnt
+NIX_CONFIG="experimental-features = nix-command flakes" nixos-install --no-root-password --show-trace --root /mnt

pkg/sandbox/vm/bootstrap/install.sh

@@ -1,11 +1,16 @@
 package main
 
 import (
+	"cmp"
 	"context"
 	"flag"
+	"fmt"
+	"io/fs"
 	"log/slog"
 	"os"
 	"os/signal"
+	"path/filepath"
+	"slices"
 	"time"
 
 	"go.jetpack.io/pkg/sandbox/vm"
@@ -15,25 +20,80 @@ func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer cancel()
 
-	vm := vm.VM{}
-	flag.StringVar(&vm.HostDataDir, "datadir", ".devbox/vm", "`path` to the directory for saving VM state")
-	flag.BoolVar(&vm.Install, "install", false, "mount NixOS install image")
+	dataDir := "./dxvm"
+	devboxDir, devboxDirFound, err := findDevboxDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "no devbox.json found, using %s for state: %v\n", dataDir, err)
+	} else if !devboxDirFound {
+		fmt.Fprintf(os.Stderr, "no devbox.json found, using %s for state: searched up to %s\n", dataDir, devboxDir)
+	} else {
+		dataDir = filepath.Join(devboxDir, ".devbox", "vm")
+	}
+
+	dxvm := vm.VM{}
+	flag.StringVar(&dxvm.HostDataDir, "datadir", dataDir, "`path` to the directory for saving VM state")
+	flag.BoolVar(&dxvm.Install, "install", false, "mount NixOS install image")
 	flag.Parse()
 
-	if vm.Install {
+	if dxvm.Install {
 		slog.Debug("downloading the NixOS installer, this make take a few minutes")
+	} else if devboxDirFound {
+		dxvm.SharedDirectories = append(dxvm.SharedDirectories, vm.SharedDirectory{
+			Path:     devboxDir,
+			HomeDir:  true,
+			ReadOnly: false,
+		})
+		fmt.Fprintln(os.Stderr, "booting virtual machine")
 	}
 
-	err := vm.Start(ctx)
-	if err != nil {
-		slog.Error("start virtual machine", "err", err)
+	go func() {
+		<-ctx.Done()
+		ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		if err := dxvm.Stop(ctx); err != nil {
+			slog.Error("stop virtual machine", "err", err)
+		}
+	}()
+	if err := dxvm.Run(ctx); err != nil {
+		slog.Error("run virtual machine install", "err", err)
 		os.Exit(1)
 	}
 
-	<-ctx.Done()
-	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	if err := vm.Stop(ctx); err != nil {
-		slog.Error("stop virtual machine", "err", err)
+	// Restart if we just finished bootrapping a new VM.
+	if dxvm.Install {
+		fmt.Fprintln(os.Stderr, "virtual machine created successfully")
+		dxvm.Install = false
+		if err := dxvm.Run(ctx); err != nil {
+			slog.Error("run virtual machine", "err", err)
+			os.Exit(1)
+		}
+	}
+}
+
+func findDevboxDir() (dir string, found bool, err error) {
+	dir = "."
+	if wd, err := os.Getwd(); err == nil {
+		dir = wd
+	}
+
+	home, _ := os.UserHomeDir()
+	vol := filepath.VolumeName(dir)
+	for {
+		// Refuse to go past the user's home directory or search root.
+		if dir == "" || dir == "/" || dir == home || dir == vol {
+			return dir, false, nil
+		}
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			return "", false, nil
+		}
+		_, found := slices.BinarySearchFunc(entries, "devbox.json", func(e fs.DirEntry, t string) int {
+			return cmp.Compare(e.Name(), t)
+		})
+		if found {
+			return dir, true, nil
+		}
+		dir = filepath.Dir(dir)
 	}
 }

pkg/sandbox/vm/cmd/dxvm/main.go

@@ -0,0 +1,70 @@
+package vm
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Code-Hex/vz/v3"
+)
+
+func scriptedConsole(ctx context.Context, logger *slog.Logger, prompt string, script []string) (*vz.VirtioConsoleDeviceSerialPortConfiguration, error) {
+	stdinr, stdinw, err := os.Pipe()
+	if err != nil {
+		return nil, fmt.Errorf("create stdin pipe: %v", err)
+	}
+	stdoutr, stdoutw, err := os.Pipe()
+	if err != nil {
+		return nil, fmt.Errorf("create stdout pipe: %v", err)
+	}
+
+	go func() {
+		var idle *time.Timer
+		idleDur := time.Second
+		sawPrompt := false
+		doneWriting := false
+		scanner := bufio.NewScanner(io.TeeReader(stdoutr, os.Stdout))
+		for scanner.Scan() && ctx.Err() == nil {
+			logger.Debug("install console", "stdout", scanner.Text())
+
+			if doneWriting {
+				continue
+			}
+			if idle != nil {
+				doneWriting = !idle.Reset(idleDur)
+				continue
+			}
+
+			sawPrompt = sawPrompt || strings.Contains(scanner.Text(), prompt)
+			if !sawPrompt {
+				continue
+			}
+			idle = time.AfterFunc(idleDur, sync.OnceFunc(func() {
+				_, err := stdinw.WriteString(strings.Join(script, " && ") + "\n")
+				if err != nil {
+					logger.Error("error writing to VM standard input", "err", err)
+				}
+				stdinw.Close()
+			}))
+		}
+		if err := scanner.Err(); err != nil {
+			logger.Error("error reading install console stdout", "err", err)
+		}
+	}()
+
+	attach, err := vz.NewFileHandleSerialPortAttachment(stdinr, stdoutw)
+	if err != nil {
+		return nil, fmt.Errorf("create serial port attachment: %v", err)
+	}
+	config, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(attach)
+	if err != nil {
+		return nil, fmt.Errorf("create serial port configuration: %v", err)
+	}
+	return config, nil
+}