From 95f23914cc66b6155455e387db403a1a67405222 Mon Sep 17 00:00:00 2001 From: paf0186 Date: Tue, 17 Nov 2026 13:06:16 -0600 Subject: [PATCH 1/4] macOS: expose Arm EL2 to ltvm guests with --nested Persist a create-time opt-in for native aarch64 HVF guests. Validate the macOS EL2 capability and the selected QEMU before boot, then enable the platform GIC and guest EL2. --- README.md | 9 ++--- docs/NESTED_VIRTUALIZATION.md | 38 ++++++++++++++++----- ltvm | 6 ++++ ltvm_pkg/qemu_run.py | 60 ++++++++++++++++++++++++++++++++-- ltvm_pkg/vm_commands.py | 8 +++++ ltvm_pkg/vm_state.py | 8 +++-- tests/test_qemu_run.py | 18 ++++++++++ tests/test_vm_create_phases.py | 28 ++++++++++++++++ 8 files changed, 156 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 281d0d68..5d74066b 100644 --- a/README.md +++ b/README.md @@ -71,10 +71,11 @@ a podman-managed Linux VM + QEMU microvms. A few things differ from Linux: ends), and `export LTVM_PODMAN_AUTOSTOP=0` keeps it up for good. A machine that lists as running but does not answer -- a failed start leaves it so -- is restarted by the next build. -- **To confine an agent to a VM**, run ltvm inside an aarch64 Linux VM - on the Mac instead: UTM on the Apple Virtualization backend with - nested virtualization on, which needs an M3 or later and macOS 15+. - ltvm then gets KVM there; see +- **To run KVM inside an ltvm guest**, create it with `--nested`. This + requires Apple M3 or later, macOS 15+, and QEMU 11.1+ (included in + `./ltvm install`). To confine an agent to a Linux VM, run ltvm inside + that guest or inside a UTM Linux guest with nested virtualization + enabled on the Apple Virtualization backend. See [docs/NESTED_VIRTUALIZATION.md](docs/NESTED_VIRTUALIZATION.md#apple-silicon----aarch64). ```bash diff --git a/docs/NESTED_VIRTUALIZATION.md b/docs/NESTED_VIRTUALIZATION.md index 055a033d..d16f984a 100644 --- a/docs/NESTED_VIRTUALIZATION.md +++ b/docs/NESTED_VIRTUALIZATION.md @@ -153,9 +153,29 @@ gcloud compute instances update \ ### Apple Silicon -- aarch64 -ltvm runs on macOS itself, with HVF. To run it inside a Linux VM on -the Mac instead -- so an agent driving ltvm is confined to that VM -- -the Linux VM needs nested virtualization, and that needs all of: +#### Expose KVM in a VM created by ltvm + +On an Apple M3 or later with macOS 15 or later, use a native aarch64 +target, a Linux guest kernel with KVM enabled, and QEMU 11.1 or later. +`./ltvm install` supplies a signed QEMU bundle for Apple Silicon. Use +`--nested` at create time to expose Arm EL2 to the guest through HVF: + +```bash +ltvm create nested-host --target rocky10 --arch aarch64 --nested \ + --mem 6144 --vcpus 4 --root-size 16G --mdt-disks 0 --ost-disks 0 +ltvm vm exec nested-host -- ls -l /dev/kvm +ltvm vm exec nested-host -- dmesg | grep -E 'started at EL2|kvm \[' +``` + +The setting persists across stop and start. The memory and disk sizes +above leave room to run VMs and keep their images inside the guest. +For an inner QEMU, use +`-accel kvm -cpu host -machine virt,gic-version=max,its=off`. + +#### Run ltvm inside a Linux VM on the Mac + +Use the Linux guest above, or use UTM to create the outer VM. For the +UTM path, use: - an **M3 or later** chip (M1 and M2 have no nested virtualization); - **macOS 15** or later; @@ -176,13 +196,13 @@ sudo usermod -aG kvm $USER # if you are not in the kvm it: `accel=kvm` and `-cpu host`. `ps -o args -C qemu-system-aarch64` shows what a running VM got. -**ltvm turns off the GIC ITS here.** On an Apple CPU (`CPU +**ltvm turns off the GIC ITS for the inner VM.** On an Apple CPU (`CPU implementer : 0x61` in `/proc/cpuinfo`), ltvm launches KVM guests with -`virt,...,its=off`. With the ITS, virtio-pci MSIs lose wakeups under -this nesting: the guest boots past init, then sits in `cpu_do_idle` -for good while its vCPU thread shows 100% CPU. Without the ITS, -virtio falls back to wired INTx interrupts, which work. Trapping WFI -in the L1 KVM does not help, so it is the MSI delivery itself. +`virt,...,its=off`. This avoids a hang observed under UTM's Apple +Virtualization backend. With the ITS, virtio-pci MSIs lose wakeups and +the guest sits in `cpu_do_idle` while its vCPU thread shows 100% CPU. +Without the ITS, virtio falls back to wired INTx interrupts. +Trapping WFI in the L1 KVM does not help. `LTVM_FORCE_TCG=1` still forces emulation, if you ever need it. Parallels Desktop and VMware Fusion have not been tested with ltvm. diff --git a/ltvm b/ltvm index 6c2127c9..f11d14cf 100755 --- a/ltvm +++ b/ltvm @@ -1762,6 +1762,12 @@ def build_parser() -> argparse.ArgumentParser: "SOCKET names one you run yourself instead." ), ) + _vm_create_args.add_argument( + "--nested", + action="store_true", + help="Expose Arm EL2 to the guest for KVM (Apple Silicon macOS, " + "QEMU 11.1 or later). Fixed at create time.", + ) _vm_create_args.add_argument( "--kernel-args", dest="kernel_args", diff --git a/ltvm_pkg/qemu_run.py b/ltvm_pkg/qemu_run.py index bb1bd532..dac1a018 100644 --- a/ltvm_pkg/qemu_run.py +++ b/ltvm_pkg/qemu_run.py @@ -7,6 +7,7 @@ import functools import math import os +import platform import pwd import re import secrets @@ -51,6 +52,51 @@ def die(msg: str, code: int = EXIT_ERROR) -> NoReturn: sys.exit(code) +def check_hvf_nested(arch: str) -> None: + """Validate the host and the QEMU binary selected for this guest.""" + if ( + not is_macos() + or platform.machine() != "arm64" + or arch != "aarch64" + or os.environ.get("LTVM_FORCE_TCG") == "1" + ): + die("--nested requires a native aarch64 guest with macOS HVF") + + if int(platform.mac_ver()[0].split(".")[0] or 0) < 15: + die("--nested requires macOS 15 or later") + + import ctypes + + try: + hvf = ctypes.CDLL( + "/System/Library/Frameworks/Hypervisor.framework/Hypervisor" + ) + query = hvf.hv_vm_config_get_el2_supported + except (OSError, AttributeError) as exc: + die(f"--nested could not query macOS Hypervisor.framework EL2: {exc}") + query.argtypes = [ctypes.POINTER(ctypes.c_bool)] + query.restype = ctypes.c_int + supported = ctypes.c_bool() + status = query(ctypes.byref(supported)) + if status != 0: + die(f"--nested EL2 capability check failed (HVF status {status})") + if not supported.value: + die("--nested requires a Mac with HVF EL2 support (Apple M3 or later)") + + binary = qemu_binary_for_arch(arch) + try: + result = run([binary, "--version"], check=True) + except (OSError, subprocess.CalledProcessError) as exc: + die(f"--nested could not check QEMU at {binary}: {exc}") + version = re.search(r"\bversion (\d+)\.(\d+)\.(\d+)\b", result.stdout) + if not version or tuple(map(int, version.groups())) < (11, 1, 0): + found = version.group(0) if version else "unknown version" + die( + f"--nested requires QEMU 11.1 or later; {binary} reports {found}. " + "Run `ltvm install` to install ltvm's macOS QEMU bundle." + ) + + def _read_meminfo_mb(key: str) -> int: """Return /proc/meminfo's value in MiB, or 0 if unreadable. @@ -961,6 +1007,9 @@ def launch_qemu(vm: VMInfo, *, wait_seconds: int = 0) -> None: print(f"VM '{vm.name}' is already running", file=sys.stderr) return + if vm.nested: + check_hvf_nested(vm.arch) + if not vm.overlay_path.exists(): die(f"overlay missing for '{vm.name}'") @@ -1294,11 +1343,15 @@ def _start_qemu(vm: VMInfo) -> None: f"{str(_nic)!r} on VM {vm.name!r}" ) - import platform as _platform - arch = vm.arch qemu_bin = qemu_binary_for_arch(arch) machine = qemu_machine_for_arch(arch) + accel_args = [] + if vm.nested: + # The platform vGIC provides the timer and interrupt interface + # KVM needs when handling its own guests at EL2. + accel_args = ["-accel", "hvf,kernel-irqchip=on"] + machine = machine.replace(",accel=hvf", "") + ",virtualization=on" # q35 (x86) and virt (aarch64) both have a PCI bus, so virtio # devices attach as virtio-*-pci. Previously x86 used microvm and @@ -1317,7 +1370,7 @@ def _start_qemu(vm: VMInfo) -> None: # those, so /sbin/init aborts with "Fatal glibc error: CPU does not # support x86-64-v2" and the kernel panics. Nehalem (Intel 2008) is # the baseline CPU model that satisfies v2 in full. - host_arch = _platform.machine() + host_arch = platform.machine() import os as _os _force_tcg = _os.environ.get("LTVM_FORCE_TCG") == "1" @@ -1345,6 +1398,7 @@ def _start_qemu(vm: VMInfo) -> None: qemu_bin, "-name", vm.name, + *accel_args, "-machine", machine, "-cpu", diff --git a/ltvm_pkg/vm_commands.py b/ltvm_pkg/vm_commands.py index bcb43fa7..b14fda14 100644 --- a/ltvm_pkg/vm_commands.py +++ b/ltvm_pkg/vm_commands.py @@ -31,6 +31,7 @@ from .paths import load_meta_safe from .priv import SudoUnavailable, chmod_regular, sudo_prime, sudo_run from .qemu_run import ( + check_hvf_nested, die, host_node_problem, is_running, @@ -708,6 +709,7 @@ def _validate_kernel_args(raw: str | None) -> str: "disk_size": "--disk-size", "root_size": "--root-size", "kernel_args": "--kernel-args", + "nested": "--nested", "numa_nodes": "--numa-nodes", "numa_dist": "--numa-dist", "disk_dir": "--disk-dir", @@ -950,6 +952,7 @@ def _allocate_and_persist_vm( os_id=os_id, kver=kver, arch=os_arts.arch, + nested=getattr(args, "nested", False), # Track the invoking human. SUDO_USER preserves the legacy # ``sudo ltvm create`` path; the unified privilege model runs # the command as the user and elevates only individual writes. @@ -1057,6 +1060,7 @@ def _print_create_plan( "target": os_target, "variant": variant, "arch": getattr(args, "arch", None) or os_arch_of(image), + "nested": getattr(args, "nested", False), "kernel": kernel, "kernel_version": kver, "image": image, @@ -1096,6 +1100,8 @@ def _print_create_plan( print(f" {kernel}") print(f" image: {image}") print(f" cpu/mem: {args.vcpus} vcpus, {args.mem} MB") + if plan["nested"]: + print(" nested: Arm EL2 (HVF)") if getattr(args, "numa_nodes", 0): print(f" numa: {args.numa_nodes} nodes") if getattr(args, "numa_dist", ""): @@ -1826,6 +1832,8 @@ def cmd_create(args: argparse.Namespace) -> None: os_arts, image, kernel, kver, os_target, variant = _resolve_os_and_kernel( args, extra_nic_types ) + if getattr(args, "nested", False): + check_hvf_nested(os_arts.arch) base_name = Path(image).name os_id = os_target diff --git a/ltvm_pkg/vm_state.py b/ltvm_pkg/vm_state.py index 91ea091e..a58d4516 100644 --- a/ltvm_pkg/vm_state.py +++ b/ltvm_pkg/vm_state.py @@ -136,9 +136,8 @@ def qemu_machine_for_arch(arch: str = "x86_64") -> str: def _nested_kvm_on_apple(cpuinfo: Path = Path("/proc/cpuinfo")) -> bool: """True when this Linux host runs on an Apple CPU (implementer 0x61). - On Apple Silicon /dev/kvm only exists inside a VM with nested - virtualization on -- UTM's Apple Virtualization backend on M3 or - later, macOS 15 or later -- so the KVM guests ltvm starts here are + On Apple Silicon /dev/kvm exists inside a VM with nested + virtualization enabled by the macOS hypervisor, so KVM guests here are nested ones. There, a guest whose virtio-pci MSIs go through the GICv3 ITS boots past init and then stops taking interrupts: the vCPU idles in WFI with nothing ever waking it (verified on an M3 @@ -619,6 +618,7 @@ class VMInfo: image_id: str = "" os_id: str = "" # OS identifier (e.g. rocky9, ubuntu24) arch: str = "x86_64" # CPU architecture (x86_64, aarch64) + nested: bool = False # Expose EL2 to an Arm guest under macOS HVF. creator: str = ( "" # username that created the VM (SUDO_USER, or "" for legacy) ) @@ -803,6 +803,7 @@ def save(self) -> None: f"IMAGE_ID={self.image_id}\n" f"OS_ID={self.os_id}\n" f"ARCH={self.arch}\n" + f"NESTED={int(self.nested)}\n" f"CREATOR={self.creator}\n" f"OWNER_ID={self.owner_id or ''}\n" f"VARIANT={self.variant}\n" @@ -1008,6 +1009,7 @@ def _int(key: str, default: int) -> int: image_id=vals.get("IMAGE_ID", ""), os_id=vals.get("OS_ID", ""), arch=vals.get("ARCH", "x86_64"), + nested=vals.get("NESTED") == "1", creator=vals.get("CREATOR", ""), owner_id=vals.get("OWNER_ID") or None, variant=vals.get("VARIANT", "base"), diff --git a/tests/test_qemu_run.py b/tests/test_qemu_run.py index 093267f6..4746794b 100644 --- a/tests/test_qemu_run.py +++ b/tests/test_qemu_run.py @@ -988,6 +988,24 @@ def _launch_macos(self, vm: VMInfo, h: _LaunchHarness) -> None: ): qemu_run.launch_qemu(vm) + def test_nested_uses_hvf_gic_and_el2(self, tmp_vmdir: Path) -> None: + vm = _make_vm(tmp_vmdir, arch="aarch64") + vm.nested = True + h = _LaunchHarness() + with ( + patch("platform.system", return_value="Darwin"), + patch("platform.machine", return_value="arm64"), + patch("ltvm_pkg.qemu_run.check_hvf_nested") as check, + ): + self._launch_macos(vm, h) + check.assert_called_once_with("aarch64") + args = h.qemu_args or [] + assert args[args.index("-accel") + 1] == "hvf,kernel-irqchip=on" + assert args[args.index("-machine") + 1] == ( + "virt,gic-version=max,virtualization=on" + ) + assert args[args.index("-cpu") + 1] == "host" + def test_mgmt_nic_uses_stream_unix(self, tmp_vmdir: Path) -> None: """-netdev stream,addr.type=unix,addr.path=,server=off.""" vm = _make_vm(tmp_vmdir) diff --git a/tests/test_vm_create_phases.py b/tests/test_vm_create_phases.py index 50613492..25d990a2 100644 --- a/tests/test_vm_create_phases.py +++ b/tests/test_vm_create_phases.py @@ -234,6 +234,34 @@ def test_kernel_args_persisted_on_vm(self, tmp_vmdir: Path) -> None: ) assert VMInfo.load("co1-kargs").kernel_args == "slub_debug=FZPU" + def test_nested_persists_and_validates_resolved_arch( + self, tmp_vmdir: Path + ) -> None: + with ( + _create_env(tmp_vmdir) as env, + patch("ltvm_pkg.vm_commands.check_hvf_nested") as check, + ): + env["arts"].arch = "aarch64" + vm_commands.cmd_create(_create_args(name="co1-el2", nested=True)) + check.assert_called_once_with("aarch64") + assert VMInfo.load("co1-el2").nested + + def test_host_without_nested_support_writes_no_vm( + self, tmp_vmdir: Path + ) -> None: + with ( + _create_env(tmp_vmdir) as env, + patch( + "ltvm_pkg.vm_commands.check_hvf_nested", + side_effect=SystemExit(1), + ), + pytest.raises(SystemExit), + ): + vm_commands.cmd_create(_create_args(name="co1-noel2", nested=True)) + env["alloc_ip"].assert_not_called() + env["launch"].assert_not_called() + assert not (tmp_vmdir / "sockets" / "co1-noel2.info").exists() + def test_refused_kernel_args_write_nothing(self, tmp_vmdir: Path) -> None: with _create_env(tmp_vmdir) as env, pytest.raises(SystemExit): vm_commands.cmd_create( From be2abac438d5ac93073b722693e9ca3916bb63f8 Mon Sep 17 00:00:00 2001 From: paf0186 Date: Tue, 17 Nov 2026 13:11:56 -0600 Subject: [PATCH 2/4] macOS: name the EL2 opt-in --enable-nesting --- README.md | 2 +- docs/NESTED_VIRTUALIZATION.md | 11 +++++++---- ltvm | 3 ++- ltvm_pkg/qemu_run.py | 20 +++++++++++++------- ltvm_pkg/vm_commands.py | 2 +- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5d74066b..4f529bf9 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ a podman-managed Linux VM + QEMU microvms. A few things differ from Linux: ends), and `export LTVM_PODMAN_AUTOSTOP=0` keeps it up for good. A machine that lists as running but does not answer -- a failed start leaves it so -- is restarted by the next build. -- **To run KVM inside an ltvm guest**, create it with `--nested`. This +- **To run KVM inside an ltvm guest**, create it with `--enable-nesting`. This requires Apple M3 or later, macOS 15+, and QEMU 11.1+ (included in `./ltvm install`). To confine an agent to a Linux VM, run ltvm inside that guest or inside a UTM Linux guest with nested virtualization diff --git a/docs/NESTED_VIRTUALIZATION.md b/docs/NESTED_VIRTUALIZATION.md index d16f984a..44699506 100644 --- a/docs/NESTED_VIRTUALIZATION.md +++ b/docs/NESTED_VIRTUALIZATION.md @@ -158,17 +158,20 @@ gcloud compute instances update \ On an Apple M3 or later with macOS 15 or later, use a native aarch64 target, a Linux guest kernel with KVM enabled, and QEMU 11.1 or later. `./ltvm install` supplies a signed QEMU bundle for Apple Silicon. Use -`--nested` at create time to expose Arm EL2 to the guest through HVF: +`--enable-nesting` at create time to expose Arm EL2 to the guest through HVF: ```bash -ltvm create nested-host --target rocky10 --arch aarch64 --nested \ +ltvm create nested-host --target rocky10 --arch aarch64 --enable-nesting \ --mem 6144 --vcpus 4 --root-size 16G --mdt-disks 0 --ost-disks 0 ltvm vm exec nested-host -- ls -l /dev/kvm ltvm vm exec nested-host -- dmesg | grep -E 'started at EL2|kvm \[' ``` -The setting persists across stop and start. The memory and disk sizes -above leave room to run VMs and keep their images inside the guest. +The setting persists across stop and start. Enabling it changes the +guest CPU features. QEMU's HVF backend suppresses Arm SME when it +exposes EL2. + +The example allocates space for inner VMs and their images. For an inner QEMU, use `-accel kvm -cpu host -machine virt,gic-version=max,its=off`. diff --git a/ltvm b/ltvm index f11d14cf..6bb54432 100755 --- a/ltvm +++ b/ltvm @@ -1763,7 +1763,8 @@ def build_parser() -> argparse.ArgumentParser: ), ) _vm_create_args.add_argument( - "--nested", + "--enable-nesting", + dest="nested", action="store_true", help="Expose Arm EL2 to the guest for KVM (Apple Silicon macOS, " "QEMU 11.1 or later). Fixed at create time.", diff --git a/ltvm_pkg/qemu_run.py b/ltvm_pkg/qemu_run.py index dac1a018..9034ca16 100644 --- a/ltvm_pkg/qemu_run.py +++ b/ltvm_pkg/qemu_run.py @@ -60,10 +60,10 @@ def check_hvf_nested(arch: str) -> None: or arch != "aarch64" or os.environ.get("LTVM_FORCE_TCG") == "1" ): - die("--nested requires a native aarch64 guest with macOS HVF") + die("--enable-nesting requires a native aarch64 guest with macOS HVF") if int(platform.mac_ver()[0].split(".")[0] or 0) < 15: - die("--nested requires macOS 15 or later") + die("--enable-nesting requires macOS 15 or later") import ctypes @@ -73,26 +73,32 @@ def check_hvf_nested(arch: str) -> None: ) query = hvf.hv_vm_config_get_el2_supported except (OSError, AttributeError) as exc: - die(f"--nested could not query macOS Hypervisor.framework EL2: {exc}") + die( + f"--enable-nesting could not query macOS Hypervisor.framework EL2: {exc}" + ) query.argtypes = [ctypes.POINTER(ctypes.c_bool)] query.restype = ctypes.c_int supported = ctypes.c_bool() status = query(ctypes.byref(supported)) if status != 0: - die(f"--nested EL2 capability check failed (HVF status {status})") + die( + f"--enable-nesting EL2 capability check failed (HVF status {status})" + ) if not supported.value: - die("--nested requires a Mac with HVF EL2 support (Apple M3 or later)") + die( + "--enable-nesting requires a Mac with HVF EL2 support (Apple M3 or later)" + ) binary = qemu_binary_for_arch(arch) try: result = run([binary, "--version"], check=True) except (OSError, subprocess.CalledProcessError) as exc: - die(f"--nested could not check QEMU at {binary}: {exc}") + die(f"--enable-nesting could not check QEMU at {binary}: {exc}") version = re.search(r"\bversion (\d+)\.(\d+)\.(\d+)\b", result.stdout) if not version or tuple(map(int, version.groups())) < (11, 1, 0): found = version.group(0) if version else "unknown version" die( - f"--nested requires QEMU 11.1 or later; {binary} reports {found}. " + f"--enable-nesting requires QEMU 11.1 or later; {binary} reports {found}. " "Run `ltvm install` to install ltvm's macOS QEMU bundle." ) diff --git a/ltvm_pkg/vm_commands.py b/ltvm_pkg/vm_commands.py index b14fda14..da755ebd 100644 --- a/ltvm_pkg/vm_commands.py +++ b/ltvm_pkg/vm_commands.py @@ -709,7 +709,7 @@ def _validate_kernel_args(raw: str | None) -> str: "disk_size": "--disk-size", "root_size": "--root-size", "kernel_args": "--kernel-args", - "nested": "--nested", + "nested": "--enable-nesting", "numa_nodes": "--numa-nodes", "numa_dist": "--numa-dist", "disk_dir": "--disk-dir", From 6958c78a261cf723e77a170568057eaa4364d293 Mon Sep 17 00:00:00 2001 From: paf0186 Date: Tue, 17 Nov 2026 13:22:00 -0600 Subject: [PATCH 3/4] macOS: enable HVF nesting by default where supported Detect host EL2 and QEMU support at cold boot and persist a create-time opt-out for individual VMs and clusters. Keep the userspace GIC available when running newer QEMU on macOS before 15. --- CLAUDE.md | 18 ++++----- README.md | 10 ++--- docs/NESTED_VIRTUALIZATION.md | 18 +++++---- ltvm | 20 +++++----- ltvm_pkg/cli/cluster.py | 1 + ltvm_pkg/qemu_run.py | 73 ++++++++++++++-------------------- ltvm_pkg/vm_cluster.py | 9 +++++ ltvm_pkg/vm_commands.py | 13 +++--- ltvm_pkg/vm_state.py | 6 +-- skills/ltvm/SKILL.md | 7 ++++ tests/test_cluster_cli.py | 15 +++++++ tests/test_qemu_run.py | 43 ++++++++++++++++---- tests/test_vm_create_phases.py | 33 +++------------ 13 files changed, 145 insertions(+), 121 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 26b75d1a..d6b63e10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1089,15 +1089,15 @@ occurrence wins (`crashkernel=`) takes the user's value. `root=`, ltvm's values, and rc.local configures the guest from `fc_*`. Fixed at create time like the disks; `cluster create` applies it to every node. -**Nested KVM on Apple Silicon:** a Linux host on an Apple CPU (`CPU -implementer : 0x61`) is a VM under Apple Virtualization -- UTM, M3 or -later, nested virtualization on -- and `qemu_machine_for_arch` gives its -KVM guests `its=off` (`vm_state._nested_kvm_on_apple`). With the ITS, -virtio-pci MSIs lose wakeups there and the guest idles in `cpu_do_idle` -forever a few seconds after init; wired INTx works. It changes nothing -on any other host, macOS's HVF included. KVM is 3-40x faster than TCG -there; the benchmarks, against TCG and against ltvm on the Mac itself, -are in [docs/NESTED_VIRTUALIZATION.md](docs/NESTED_VIRTUALIZATION.md). +**Nested KVM on Apple Silicon:** on macOS, `qemu_run` checks the host's +EL2 capability and the selected QEMU binary at boot. Native Arm HVF +guests get EL2 on macOS 15+ with compatible hardware and QEMU 11.1+. +`create --disable-nesting` saves `DISABLE_NESTING=1` in the VM's +`.info` file. Inside an Apple-hosted Linux VM, `qemu_machine_for_arch` +gives KVM guests `its=off` (`vm_state._nested_kvm_on_apple`). This +uses wired INTx to avoid a virtio interrupt hang observed under +UTM's Apple Virtualization backend. See the setup and benchmarks in +[docs/NESTED_VIRTUALIZATION.md](docs/NESTED_VIRTUALIZATION.md). **Naming:** `-`, where `` is the first two consonants of the host's short name (`bl` on buildhost, `ts` on testnode) and N is unused diff --git a/README.md b/README.md index 4f529bf9..4b1a2719 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,11 @@ a podman-managed Linux VM + QEMU microvms. A few things differ from Linux: ends), and `export LTVM_PODMAN_AUTOSTOP=0` keeps it up for good. A machine that lists as running but does not answer -- a failed start leaves it so -- is restarted by the next build. -- **To run KVM inside an ltvm guest**, create it with `--enable-nesting`. This - requires Apple M3 or later, macOS 15+, and QEMU 11.1+ (included in - `./ltvm install`). To confine an agent to a Linux VM, run ltvm inside - that guest or inside a UTM Linux guest with nested virtualization - enabled on the Apple Virtualization backend. See +- **Native Arm guests can run KVM by default** on a Mac with hardware + EL2 support, macOS 15+, and QEMU 11.1+ (included in `./ltvm install`). + Pass `--disable-nesting` at create time to opt out. To confine an agent + to a Linux VM, run ltvm inside that guest or inside a UTM Linux guest + with nested virtualization enabled on the Apple Virtualization backend. See [docs/NESTED_VIRTUALIZATION.md](docs/NESTED_VIRTUALIZATION.md#apple-silicon----aarch64). ```bash diff --git a/docs/NESTED_VIRTUALIZATION.md b/docs/NESTED_VIRTUALIZATION.md index 44699506..02136ba4 100644 --- a/docs/NESTED_VIRTUALIZATION.md +++ b/docs/NESTED_VIRTUALIZATION.md @@ -155,21 +155,23 @@ gcloud compute instances update \ #### Expose KVM in a VM created by ltvm -On an Apple M3 or later with macOS 15 or later, use a native aarch64 -target, a Linux guest kernel with KVM enabled, and QEMU 11.1 or later. -`./ltvm install` supplies a signed QEMU bundle for Apple Silicon. Use -`--enable-nesting` at create time to expose Arm EL2 to the guest through HVF: +ltvm exposes Arm EL2 to native aarch64 guests by default on compatible +Apple Silicon (M3 or later) with macOS 15+ and QEMU 11.1+. A Linux guest +kernel with KVM enabled then provides `/dev/kvm`. `./ltvm install` +supplies a signed QEMU bundle for Apple Silicon. ```bash -ltvm create nested-host --target rocky10 --arch aarch64 --enable-nesting \ +ltvm create nested-host --target rocky10 --arch aarch64 \ --mem 6144 --vcpus 4 --root-size 16G --mdt-disks 0 --ost-disks 0 ltvm vm exec nested-host -- ls -l /dev/kvm ltvm vm exec nested-host -- dmesg | grep -E 'started at EL2|kvm \[' ``` -The setting persists across stop and start. Enabling it changes the -guest CPU features. QEMU's HVF backend suppresses Arm SME when it -exposes EL2. +ltvm checks support at each cold boot, including for existing VMs. +On an older host or QEMU, the VM boots with ordinary HVF. To opt out, +pass `--disable-nesting` to `ltvm create` or `ltvm cluster create`. +ltvm saves the opt-out for each VM. QEMU suppresses Arm SME when it +exposes EL2 through HVF, so opt out for workloads that need SME. The example allocates space for inner VMs and their images. For an inner QEMU, use diff --git a/ltvm b/ltvm index 6bb54432..2a060c40 100755 --- a/ltvm +++ b/ltvm @@ -1595,7 +1595,16 @@ def build_parser() -> argparse.ArgumentParser: target_parser.set_defaults(func=_target_no_action) # -- VM lifecycle: create -- - _vm_create_args = argparse.ArgumentParser(add_help=False) + _nesting_args = argparse.ArgumentParser(add_help=False) + _nesting_args.add_argument( + "--disable-nesting", + action="store_true", + help="Keep Arm EL2 unavailable to guests on macOS HVF. " + "Fixed at create time.", + ) + _vm_create_args = argparse.ArgumentParser( + add_help=False, parents=[_nesting_args] + ) _vm_create_args.add_argument( "--vcpus", type=int, default=2, help="vCPU count (default: 2)" ) @@ -1762,13 +1771,6 @@ def build_parser() -> argparse.ArgumentParser: "SOCKET names one you run yourself instead." ), ) - _vm_create_args.add_argument( - "--enable-nesting", - dest="nested", - action="store_true", - help="Expose Arm EL2 to the guest for KVM (Apple Silicon macOS, " - "QEMU 11.1 or later). Fixed at create time.", - ) _vm_create_args.add_argument( "--kernel-args", dest="kernel_args", @@ -2575,7 +2577,7 @@ def build_parser() -> argparse.ArgumentParser: ccp = cluster_sub.add_parser( "create", - parents=[common], + parents=[common, _nesting_args], help="Create a cluster from node specs (needs root)", ) ccp.add_argument("name", help="Cluster name (e.g. co2)") diff --git a/ltvm_pkg/cli/cluster.py b/ltvm_pkg/cli/cluster.py index 6dfef1b7..f5991e48 100644 --- a/ltvm_pkg/cli/cluster.py +++ b/ltvm_pkg/cli/cluster.py @@ -220,6 +220,7 @@ def cmd_cluster_create(args: argparse.Namespace) -> int: eth0_speed=args.eth0_speed, nic=list(args.nic or []), kernel_args=args.kernel_args, + disable_nesting=getattr(args, "disable_nesting", False), numa_nodes=args.numa_nodes, numa_dist=args.numa_dist, numa_dist_file=args.numa_dist_file, diff --git a/ltvm_pkg/qemu_run.py b/ltvm_pkg/qemu_run.py index 9034ca16..3a783ab7 100644 --- a/ltvm_pkg/qemu_run.py +++ b/ltvm_pkg/qemu_run.py @@ -52,19 +52,8 @@ def die(msg: str, code: int = EXIT_ERROR) -> NoReturn: sys.exit(code) -def check_hvf_nested(arch: str) -> None: - """Validate the host and the QEMU binary selected for this guest.""" - if ( - not is_macos() - or platform.machine() != "arm64" - or arch != "aarch64" - or os.environ.get("LTVM_FORCE_TCG") == "1" - ): - die("--enable-nesting requires a native aarch64 guest with macOS HVF") - - if int(platform.mac_ver()[0].split(".")[0] or 0) < 15: - die("--enable-nesting requires macOS 15 or later") - +def _hvf_el2_supported() -> bool: + """Query the host's Arm EL2 capability on macOS 15 or later.""" import ctypes try: @@ -72,35 +61,26 @@ def check_hvf_nested(arch: str) -> None: "/System/Library/Frameworks/Hypervisor.framework/Hypervisor" ) query = hvf.hv_vm_config_get_el2_supported - except (OSError, AttributeError) as exc: - die( - f"--enable-nesting could not query macOS Hypervisor.framework EL2: {exc}" - ) + except (OSError, AttributeError): + return False query.argtypes = [ctypes.POINTER(ctypes.c_bool)] query.restype = ctypes.c_int supported = ctypes.c_bool() - status = query(ctypes.byref(supported)) - if status != 0: - die( - f"--enable-nesting EL2 capability check failed (HVF status {status})" - ) - if not supported.value: - die( - "--enable-nesting requires a Mac with HVF EL2 support (Apple M3 or later)" - ) + return query(ctypes.byref(supported)) == 0 and supported.value - binary = qemu_binary_for_arch(arch) + +def _qemu_supports_hvf_vgic(binary: str) -> bool: + """QEMU 11.1 introduced both the HVF platform GIC and EL2 support.""" try: - result = run([binary, "--version"], check=True) - except (OSError, subprocess.CalledProcessError) as exc: - die(f"--enable-nesting could not check QEMU at {binary}: {exc}") + result = run([binary, "--version"], check=True, timeout=5) + except (OSError, subprocess.SubprocessError): + return False version = re.search(r"\bversion (\d+)\.(\d+)\.(\d+)\b", result.stdout) - if not version or tuple(map(int, version.groups())) < (11, 1, 0): - found = version.group(0) if version else "unknown version" - die( - f"--enable-nesting requires QEMU 11.1 or later; {binary} reports {found}. " - "Run `ltvm install` to install ltvm's macOS QEMU bundle." - ) + return version is not None and tuple(map(int, version.groups())) >= ( + 11, + 1, + 0, + ) def _read_meminfo_mb(key: str) -> int: @@ -1013,9 +993,6 @@ def launch_qemu(vm: VMInfo, *, wait_seconds: int = 0) -> None: print(f"VM '{vm.name}' is already running", file=sys.stderr) return - if vm.nested: - check_hvf_nested(vm.arch) - if not vm.overlay_path.exists(): die(f"overlay missing for '{vm.name}'") @@ -1353,11 +1330,19 @@ def _start_qemu(vm: VMInfo) -> None: qemu_bin = qemu_binary_for_arch(arch) machine = qemu_machine_for_arch(arch) accel_args = [] - if vm.nested: - # The platform vGIC provides the timer and interrupt interface - # KVM needs when handling its own guests at EL2. - accel_args = ["-accel", "hvf,kernel-irqchip=on"] - machine = machine.replace(",accel=hvf", "") + ",virtualization=on" + if arch == "aarch64" and "accel=hvf" in machine: + macos_major = int(platform.mac_ver()[0].partition(".")[0] or 0) + if macos_major < 15: + if _qemu_supports_hvf_vgic(qemu_bin): + # Newer QEMU defaults to a GIC that needs macOS 15. + machine = machine.replace(",accel=hvf", "") + accel_args = ["-accel", "hvf,kernel-irqchip=off"] + elif ( + not vm.disable_nesting + and _hvf_el2_supported() + and _qemu_supports_hvf_vgic(qemu_bin) + ): + machine += ",virtualization=on" # q35 (x86) and virt (aarch64) both have a PCI bus, so virtio # devices attach as virtio-*-pci. Previously x86 used microvm and diff --git a/ltvm_pkg/vm_cluster.py b/ltvm_pkg/vm_cluster.py index 4080dd3e..e7b84977 100644 --- a/ltvm_pkg/vm_cluster.py +++ b/ltvm_pkg/vm_cluster.py @@ -606,6 +606,7 @@ def _create_one_node( disk_cache: str = "", disk_prealloc: str = "", eth0_speed: int | None = None, + disable_nesting: bool = False, ) -> tuple[str, int, str]: """Create a single cluster VM via ltvm subprocess. @@ -679,6 +680,8 @@ def _create_one_node( if kernel_args: # One argv word, so a value starting with '-' is not an option. cmd += [f"--kernel-args={kernel_args}"] + if disable_nesting: + cmd.append("--disable-nesting") if numa_nodes: cmd += ["--numa-nodes", str(numa_nodes)] if numa_dist: @@ -880,6 +883,7 @@ def _print_cluster_plan( disk_dir: str = "", disk_cache: str = "", disk_prealloc: str = "", + disable_nesting: bool = False, ) -> None: """Report what `cluster create` would do, for --dry-run. @@ -930,6 +934,8 @@ def _print_cluster_plan( print(f" nics: eth0 (mgmt) + {', '.join(nics)}") if kernel_args: print(f" cmdline: + {kernel_args}") + if disable_nesting: + print(" nesting: disabled for every node (macOS HVF)") print(f" owner: {owner_id}") print("Nothing was written. Re-run without --dry-run to create it.") @@ -1140,6 +1146,7 @@ def cmd_cluster_create(args: argparse.Namespace) -> None: die(network_issue) nics = [str(n) for n in nic_specs] kernel_args = _validate_kernel_args(getattr(args, "kernel_args", None)) + disable_nesting = getattr(args, "disable_nesting", False) wait_seconds: int = getattr(args, "wait", 0) numa_nodes: int = getattr(args, "numa_nodes", 0) or 0 problem = nic_numa_problem(nic_specs, numa_nodes) @@ -1202,6 +1209,7 @@ def cmd_cluster_create(args: argparse.Namespace) -> None: nics=nics, kernel_args=kernel_args, owner_id=owner_id, + disable_nesting=disable_nesting, **disk_opts, ) return @@ -1256,6 +1264,7 @@ def create_wave(wave: list[ClusterNode]) -> list[str]: variant="base" if node_targets[node.name] else variant, share_disks=node.name in shared, eth0_speed=getattr(args, "eth0_speed", None), + disable_nesting=disable_nesting, **disk_opts, ): node for node in wave diff --git a/ltvm_pkg/vm_commands.py b/ltvm_pkg/vm_commands.py index da755ebd..68ed2fa2 100644 --- a/ltvm_pkg/vm_commands.py +++ b/ltvm_pkg/vm_commands.py @@ -31,7 +31,6 @@ from .paths import load_meta_safe from .priv import SudoUnavailable, chmod_regular, sudo_prime, sudo_run from .qemu_run import ( - check_hvf_nested, die, host_node_problem, is_running, @@ -709,7 +708,7 @@ def _validate_kernel_args(raw: str | None) -> str: "disk_size": "--disk-size", "root_size": "--root-size", "kernel_args": "--kernel-args", - "nested": "--enable-nesting", + "disable_nesting": "--disable-nesting", "numa_nodes": "--numa-nodes", "numa_dist": "--numa-dist", "disk_dir": "--disk-dir", @@ -952,7 +951,7 @@ def _allocate_and_persist_vm( os_id=os_id, kver=kver, arch=os_arts.arch, - nested=getattr(args, "nested", False), + disable_nesting=getattr(args, "disable_nesting", False), # Track the invoking human. SUDO_USER preserves the legacy # ``sudo ltvm create`` path; the unified privilege model runs # the command as the user and elevates only individual writes. @@ -1060,7 +1059,7 @@ def _print_create_plan( "target": os_target, "variant": variant, "arch": getattr(args, "arch", None) or os_arch_of(image), - "nested": getattr(args, "nested", False), + "disable_nesting": getattr(args, "disable_nesting", False), "kernel": kernel, "kernel_version": kver, "image": image, @@ -1100,8 +1099,8 @@ def _print_create_plan( print(f" {kernel}") print(f" image: {image}") print(f" cpu/mem: {args.vcpus} vcpus, {args.mem} MB") - if plan["nested"]: - print(" nested: Arm EL2 (HVF)") + if plan["disable_nesting"]: + print(" nesting: disabled (macOS HVF)") if getattr(args, "numa_nodes", 0): print(f" numa: {args.numa_nodes} nodes") if getattr(args, "numa_dist", ""): @@ -1832,8 +1831,6 @@ def cmd_create(args: argparse.Namespace) -> None: os_arts, image, kernel, kver, os_target, variant = _resolve_os_and_kernel( args, extra_nic_types ) - if getattr(args, "nested", False): - check_hvf_nested(os_arts.arch) base_name = Path(image).name os_id = os_target diff --git a/ltvm_pkg/vm_state.py b/ltvm_pkg/vm_state.py index a58d4516..e9182d30 100644 --- a/ltvm_pkg/vm_state.py +++ b/ltvm_pkg/vm_state.py @@ -618,7 +618,7 @@ class VMInfo: image_id: str = "" os_id: str = "" # OS identifier (e.g. rocky9, ubuntu24) arch: str = "x86_64" # CPU architecture (x86_64, aarch64) - nested: bool = False # Expose EL2 to an Arm guest under macOS HVF. + disable_nesting: bool = False # Keep EL2 off under macOS HVF. creator: str = ( "" # username that created the VM (SUDO_USER, or "" for legacy) ) @@ -803,7 +803,7 @@ def save(self) -> None: f"IMAGE_ID={self.image_id}\n" f"OS_ID={self.os_id}\n" f"ARCH={self.arch}\n" - f"NESTED={int(self.nested)}\n" + f"DISABLE_NESTING={int(self.disable_nesting)}\n" f"CREATOR={self.creator}\n" f"OWNER_ID={self.owner_id or ''}\n" f"VARIANT={self.variant}\n" @@ -1009,7 +1009,7 @@ def _int(key: str, default: int) -> int: image_id=vals.get("IMAGE_ID", ""), os_id=vals.get("OS_ID", ""), arch=vals.get("ARCH", "x86_64"), - nested=vals.get("NESTED") == "1", + disable_nesting=vals.get("DISABLE_NESTING") == "1", creator=vals.get("CREATOR", ""), owner_id=vals.get("OWNER_ID") or None, variant=vals.get("VARIANT", "base"), diff --git a/skills/ltvm/SKILL.md b/skills/ltvm/SKILL.md index f2797ade..bb48326e 100644 --- a/skills/ltvm/SKILL.md +++ b/skills/ltvm/SKILL.md @@ -27,6 +27,13 @@ On macOS it is `./ltvm install` with no `sudo`: it refuses to run as root there and prompts for sudo itself where it must (README.md, "Running on macOS"). +Native aarch64 VMs on macOS expose Arm EL2 by default on capable +Apple Silicon (M3 or later), macOS 15+, and QEMU 11.1+. A Linux guest +with KVM enabled gets `/dev/kvm` and can run its own accelerated VMs. +Older setups boot with ordinary HVF. Pass `--disable-nesting` to +`ltvm create` or `ltvm cluster create` to save a per-VM opt-out. +Use `ltvm vm exec -- ls -l /dev/kvm` to check the guest. + `install` also sets up tab completion for the human's shell (bash, zsh, fish), which needs a new shell before it works -- worth saying when you walk someone through setup, since nothing else announces it. `ltvm diff --git a/tests/test_cluster_cli.py b/tests/test_cluster_cli.py index d7d025df..0f259dd4 100644 --- a/tests/test_cluster_cli.py +++ b/tests/test_cluster_cli.py @@ -428,6 +428,12 @@ def test_kernel_args_parsed(self) -> None: self._captured_ns().kernel_args == "slub_debug=FZPU page_owner=on" ) + def test_disable_nesting_parsed(self) -> None: + cmd_cluster( + _ns("create", "co1", "--disable-nesting", "mgs+mds:co1-mds:1") + ) + assert self._captured_ns().disable_nesting + def test_unknown_flag_errors(self) -> None: err = _expect_usage_error( "create", "co1", "mgs+mds:co1-mds:1", "--frobnicate" @@ -2315,6 +2321,15 @@ def test_role_match_invokes_execvp(self, tmp_sockets: Path) -> None: class TestClusterNodeDiskArgs: """`cluster create` must state disk counts explicitly.""" + def test_disable_nesting_reaches_each_node(self) -> None: + node = vm_cluster.parse_node_spec("oss:co9-oss:3") + with patch.object(vm_cluster, "_run_prefixed") as run: + run.return_value = MagicMock(returncode=0, stdout="", stderr="") + vm_cluster._create_one_node( + node, vcpus=2, mem=None, disable_nesting=True + ) + assert "--disable-nesting" in run.call_args.args[0] + def test_kernel_and_variant_reach_each_node(self) -> None: node = vm_cluster.parse_node_spec("oss:co9-oss:3") with patch.object(vm_cluster, "_run_prefixed") as run: diff --git a/tests/test_qemu_run.py b/tests/test_qemu_run.py index 4746794b..21512421 100644 --- a/tests/test_qemu_run.py +++ b/tests/test_qemu_run.py @@ -988,22 +988,49 @@ def _launch_macos(self, vm: VMInfo, h: _LaunchHarness) -> None: ): qemu_run.launch_qemu(vm) - def test_nested_uses_hvf_gic_and_el2(self, tmp_vmdir: Path) -> None: + @pytest.mark.parametrize( + ("macos", "el2", "qemu", "disabled", "want_el2", "want_userspace_gic"), + [ + ("26.6", True, True, False, True, False), + ("26.6", True, True, True, False, False), + ("26.6", True, False, False, False, False), + ("15.7", False, True, False, False, False), + ("14.8", False, True, False, False, True), + ("14.8", False, False, False, False, False), + ], + ) + def test_arm_hvf_nesting( + self, + tmp_vmdir: Path, + macos: str, + el2: bool, + qemu: bool, + disabled: bool, + want_el2: bool, + want_userspace_gic: bool, + ) -> None: vm = _make_vm(tmp_vmdir, arch="aarch64") - vm.nested = True + vm.disable_nesting = disabled h = _LaunchHarness() with ( patch("platform.system", return_value="Darwin"), patch("platform.machine", return_value="arm64"), - patch("ltvm_pkg.qemu_run.check_hvf_nested") as check, + patch("platform.mac_ver", return_value=(macos, (), "arm64")), + patch("ltvm_pkg.qemu_run._hvf_el2_supported", return_value=el2), + patch( + "ltvm_pkg.qemu_run._qemu_supports_hvf_vgic", return_value=qemu + ), ): self._launch_macos(vm, h) - check.assert_called_once_with("aarch64") args = h.qemu_args or [] - assert args[args.index("-accel") + 1] == "hvf,kernel-irqchip=on" - assert args[args.index("-machine") + 1] == ( - "virt,gic-version=max,virtualization=on" - ) + machine = args[args.index("-machine") + 1] + assert ("virtualization=on" in machine) is want_el2 + if want_userspace_gic: + assert args[args.index("-accel") + 1] == "hvf,kernel-irqchip=off" + assert machine == "virt,gic-version=max" + else: + assert "-accel" not in args + assert "accel=hvf" in machine assert args[args.index("-cpu") + 1] == "host" def test_mgmt_nic_uses_stream_unix(self, tmp_vmdir: Path) -> None: diff --git a/tests/test_vm_create_phases.py b/tests/test_vm_create_phases.py index 25d990a2..20130b88 100644 --- a/tests/test_vm_create_phases.py +++ b/tests/test_vm_create_phases.py @@ -234,33 +234,12 @@ def test_kernel_args_persisted_on_vm(self, tmp_vmdir: Path) -> None: ) assert VMInfo.load("co1-kargs").kernel_args == "slub_debug=FZPU" - def test_nested_persists_and_validates_resolved_arch( - self, tmp_vmdir: Path - ) -> None: - with ( - _create_env(tmp_vmdir) as env, - patch("ltvm_pkg.vm_commands.check_hvf_nested") as check, - ): - env["arts"].arch = "aarch64" - vm_commands.cmd_create(_create_args(name="co1-el2", nested=True)) - check.assert_called_once_with("aarch64") - assert VMInfo.load("co1-el2").nested - - def test_host_without_nested_support_writes_no_vm( - self, tmp_vmdir: Path - ) -> None: - with ( - _create_env(tmp_vmdir) as env, - patch( - "ltvm_pkg.vm_commands.check_hvf_nested", - side_effect=SystemExit(1), - ), - pytest.raises(SystemExit), - ): - vm_commands.cmd_create(_create_args(name="co1-noel2", nested=True)) - env["alloc_ip"].assert_not_called() - env["launch"].assert_not_called() - assert not (tmp_vmdir / "sockets" / "co1-noel2.info").exists() + def test_disable_nesting_persists(self, tmp_vmdir: Path) -> None: + with _create_env(tmp_vmdir): + vm_commands.cmd_create( + _create_args(name="co1-noel2", disable_nesting=True) + ) + assert VMInfo.load("co1-noel2").disable_nesting def test_refused_kernel_args_write_nothing(self, tmp_vmdir: Path) -> None: with _create_env(tmp_vmdir) as env, pytest.raises(SystemExit): From 16caf417ea026c3ef3e62ed344537b799cc51392 Mon Sep 17 00:00:00 2001 From: paf0186 Date: Tue, 17 Nov 2026 13:25:00 -0600 Subject: [PATCH 4/4] doctor: offer a QEMU upgrade for Apple Silicon nesting --- docs/NESTED_VIRTUALIZATION.md | 6 ++++-- ltvm_pkg/qemu_run.py | 15 ++++++++------ ltvm_pkg/vm_commands.py | 33 ++++++++++++++++++++++++++++++ skills/ltvm/SKILL.md | 2 ++ tests/test_qemu_run.py | 4 ++-- tests/test_vm_doctor.py | 38 +++++++++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 10 deletions(-) diff --git a/docs/NESTED_VIRTUALIZATION.md b/docs/NESTED_VIRTUALIZATION.md index 02136ba4..70b4cde3 100644 --- a/docs/NESTED_VIRTUALIZATION.md +++ b/docs/NESTED_VIRTUALIZATION.md @@ -157,8 +157,10 @@ gcloud compute instances update \ ltvm exposes Arm EL2 to native aarch64 guests by default on compatible Apple Silicon (M3 or later) with macOS 15+ and QEMU 11.1+. A Linux guest -kernel with KVM enabled then provides `/dev/kvm`. `./ltvm install` -supplies a signed QEMU bundle for Apple Silicon. +kernel with KVM enabled then provides `/dev/kvm`. On a capable Mac, +`ltvm doctor` flags an older QEMU. `ltvm doctor --fix` or +`ltvm install --qemu` installs the signed Apple Silicon QEMU bundle +from [ltvm's GitHub releases](https://github.com/lustre-tools/lustre-test-vms/releases). ```bash ltvm create nested-host --target rocky10 --arch aarch64 \ diff --git a/ltvm_pkg/qemu_run.py b/ltvm_pkg/qemu_run.py index 3a783ab7..48b48529 100644 --- a/ltvm_pkg/qemu_run.py +++ b/ltvm_pkg/qemu_run.py @@ -52,8 +52,11 @@ def die(msg: str, code: int = EXIT_ERROR) -> NoReturn: sys.exit(code) -def _hvf_el2_supported() -> bool: - """Query the host's Arm EL2 capability on macOS 15 or later.""" +def hvf_el2_supported() -> bool: + """Return whether this Apple Silicon host can expose Arm EL2.""" + if not is_macos() or platform.machine() != "arm64": + return False + import ctypes try: @@ -69,7 +72,7 @@ def _hvf_el2_supported() -> bool: return query(ctypes.byref(supported)) == 0 and supported.value -def _qemu_supports_hvf_vgic(binary: str) -> bool: +def qemu_supports_hvf_vgic(binary: str) -> bool: """QEMU 11.1 introduced both the HVF platform GIC and EL2 support.""" try: result = run([binary, "--version"], check=True, timeout=5) @@ -1333,14 +1336,14 @@ def _start_qemu(vm: VMInfo) -> None: if arch == "aarch64" and "accel=hvf" in machine: macos_major = int(platform.mac_ver()[0].partition(".")[0] or 0) if macos_major < 15: - if _qemu_supports_hvf_vgic(qemu_bin): + if qemu_supports_hvf_vgic(qemu_bin): # Newer QEMU defaults to a GIC that needs macOS 15. machine = machine.replace(",accel=hvf", "") accel_args = ["-accel", "hvf,kernel-irqchip=off"] elif ( not vm.disable_nesting - and _hvf_el2_supported() - and _qemu_supports_hvf_vgic(qemu_bin) + and hvf_el2_supported() + and qemu_supports_hvf_vgic(qemu_bin) ): machine += ",virtualization=on" diff --git a/ltvm_pkg/vm_commands.py b/ltvm_pkg/vm_commands.py index 68ed2fa2..8dd8462a 100644 --- a/ltvm_pkg/vm_commands.py +++ b/ltvm_pkg/vm_commands.py @@ -33,10 +33,12 @@ from .qemu_run import ( die, host_node_problem, + hvf_el2_supported, is_running, kill_qemu, launch_qemu, numa_problem, + qemu_supports_hvf_vgic, run, ) from .vm_net import ( @@ -85,6 +87,7 @@ drop_orphan_clusters, lustre_libdir, nic_ip6, + qemu_binary_for_arch, resolve_os_artifacts, ) @@ -4010,6 +4013,33 @@ def _unreadable_vm_taps(name: str) -> set[str]: return taps_for_vm(name, count) +def _check_macos_nesting_qemu(fix: bool) -> tuple[int, int]: + if not hvf_el2_supported(): + return 0, 0 + binary = qemu_binary_for_arch("aarch64") + if qemu_supports_hvf_vgic(binary): + return 0, 0 + + print( + f"nested virtualization needs QEMU 11.1 or newer at {binary}; " + "run `ltvm install --qemu` to install ltvm's macOS bundle from GitHub" + ) + if not fix: + return 1, 0 + + from .host_setup import install_qemu_macos_vhost + + try: + install_qemu_macos_vhost() + if not qemu_supports_hvf_vgic(qemu_binary_for_arch("aarch64")): + raise RuntimeError("selected QEMU is still missing or too old") + except (RuntimeError, OSError, subprocess.CalledProcessError) as exc: + print(f" FAILED to update macOS QEMU: {exc}") + return 1, 1 + print(" fixed: installed QEMU with nesting support") + return 1, 0 + + def cmd_doctor(args: argparse.Namespace) -> int: issues = 0 # Counts repairs that were attempted and did not work, so --fix @@ -4298,6 +4328,9 @@ def cmd_doctor(args: argparse.Namespace) -> int: if is_macos(): print("note: `ltvm target export` needs a Linux host; not checked") + found, failed = _check_macos_nesting_qemu(args.fix) + issues += found + fix_failures += failed else: from ltvm_pkg import image_export diff --git a/skills/ltvm/SKILL.md b/skills/ltvm/SKILL.md index bb48326e..a4fabe6c 100644 --- a/skills/ltvm/SKILL.md +++ b/skills/ltvm/SKILL.md @@ -32,6 +32,8 @@ Apple Silicon (M3 or later), macOS 15+, and QEMU 11.1+. A Linux guest with KVM enabled gets `/dev/kvm` and can run its own accelerated VMs. Older setups boot with ordinary HVF. Pass `--disable-nesting` to `ltvm create` or `ltvm cluster create` to save a per-VM opt-out. +On a capable Mac, `ltvm doctor` flags an older QEMU. Run +`ltvm doctor --fix` to install the current bundle from GitHub. Use `ltvm vm exec -- ls -l /dev/kvm` to check the guest. `install` also sets up tab completion for the human's shell (bash, zsh, diff --git a/tests/test_qemu_run.py b/tests/test_qemu_run.py index 21512421..8fb29a0e 100644 --- a/tests/test_qemu_run.py +++ b/tests/test_qemu_run.py @@ -1016,9 +1016,9 @@ def test_arm_hvf_nesting( patch("platform.system", return_value="Darwin"), patch("platform.machine", return_value="arm64"), patch("platform.mac_ver", return_value=(macos, (), "arm64")), - patch("ltvm_pkg.qemu_run._hvf_el2_supported", return_value=el2), + patch("ltvm_pkg.qemu_run.hvf_el2_supported", return_value=el2), patch( - "ltvm_pkg.qemu_run._qemu_supports_hvf_vgic", return_value=qemu + "ltvm_pkg.qemu_run.qemu_supports_hvf_vgic", return_value=qemu ), ): self._launch_macos(vm, h) diff --git a/tests/test_vm_doctor.py b/tests/test_vm_doctor.py index 918e23d3..07e1644f 100644 --- a/tests/test_vm_doctor.py +++ b/tests/test_vm_doctor.py @@ -81,6 +81,7 @@ def doctor_env(tmp_vmdir: Path, tmp_path: Path) -> Iterator[dict]: "ltvm_pkg.vm_commands._check_completion", return_value=([], [], 0), ), + patch("ltvm_pkg.vm_commands.hvf_el2_supported", return_value=False), patch("ltvm_pkg.vm_commands.is_running", return_value=False), patch("shutil.disk_usage", return_value=plenty), ): @@ -98,6 +99,43 @@ def _make_args(fix: bool = False) -> argparse.Namespace: return argparse.Namespace(fix=fix, json=False) +@pytest.mark.parametrize( + ("fix", "supported_after_fix"), + [ + (False, False), + (True, True), + (True, False), + ], +) +def test_doctor_checks_and_repairs_selected_qemu_for_nesting( + doctor_env: dict, + capsys: pytest.CaptureFixture[str], + fix: bool, + supported_after_fix: bool, +) -> None: + binary = "/custom/qemu/bin/qemu-system-aarch64" + with ( + patch("ltvm_pkg.vm_commands.is_macos", return_value=True), + patch("ltvm_pkg.vm_commands.hvf_el2_supported", return_value=True), + patch("ltvm_pkg.vm_commands.qemu_binary_for_arch", return_value=binary), + patch( + "ltvm_pkg.vm_commands.qemu_supports_hvf_vgic", + side_effect=[False, supported_after_fix], + ) as check, + patch("ltvm_pkg.host_setup.install_qemu_macos_vhost") as install, + ): + rc = vm_commands.cmd_doctor(_make_args(fix=fix)) + output = capsys.readouterr().out + assert ( + f"nested virtualization needs QEMU 11.1 or newer at {binary}" in output + ) + assert install.call_count == int(fix) + assert check.call_count == 1 + int(fix) + assert all(call.args == (binary,) for call in check.call_args_list) + assert ("fixed: installed QEMU" in output) is supported_after_fix + assert (rc == 0) is supported_after_fix + + # ── clean ─────────────────────────────────────────────────