Detect and quarantine wedged vGPU VFs

cmd/api/main.go

@@ -657,6 +657,12 @@ func run() error {
 			return app.HealthCheckController.Run(gctx)
 		})
 	}
+	if app.VGPUSentinelController != nil {
+		grp.Go(func() error {
+			logger.Info("starting vGPU sentinel controller")
+			return app.VGPUSentinelController.Run(gctx)
+		})
+	}
 	if restartController, ok := app.InstanceManager.(interface {
 		StartRestartPolicyController(context.Context) error
 	}); ok {

cmd/api/wire.go

@@ -29,26 +29,27 @@ import (
 
 // application struct to hold initialized components
 type application struct {
-	Ctx                   context.Context
-	Logger                *slog.Logger
-	Config                *config.Config
-	ImageManager          images.Manager
-	SystemManager         system.Manager
-	NetworkManager        network.Manager
-	DeviceManager         devices.Manager
-	InstanceManager       instances.Manager
-	VolumeManager         volumes.Manager
-	BuilderManager        builders.Manager
-	IngressManager        ingress.Manager
-	BuildManager          builds.Manager
-	PushManager           imagepush.Manager
-	ResourceManager       *resources.Manager
-	GuestMemoryController guestmemory.Controller
-	AutoStandbyController *autostandby.Controller
-	HealthCheckController *instances.HealthCheckController
-	VMMetricsManager      *vm_metrics.Manager
-	Registry              *registry.Registry
-	ApiService            *api.ApiService
+	Ctx                    context.Context
+	Logger                 *slog.Logger
+	Config                 *config.Config
+	ImageManager           images.Manager
+	SystemManager          system.Manager
+	NetworkManager         network.Manager
+	DeviceManager          devices.Manager
+	InstanceManager        instances.Manager
+	VolumeManager          volumes.Manager
+	BuilderManager         builders.Manager
+	IngressManager         ingress.Manager
+	BuildManager           builds.Manager
+	PushManager            imagepush.Manager
+	ResourceManager        *resources.Manager
+	GuestMemoryController  guestmemory.Controller
+	AutoStandbyController  *autostandby.Controller
+	HealthCheckController  *instances.HealthCheckController
+	VGPUSentinelController *instances.VGPUSentinelController
+	VMMetricsManager       *vm_metrics.Manager
+	Registry               *registry.Registry
+	ApiService             *api.ApiService
 }
 
 // initializeApp is the injector function
@@ -72,6 +73,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideGuestMemoryController,
 		providers.ProvideAutoStandbyController,
 		providers.ProvideHealthCheckController,
+		providers.ProvideVGPUSentinelController,
 		providers.ProvideVMMetricsManager,
 		providers.ProvideRegistry,
 		api.New,

lib/devices/GPU.md

@@ -282,13 +282,41 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)
 ```
 
 (0x65 = timeout; the guest's init requests are never answered, and
-`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because
-placement is deterministic least-loaded, an idle host re-picks the same VF for
-every request, so one wedged VF presents as all vGPU instances failing while
-`/resources` reports full capacity.
-
-The wedge itself leaves no host-side log: no kernel error, no XID, no plugin
-crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
+`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle).
+
+Hypeman detects this automatically: the guest agent watches the guest kernel
+log (`/dev/kmsg`) for that line and reports it as a `HYPEMAN-GPU-INIT-FAILED`
+marker in the instance's `logs/app.log`, which the vGPU sentinel controller
+scans for every vendor VFIO instance. A match quarantines the VF in
+`<data-dir>/gpu/vf-health.json` (it survives restarts): the VF is excluded
+from placement and from advertised profile availability, and its parent GPU
+becomes overflow-only — deprioritized for new placements. The conviction is
+logged at error level (`quarantined wedged vGPU VF`) and counted in
+`hypeman_instances_vgpu_sentinel_convictions_total`;
+`hypeman_instances_vgpu_quarantined_vfs` gauges the current quarantine count.
+There is no rate limit on convictions: a systemic non-wedge init failure
+(e.g. a guest/host driver mismatch) emits the same line on every VF and
+would quarantine the whole host, so such changes must be validated on a
+test host first, and the convictions counter is the signal to alert on if
+one gets through.
+
+Detection requires the hypeman guest agent: an image that skips the agent
+never reports, so a wedge hit exclusively by such images stays undetected in
+v1. Start archives the previous boot's app log before persisting a new
+assignment, so a marker written within one scan interval of a stop/start is
+archived unscanned; that VF returns to the pool unconvicted until the next
+victim boot re-emits the marker. The scanner requires the complete standalone
+guest-agent log envelope, so customer commands and ordinary output containing
+the marker do not match. The
+serial console remains guest-writable: a root guest can deliberately imitate
+the entire line and quarantine the VF its own instance holds; the quarantine
+only ever removes capacity, never touches the instance.
+
+The wedge-creating kill itself leaves no host-side log: no kernel error, no
+XID, no plugin crash. Detection therefore happens on the next boot that lands
+on the VF, whose guest driver starts failing ~27s after spawn.
+
+The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
 still initializing the VF (roughly the first seconds after process start):
 a single hard kill in that window wedges the VF near-deterministically,
 while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM —
@@ -303,18 +331,47 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it.
 Confirm by assigning the same profile on a different VF: if that guest
 initializes, the VF is wedged, not the driver stack. Remediate by cycling
 SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it
-requires no vGPU assignments on that GPU):
+requires no vGPU assignments on that GPU). The DCGM quiesce is not optional:
+with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d`
+fails with `Cannot obtain unbindLock` on first contact.
+
+Any manual edit to `vf-health.json` needs an immediate hypeman restart: the
+store loads only at startup, and a conviction landing first re-persists the
+in-memory set over your edit. The restart does not disturb running VMs —
+startup reconciliation protects live VFs.
+
+**Draining the parent GPU.** Overflow-only is a preference, not a cordon:
+under capacity pressure new placements still land on the card's healthy VFs
+and refill it. To drain the card, quarantine all of its VFs by hand — add
+records to `vf-health.json` (the shape of a real conviction: `vf_address`
+plus `quarantined_at`) and restart. Running instances are untouched and
+drain through their normal lifecycle: standby is blocked for vGPU instances,
+so only a running VM pins a VF, and each stop or delete frees one for good.
+Monitor by listing instances whose `gpu.device_path` sits under the parent
+GPU; once none remain, run the cycle below.
 
 ```bash
+# 1. Quiesce the services holding the GPU (required for the unbind lock).
+systemctl stop nvidia-dcgm-exporter nvidia-dcgm
+
+# 2. Cycle SR-IOV on the parent GPU.
 /usr/lib/nvidia/sriov-manage -d <parent-gpu-pci-addr>
 /usr/lib/nvidia/sriov-manage -e <parent-gpu-pci-addr>
+
+# 3. Restart the quiesced services.
+systemctl start nvidia-dcgm nvidia-dcgm-exporter
 ```
 
+After the cycle, clear the quarantine by removing the card's entries from
+`vf-health.json` (restart rule above), then boot a GPU instance as
+verification: placement excludes quarantined VFs, so the recovered VF cannot
+be targeted while its entry exists, and there is no VF-pin API — clearing
+first is safe because the sentinel automatically re-quarantines the VF if the
+cycle did not cure it (every cycle in hardware validation did).
+
 Do not unbind/rebind the VF from the nvidia driver — it breaks the
 nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`)
 and the VF stops accepting assignments entirely until the SR-IOV cycle.
-Services holding the GPU (DCGM, persistenced) must be stopped for the cycle
-to obtain the unbind lock.
 
 ### vGPU assignment fails
 

lib/devices/manager.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"os"
 	"runtime"
 	"strings"
@@ -85,6 +86,12 @@ type manager struct {
 // NewManager creates a new device manager.
 // Use SetLivenessChecker after construction to enable accurate orphan detection.
 func NewManager(p *paths.Paths) Manager {
+	// The VF health store lives with the device manager: constructing one
+	// without loading the store would run placement against an empty,
+	// non-persisting quarantine set.
+	if err := InitVFHealth(p.VFHealthState()); err != nil {
+		slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err)
+	}
 	return &manager{
 		paths:      p,
 		vfioBinder: NewVFIOBinder(),