Detect external bootc status changes via fsnotify (milestone 4d)

Makefile

@@ -68,6 +68,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
 		BINK_LOCAL_REGISTRY_NODE_IMAGE=$(BINK_LOCAL_REGISTRY_NODE_IMAGE) \
 		ARTIFACTS=$(ARTIFACTS) \
 		BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \
+		BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \
 		go test -timeout 10m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) .
 
 ##@ Build

cmd/daemon/main.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -14,6 +15,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	bootcv1alpha1 "github.com/jlebon/bootc-operator/api/v1alpha1"
@@ -32,6 +34,9 @@ func init() {
 }
 
 func main() {
+	var pollInterval time.Duration
+	flag.DurationVar(&pollInterval, "poll-interval", 5*time.Minute, "Interval for polling bootc status as a fallback to fsnotify")
+
 	opts := zap.Options{
 		Development: true,
 	}
@@ -62,17 +67,32 @@ func main() {
 		os.Exit(1)
 	}
 
+	statusChanged := make(chan event.GenericEvent, 1)
+
 	if err := (&daemon.BootcNodeReconciler{
-		Client:   mgr.GetClient(),
-		Scheme:   mgr.GetScheme(),
-		NodeName: nodeName,
-		Executor: bootc.NewHostExecutor(),
+		Client:        mgr.GetClient(),
+		Scheme:        mgr.GetScheme(),
+		NodeName:      nodeName,
+		Executor:      bootc.NewHostExecutor(),
+		StatusChanged: statusChanged,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Failed to create controller", "controller", "bootcnode")
 		os.Exit(1)
 	}
 
-	setupLog.Info("Starting daemon", "node", nodeName)
+	watcher := &daemon.StatusWatcher{
+		PollInterval: pollInterval,
+		PrimaryPath:  daemon.DefaultPrimaryPath,
+		FallbackPath: daemon.DefaultFallbackPath,
+		Events:       statusChanged,
+		NodeName:     nodeName,
+	}
+	if err := mgr.Add(watcher); err != nil {
+		setupLog.Error(err, "Failed to add status watcher")
+		os.Exit(1)
+	}
+
+	setupLog.Info("Starting daemon", "node", nodeName, "pollInterval", pollInterval)
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "Failed to run daemon")
 		os.Exit(1)

config/daemon/daemon.yaml

@@ -38,7 +38,7 @@ spec:
         resources:
           limits:
             cpu: 500m
-            memory: 128Mi
+            memory: 512Mi
           requests:
             cpu: 10m
             memory: 64Mi

internal/bootc/executor.go

@@ -6,13 +6,18 @@ import (
 	"context"
 	"fmt"
 	"os/exec"
+	"strings"
+
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 // Executor abstracts the execution of bootc commands on the host.
 // The real implementation uses nsenter to enter the host's mount and
 // PID namespaces. Tests can provide a fake implementation.
 type Executor interface {
 	Status(ctx context.Context) ([]byte, error)
+	Stage(ctx context.Context, image string) error
+	Reboot(ctx context.Context) error
 }
 
 // HostExecutor runs bootc commands on the host via nsenter.
@@ -23,21 +28,46 @@ func NewHostExecutor() *HostExecutor {
 	return &HostExecutor{}
 }
 
-func (e *HostExecutor) Status(ctx context.Context) ([]byte, error) {
-	cmd := exec.CommandContext(ctx,
-		"nsenter",
+func (e *HostExecutor) nsenterCmd(ctx context.Context, args ...string) *exec.Cmd {
+	base := []string{
 		"--target", "1",
-		"--mount",
-		"--pid",
-		"--setuid", "0",
-		"--setgid", "0",
-		"--env",
-		"--",
-		"bootc", "status", "--json", "--format-version", "1",
-	)
+		"--mount", "--pid",
+		"--setuid", "0", "--setgid", "0",
+		"--env", "--",
+	}
+	return exec.CommandContext(ctx, "nsenter", append(base, args...)...)
+}
+
+func (e *HostExecutor) Status(ctx context.Context) ([]byte, error) {
+	cmd := e.nsenterCmd(ctx, "bootc", "status", "--json", "--format-version", "1")
 	out, err := cmd.Output()
 	if err != nil {
 		return nil, fmt.Errorf("running bootc status: %w", err)
 	}
 	return out, nil
 }
+
+func (e *HostExecutor) Stage(ctx context.Context, image string) error {
+	log := logf.FromContext(ctx)
+
+	// TODO: use --download-only once available (https://github.com/bootc-dev/bootc/issues/2137)
+	cmd := e.nsenterCmd(ctx, "bootc", "switch", image)
+	log.Info("Executing", "cmd", strings.Join(cmd.Args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running bootc switch: %s: %w", out, err)
+	}
+	return nil
+}
+
+func (e *HostExecutor) Reboot(ctx context.Context) error {
+	log := logf.FromContext(ctx)
+
+	cmd := e.nsenterCmd(ctx, "systemctl", "reboot")
+	log.Info("Executing", "cmd", strings.Join(cmd.Args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running systemctl reboot: %s: %w", out, err)
+	}
+	return nil
+}

internal/daemon/fake_test.go

@@ -4,24 +4,127 @@ package daemon
 
 import (
 	"context"
+	"encoding/json"
+	"strings"
 	"sync"
+
+	testutil "github.com/jlebon/bootc-operator/test/util"
+
+	"github.com/jlebon/bootc-operator/internal/bootc"
 )
 
 type fakeExecutor struct {
-	mu   sync.Mutex
-	data []byte
-	err  error
+	mu        sync.Mutex
+	status    bootc.Status
+	statusErr error
+
+	stageErr  error
+	stageImg  string
+	stageHook func()
+
+	rebooted bool
 }
 
 func (f *fakeExecutor) Status(_ context.Context) ([]byte, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	return f.data, f.err
+	if f.statusErr != nil {
+		return nil, f.statusErr
+	}
+	data, err := json.Marshal(f.status)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (f *fakeExecutor) Stage(_ context.Context, image string) error {
+	f.mu.Lock()
+	f.stageImg = image
+	f.mu.Unlock()
+
+	if f.stageHook != nil {
+		f.stageHook()
+	}
+	if f.stageErr != nil {
+		return f.stageErr
+	}
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, digest, _ := strings.Cut(image, "@")
+	f.status.Status.Staged = newBootEntry(image, digest)
+	return nil
+}
+
+func (f *fakeExecutor) Reboot(_ context.Context) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.rebooted = true
+	return nil
 }
 
-func (f *fakeExecutor) set(data []byte, err error) {
+func (f *fakeExecutor) setStatusErr(err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	f.data = data
-	f.err = err
+	f.statusErr = err
+}
+
+func (f *fakeExecutor) setStageErr(err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.stageErr = err
+}
+
+func (f *fakeExecutor) setStageHook(hook func()) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.stageHook = hook
+}
+
+func (f *fakeExecutor) getStageImg() string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.stageImg
+}
+
+func (f *fakeExecutor) getRebooted() bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.rebooted
+}
+
+func (f *fakeExecutor) reset() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.status = bootc.Status{}
+	f.statusErr = nil
+	f.stageErr = nil
+	f.stageImg = ""
+	f.stageHook = nil
+	f.rebooted = false
+}
+
+func newBootEntry(image, digest string) *bootc.BootEntry {
+	return &bootc.BootEntry{
+		Image: &bootc.ImageStatus{
+			Image:        bootc.ImageReference{Image: image, Transport: "registry"},
+			ImageDigest:  digest,
+			Architecture: "amd64",
+		},
+	}
+}
+
+func newBootcStatus(bootedDigest string) bootc.Status {
+	return bootc.Status{
+		APIVersion: "org.containers.bootc/v1alpha1",
+		Kind:       "BootcHost",
+		Spec: bootc.StatusSpec{
+			Image:     &bootc.ImageReference{Image: testutil.ImageTaggedRef, Transport: "registry"},
+			BootOrder: "default",
+		},
+		Status: bootc.StatusBody{
+			Booted: newBootEntry(testutil.ImageTaggedRef, bootedDigest),
+		},
+	}
 }