feat(recipes): dependency ordering, dry-run, and cloudinit post-boot apply

docs/recipe-spec-v2.md

@@ -137,6 +137,14 @@ VM creation (disk mode)
   -> provision-stage recipes run
 ```
 
+## Reboot Behavior
+
+A recipe declares `reboot = true` in its manifest to mark that the guest needs a reboot before its changes take effect. Examples: switching init systems, loading a new kernel module.
+
+Stoat reboots the guest once, after every recipe in the apply run finishes. The reboot happens at the end of the run, not after the individual recipe that requested it. If several recipes in the run declare `reboot = true`, stoat still reboots only once.
+
+This reboot applies to disk-mode VMs only. Live-mode VMs run their root filesystem on tmpfs, so a reboot wipes it. A live recipe that needs to restart a session restarts it in place instead, for example with `kill -HUP 1`.
+
 ## State Tracking
 
 Add to `vm.toml`:

internal/backend/cloudinit.go

@@ -68,24 +68,45 @@ func (cloudinitBackend) Prepare(v *config.VM) error {
 	if err != nil {
 		return err
 	}
-	// v.Recipes only holds names the form offered for this VM's os/backend.
-	// recipes.List already filters by backend at selection time, so every
-	// entry here is already a cloud fragment. No extra backend check is
-	// needed before reading them.
-	var recipeBodies []string
-	for _, name := range v.Recipes {
-		body, err := recipes.Read(name)
-		if err != nil {
-			return fmt.Errorf("reading recipe %s: %w", name, err)
-		}
-		recipeBodies = append(recipeBodies, body)
+	scripts, err := recipeScripts(v)
+	if err != nil {
+		return err
+	}
+	// WrapScripts renders every recipe into one write_files+runcmd fragment
+	// cloud-init runs in order at first boot. An empty selection wraps to "",
+	// which Seed must not carry as a document; pass no bodies instead.
+	var bodies []string
+	if frag := cloudinit.WrapScripts(scripts); frag != "" {
+		bodies = []string{frag}
 	}
-	if _, err := cloudinit.Seed(v, pub, recipeBodies); err != nil {
+	if _, err := cloudinit.Seed(v, pub, bodies); err != nil {
 		return err
 	}
 	return nil
 }
 
+// recipeScripts resolves v.Recipes to the cloud-init scripts WrapScripts
+// renders: each recipe's manifest, then the script body for v.OS. A recipe
+// with no recipe.toml went missing since create time and errors here.
+func recipeScripts(v *config.VM) ([]cloudinit.Script, error) {
+	var scripts []cloudinit.Script
+	for _, name := range v.Recipes {
+		m, ok, err := recipes.ManifestFor(name)
+		if err != nil {
+			return nil, fmt.Errorf("reading recipe %s: %w", name, err)
+		}
+		if !ok {
+			return nil, fmt.Errorf("reading recipe %s: no recipe.toml", name)
+		}
+		body, err := m.ScriptContent(v.OS)
+		if err != nil {
+			return nil, fmt.Errorf("reading recipe %s: %w", name, err)
+		}
+		scripts = append(scripts, cloudinit.Script{Name: name, Content: body})
+	}
+	return scripts, nil
+}
+
 // Args attaches the seed cdrom only in cloud mode; see the mode-guard note
 // on cloudinitBackend.
 func (cloudinitBackend) Args(v *config.VM) []string {

internal/cli/cli.go

@@ -51,6 +51,10 @@ type Args struct {
 	// behavior existed.
 	NoApply bool
 
+	// DryRun belongs to "apply": it prints the plan (core.PlanApply) and runs
+	// nothing.
+	DryRun bool
+
 	// JSON is set by Main from the pre-parse argv scan, never by Parse: the
 	// flag has to be recognized before any parser exists so a usage error
 	// can still produce an envelope. It implies Quiet, so every prose line

internal/cli/grammar.go

@@ -198,8 +198,9 @@ type waitCmd struct {
 }
 
 type applyCmd struct {
-	VM   string   `arg:"" help:"vm name"`
-	Only []string `help:"subset of the VM's own recipes"`
+	VM     string   `arg:"" help:"vm name"`
+	Only   []string `help:"subset of the VM's own recipes"`
+	DryRun bool     `name:"dry-run" help:"print what would run without running it"`
 }
 
 type recipesCmd struct {
@@ -383,10 +384,10 @@ func (g *grammar) toArgs(path string) (*Args, error) {
 		a.VM, a.Until, a.Timeout = w.VM, core.Until(w.Until), w.Timeout
 
 	case "apply":
-		a.VM, a.Only = g.Apply.VM, trimList(g.Apply.Only)
+		a.VM, a.Only, a.DryRun = g.Apply.VM, trimList(g.Apply.Only), g.Apply.DryRun
 
 	case "provision":
-		a.VM, a.Only = g.Provision.VM, trimList(g.Provision.Only)
+		a.VM, a.Only, a.DryRun = g.Provision.VM, trimList(g.Provision.Only), g.Provision.DryRun
 
 	case "recipes":
 		a.OS, a.Backend = g.Recipes.OS, g.Recipes.Backend

internal/cli/run_apply.go

@@ -24,6 +24,10 @@ func runApply(a *Args, stdout, stderr io.Writer) int {
 		return a.fail(stdout, stderr, err)
 	}
 
+	if a.DryRun {
+		return runApplyDryRun(a, stdout, stderr)
+	}
+
 	applied := a.Only
 	if len(applied) == 0 {
 		applied = v.Recipes
@@ -60,9 +64,6 @@ func runApply(a *Args, stdout, stderr io.Writer) int {
 		return ExitOK
 	}
 	if aerr != nil {
-		// core.ErrAppliedAtBoot is a real outcome for a cloud VM, mapped to
-		// applied_at_boot by wire's error table; it is not special-cased into
-		// a success here.
 		return a.fail(stdout, stderr, aerr)
 	}
 
@@ -72,3 +73,24 @@ func runApply(a *Args, stdout, stderr io.Writer) int {
 	fmt.Fprintf(stdout, "%s: recipes applied\n", a.VM)
 	return ExitOK
 }
+
+// runApplyDryRun prints core.PlanApply's plan and runs nothing. Human output
+// is one line per recipe ("xfce (run, never applied)"); --json emits the plan
+// array. The plan is computed host-side, so the VM need not be running.
+func runApplyDryRun(a *Args, stdout, stderr io.Writer) int {
+	plan, err := core.PlanApply(a.VM, core.ApplyOpts{Only: a.Only})
+	if err != nil {
+		return a.fail(stdout, stderr, err)
+	}
+	if a.JSON {
+		return a.ok(stdout, plan)
+	}
+	for _, p := range plan {
+		reason := p.Reason
+		if p.Version != "" {
+			reason = fmt.Sprintf("%s at %s", p.Reason, p.Version)
+		}
+		fmt.Fprintf(stdout, "%s (%s, %s)\n", p.Name, p.Action, reason)
+	}
+	return ExitOK
+}

internal/cli/subcommands_test.go

@@ -283,29 +283,22 @@ func TestApplyMissingVMIsNotFound(t *testing.T) {
 	}
 }
 
-// A cloud VM's recipes ran at first boot, so apply has nothing to do and must
-// say which of the two it is, not report a generic failure.
-//
-// The VM has to look RUNNING to reach that check: core.Apply refuses a stopped
-// VM first, so a stopped cloud VM answers not_running, which is also true and
-// is not the answer this is about.
-func TestApplyOnACloudVMIsAppliedAtBoot(t *testing.T) {
+// A cloudinit VM applies over ssh after first boot (item 3), so apply no
+// longer refuses it. With no recipes there is nothing to run, so apply is a
+// clean no-op rather than the old applied_at_boot refusal.
+func TestApplyOnACloudVMIsNoLongerRefused(t *testing.T) {
 	dir := cliRoot(t)
 	v := saveVM(t, &config.VM{
 		Name: "cloudy", OS: "ubuntu", Mode: "cloud", Backend: "cloudinit",
-		RAM: 2048, CPUs: 2, SSHPort: 2201, Recipes: []string{"xfce"},
+		RAM: 2048, CPUs: 2, SSHPort: 2201,
 	})
 	v.Dir = filepath.Join(dir, "cloudy")
 	stop := fakeRunning(t, v)
 	defer stop()
 
 	code, objs := runJSON(t, "apply", "cloudy")
-	if code != ExitFail {
-		t.Fatalf("exit = %d, want %d: %v", code, ExitFail, objs)
-	}
-	errObj, _ := result(t, objs)["error"].(map[string]any)
-	if errObj["code"] != wire.CodeAppliedAtBoot {
-		t.Errorf("code = %v, want %q", errObj["code"], wire.CodeAppliedAtBoot)
+	if code != ExitOK {
+		t.Fatalf("exit = %d, want %d: %v", code, ExitOK, objs)
 	}
 }
 

internal/cli/wire/errors.go

@@ -25,7 +25,6 @@ const (
 	CodeImmutableField       = "immutable_field"
 	CodeDiskShrink           = "disk_shrink"
 	CodeCannotReach          = "cannot_reach"
-	CodeAppliedAtBoot        = "applied_at_boot"
 	CodeUnknownLog           = "unknown_log"
 	CodeTimeout              = "timeout"
 	CodeCanceled             = "canceled"
@@ -65,7 +64,6 @@ var codeTable = []struct {
 	{CodeImmutableField, core.ErrImmutableField},
 	{CodeDiskShrink, core.ErrDiskShrink},
 	{CodeCannotReach, core.ErrCannotReach},
-	{CodeAppliedAtBoot, core.ErrAppliedAtBoot},
 	{CodeUnknownLog, core.ErrUnknownWhich},
 	{CodeTimeout, context.DeadlineExceeded},
 	{CodeCanceled, context.Canceled},

internal/cli/wire/errors_test.go

@@ -26,7 +26,6 @@ func TestMapErrorEveryCoreSentinel(t *testing.T) {
 		{fmt.Errorf("%w: os", core.ErrImmutableField), CodeImmutableField},
 		{fmt.Errorf("%w: 8G -> 4G", core.ErrDiskShrink), CodeDiskShrink},
 		{fmt.Errorf("%w: work: not running", core.ErrCannotReach), CodeCannotReach},
-		{fmt.Errorf("%w: work", core.ErrAppliedAtBoot), CodeAppliedAtBoot},
 		{fmt.Errorf("%w: %q", core.ErrUnknownWhich, "bogus"), CodeUnknownLog},
 		{context.DeadlineExceeded, CodeTimeout},
 		{context.Canceled, CodeCanceled},

internal/cloudinit/scripts.go

@@ -9,6 +9,12 @@ import (
 // matching docs/recipe-spec-v2.md's cloudinit execution model.
 const scriptDir = "/var/lib/stoat/recipes"
 
+// MarkerDir holds one empty file per recipe that ran successfully at first
+// boot. core.Apply reads it over ssh to rebuild v.Applied for a cloudinit VM,
+// which never populated it at create time. The name is the recipe's, with no
+// extension.
+const MarkerDir = "/var/lib/stoat/.applied"
+
 // Script pairs a recipe's Name with the body WrapScripts should run for it,
 // i.e. the manifest's Name and manifest.ScriptContent(osName) for the guest
 // being provisioned.
@@ -41,7 +47,12 @@ func WrapScripts(scripts []Script) string {
 		wf.WriteString("    permissions: '0755'\n")
 		wf.WriteString("    content: |\n")
 		wf.WriteString(indentBlock(s.Content))
-		rc.WriteString(fmt.Sprintf("  - %s\n", path))
+		// The script runs, then drops a marker on success. core.Apply reads
+		// MarkerDir over ssh to rebuild v.Applied post-boot. The && chain
+		// leaves no marker for a script that failed, so a failed recipe stays
+		// pending instead of being recorded as applied.
+		marker := fmt.Sprintf("%s/%s", MarkerDir, s.Name)
+		rc.WriteString(fmt.Sprintf("  - %s && mkdir -p %s && touch %s\n", path, MarkerDir, marker))
 	}
 
 	return "#cloud-config\n" + wf.String() + rc.String()

internal/cloudinit/scripts_test.go

@@ -65,8 +65,9 @@ func TestWrapScriptsSingleScript(t *testing.T) {
 	if len(f.Runcmd) != 1 {
 		t.Fatalf("runcmd has %d entries, want 1:\n%s", len(f.Runcmd), body)
 	}
-	if f.Runcmd[0] != "/var/lib/stoat/recipes/xfce.sh" {
-		t.Errorf("runcmd[0] = %q, want /var/lib/stoat/recipes/xfce.sh", f.Runcmd[0])
+	wantCmd := "/var/lib/stoat/recipes/xfce.sh && mkdir -p /var/lib/stoat/.applied && touch /var/lib/stoat/.applied/xfce"
+	if f.Runcmd[0] != wantCmd {
+		t.Errorf("runcmd[0] = %q, want %q", f.Runcmd[0], wantCmd)
 	}
 }
 
@@ -100,7 +101,9 @@ func TestWrapScriptsPreservesOrder(t *testing.T) {
 	if len(f.Runcmd) != len(wantPaths) {
 		t.Fatalf("runcmd has %d entries, want %d:\n%s", len(f.Runcmd), len(wantPaths), body)
 	}
-	for i, want := range wantPaths {
+	names := []string{"base", "docker", "xfce"}
+	for i, path := range wantPaths {
+		want := path + " && mkdir -p /var/lib/stoat/.applied && touch /var/lib/stoat/.applied/" + names[i]
 		if f.Runcmd[i] != want {
 			t.Errorf("runcmd[%d] = %q, want %q", i, f.Runcmd[i], want)
 		}