fix(daemon): end the session and clear its claim when its lease ends

docs/adr/0007-remote-device-leases.md

@@ -53,6 +53,15 @@ an idle daemon alive; other leases, including human-control holds, do not. A
 retained lease the caller stops heartbeating expires after its `ttlMs`, and idle
 reap then shuts the daemon down within one more idle window.
 
+A session cannot outlive the lease it was opened under. When the lease is
+released (tenant `lease_release` or host `DELETE /admin/leases/<lease-id>`) or
+expires, the daemon tears the session down under its execution locks and clears
+its device claim, even when the device no longer answers. The daemon sweeps for
+expired leases every second, so expiry does not wait for a later request;
+in-flight work and human-control holds still keep a lease from expiring. Once
+daemon shutdown begins, no new lease-end teardown starts, and shutdown waits for
+any that already started before it tears sessions down itself.
+
 ## Consequences
 
 Device contention can fail before platform execution with an explicit

packages/daemon/src/server/__tests__/host-lease-http.test.ts

@@ -23,12 +23,14 @@ const BODY = {
 async function withServer(
   t: Parameters<typeof skipWhenLoopbackUnavailable>[0],
   run: (origin: string, registry: LeaseRegistry) => Promise<void>,
+  endSessionsForLease?: (leaseId: string) => Promise<void>,
 ): Promise<void> {
   if (await skipWhenLoopbackUnavailable(t)) return;
   const registry = new LeaseRegistry();
   const server = await createDaemonHttpServer({
     token: 'daemon-token',
     leaseRegistry: registry,
+    endSessionsForLease,
     handleRequest: async () => ({ ok: true, data: {} }),
   });
   try {
@@ -99,20 +101,32 @@ test('the route takes only the daemon token and only macos-app leases', async (t
   });
 });
 
-test('DELETE revokes the lease so the next admission fails', async (t) => {
-  await withServer(t, async (origin, registry) => {
-    assert.equal((await put(origin, LEASE_ID, BODY)).status, 200);
-    const removed = await fetch(`${origin}${HOST_LEASE_HTTP_PREFIX}/${LEASE_ID}`, {
-      method: 'DELETE',
-      headers: { authorization: 'Bearer daemon-token' },
-    });
-    assert.equal(((await removed.json()) as { released: boolean }).released, true);
-    assert.throws(
-      () =>
-        registry.assertLeaseAdmission({ leaseId: LEASE_ID, tenantId: 'stim', runId: 'session-1' }),
-      { code: 'UNAUTHORIZED' },
-    );
-  });
+test('DELETE revokes the lease so the next admission fails and ends its session', async (t) => {
+  const ended: string[] = [];
+  await withServer(
+    t,
+    async (origin, registry) => {
+      assert.equal((await put(origin, LEASE_ID, BODY)).status, 200);
+      const removed = await fetch(`${origin}${HOST_LEASE_HTTP_PREFIX}/${LEASE_ID}`, {
+        method: 'DELETE',
+        headers: { authorization: 'Bearer daemon-token' },
+      });
+      assert.equal(((await removed.json()) as { released: boolean }).released, true);
+      assert.throws(
+        () =>
+          registry.assertLeaseAdmission({
+            leaseId: LEASE_ID,
+            tenantId: 'stim',
+            runId: 'session-1',
+          }),
+        { code: 'UNAUTHORIZED' },
+      );
+      assert.deepEqual(ended, [LEASE_ID]);
+    },
+    async (leaseId) => {
+      ended.push(leaseId);
+    },
+  );
 });
 
 test('/health names macos-app among the lease backends only on a macOS host', async (t) => {

packages/daemon/src/server/__tests__/request-execution-scope-lease-session-end.test.ts

@@ -0,0 +1,188 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { expect, test, vi } from 'vitest';
+import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir';
+import { authoringPublication, makeIosSession } from '../../__tests__/support/session-factories.ts';
+import { makeSessionStore } from '../../__tests__/support/store-factory.ts';
+import {
+  isolatedDeviceClaimStores,
+  retainOrphanedDeviceClaims,
+} from '../../__tests__/support/device-claim-store.ts';
+import { acquireDeviceClaim } from '../../resources/device/device-claims.ts';
+import { resolveDeviceClaimPath } from '../../resources/device/device-claim-paths.ts';
+import { existingSessionExecutionLockKeys } from '../request-binding.ts';
+import { withRequestExecutionLockKeys } from '../request-execution-locks.ts';
+import type { SessionStore } from '../../core/session-store.ts';
+import type { SessionTeardown } from '../../sessions/lease-lifecycle.ts';
+import { createLeaseSessionEnd } from '../request-execution-scope.ts';
+
+vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@agent-device/host-kit/diagnostics')>();
+  return { ...actual, emitDiagnostic: vi.fn() };
+});
+
+const claimStores = isolatedDeviceClaimStores('agent-device-lease-end-claims-');
+
+async function publishLeasedSession(
+  sessionStore: SessionStore,
+  name: string,
+  leaseId: string,
+): Promise<string> {
+  const { stateDir } = claimStores();
+  const session = makeIosSession(name, {
+    lease: { leaseId, tenantId: 'tenant-a', runId: 'run-1' },
+  });
+  const acquired = await acquireDeviceClaim({
+    device: session.device,
+    session: name,
+    workspace: stateDir,
+    stateDir,
+    reconcileOrphanedDeviceClaim: retainOrphanedDeviceClaims,
+  });
+  if (acquired.status !== 'acquired') throw new Error('expected an acquired claim');
+  sessionStore.publish(name, { ...session, deviceClaim: acquired.ownership });
+  return resolveDeviceClaimPath(acquired.ownership.deviceKey);
+}
+
+function holdSessionLocks(
+  executionLocks: Map<string, Promise<unknown>>,
+  sessionStore: SessionStore,
+  name: string,
+): { finish: () => void; held: Promise<void> } {
+  let finish!: () => void;
+  const running = new Promise<void>((resolve) => {
+    finish = resolve;
+  });
+  const held = withRequestExecutionLockKeys(
+    executionLocks,
+    existingSessionExecutionLockKeys(name, sessionStore.get(name)!.device.id),
+    async () => await running,
+  );
+  return { finish, held };
+}
+
+function leaseSessionEnd(sessionStore: SessionStore, teardownSession: SessionTeardown) {
+  const executionLocks = new Map<string, Promise<unknown>>();
+  return {
+    executionLocks,
+    ender: createLeaseSessionEnd({ sessionStore, executionLocks, teardownSession }),
+  };
+}
+
+const settle = async (): Promise<void> => await new Promise((resolve) => setImmediate(resolve));
+
+test('end publishes the recorded script of the session it ends', async () => {
+  const sessionStore = makeSessionStore('agent-device-lease-end-script-');
+  await publishLeasedSession(sessionStore, 'leased', 'lease-1');
+  const scriptPath = path.join(mkdtempForTestSync('agent-device-lease-end-ad-'), 'flow.ad');
+  sessionStore.update(sessionStore.lookup('leased')!, {
+    scriptPublication: authoringPublication('armed', { path: scriptPath }),
+    actions: [{ ts: 1, command: 'click', positionals: ['id="recorded-step"'], flags: {} }],
+  });
+
+  await leaseSessionEnd(sessionStore, async () => {}).ender.end('lease-1', 'LEASE_EXPIRED');
+
+  expect(sessionStore.get('leased')).toBeUndefined();
+  expect(fs.readFileSync(scriptPath, 'utf8')).toMatch(/recorded-step/);
+});
+
+test('end tears down the session opened under the lease and clears its device claim', async () => {
+  const sessionStore = makeSessionStore('agent-device-lease-end-');
+  const claimPath = await publishLeasedSession(sessionStore, 'leased', 'lease-1');
+  sessionStore.publish(
+    'other',
+    makeIosSession('other', {
+      lease: { leaseId: 'lease-2', tenantId: 'tenant-b', runId: 'run-2' },
+    }),
+  );
+  expect(fs.existsSync(claimPath)).toBe(true);
+  const teardownSession = vi.fn(async () => {});
+
+  await leaseSessionEnd(sessionStore, teardownSession).ender.end('lease-1', 'LEASE_RELEASED');
+
+  expect(teardownSession).toHaveBeenCalledTimes(1);
+  expect(teardownSession).toHaveBeenCalledWith(expect.objectContaining({ address: 'leased' }));
+  expect(sessionStore.get('leased')).toBeUndefined();
+  expect(fs.existsSync(claimPath)).toBe(false);
+  expect(sessionStore.get('other')).toBeDefined();
+});
+
+test('end clears the claim of a session whose device no longer answers', async () => {
+  const sessionStore = makeSessionStore('agent-device-lease-end-unreachable-');
+  const claimPath = await publishLeasedSession(sessionStore, 'leased', 'lease-1');
+
+  await leaseSessionEnd(sessionStore, async () => {
+    throw new Error('device unreachable');
+  }).ender.end('lease-1', 'LEASE_EXPIRED');
+
+  expect(sessionStore.get('leased')).toBeUndefined();
+  expect(fs.existsSync(claimPath)).toBe(false);
+});
+
+test('end waits for the command still holding the session execution locks', async () => {
+  const sessionStore = makeSessionStore('agent-device-lease-end-locked-');
+  await publishLeasedSession(sessionStore, 'leased', 'lease-1');
+  const teardownSession = vi.fn(async () => {});
+  const { executionLocks, ender } = leaseSessionEnd(sessionStore, teardownSession);
+  const command = holdSessionLocks(executionLocks, sessionStore, 'leased');
+
+  const ending = ender.end('lease-1', 'LEASE_RELEASED');
+  await settle();
+  expect(teardownSession).not.toHaveBeenCalled();
+  expect(sessionStore.get('leased')).toBeDefined();
+
+  command.finish();
+  await command.held;
+  await ending;
+  expect(teardownSession).toHaveBeenCalledTimes(1);
+  expect(sessionStore.get('leased')).toBeUndefined();
+});
+
+test('cancel stops a lease end still queued on the session locks from tearing down', async () => {
+  const sessionStore = makeSessionStore('agent-device-lease-end-cancel-');
+  const claimPath = await publishLeasedSession(sessionStore, 'leased', 'lease-1');
+  const teardownSession = vi.fn(async () => {});
+  const { executionLocks, ender } = leaseSessionEnd(sessionStore, teardownSession);
+  const command = holdSessionLocks(executionLocks, sessionStore, 'leased');
+
+  const ending = ender.end('lease-1', 'LEASE_EXPIRED');
+  await settle();
+  ender.cancel();
+  command.finish();
+  await command.held;
+  await ending;
+
+  expect(teardownSession).not.toHaveBeenCalled();
+  expect(sessionStore.get('leased')).toBeDefined();
+  expect(fs.existsSync(claimPath)).toBe(true);
+  await ender.end('lease-1', 'LEASE_RELEASED');
+  expect(teardownSession).not.toHaveBeenCalled();
+});
+
+test('drain waits for a lease end whose teardown is already running', async () => {
+  const sessionStore = makeSessionStore('agent-device-lease-end-drain-');
+  await publishLeasedSession(sessionStore, 'leased', 'lease-1');
+  let finishTeardown!: () => void;
+  const teardownStarted = vi.fn();
+  const { ender } = leaseSessionEnd(sessionStore, async () => {
+    teardownStarted();
+    await new Promise<void>((resolve) => {
+      finishTeardown = resolve;
+    });
+  });
+
+  void ender.end('lease-1', 'LEASE_EXPIRED');
+  await settle();
+  expect(teardownStarted).toHaveBeenCalledTimes(1);
+  ender.cancel();
+  let drained = false;
+  const drain = ender.drain().then(() => {
+    drained = true;
+  });
+  await settle();
+  expect(drained).toBe(false);
+
+  finishTeardown();
+  await drain;
+  expect(sessionStore.get('leased')).toBeUndefined();
+});

packages/daemon/src/server/host-lease-http.ts

@@ -45,6 +45,7 @@ type HostAdminHttpParams = {
   res: http.ServerResponse;
   expectedToken: string;
   registry: LeaseRegistry;
+  endSessionsForLease?: (leaseId: string) => Promise<void>;
 };
 
 /** Every host administration route: lease allocation and human-control holds. */
@@ -82,6 +83,7 @@ async function handleHostLeaseRoute(
       const released = lease
         ? registry.releaseLease(leaseReleaseRequestFor(lease)).released
         : false;
+      if (lease && released) await params.endSessionsForLease?.(lease.leaseId);
       sendJson(res, { ok: true, released, ...(lease && released ? { lease } : {}) });
       return;
     }

packages/daemon/src/server/http/daemon-runtime.ts

@@ -19,7 +19,11 @@ import { LeaseRegistry } from '../../resources/lease-registry.ts';
 import { createExpiredProviderLeaseReleaser } from '../provider-lease-expiry.ts';
 import { createRequestHandler } from '../request-router.ts';
 import type { RequestRouterDeps } from '../request-router.ts';
-import { getLeaseRegistryExecutionLocks } from '../request-execution-scope.ts';
+import {
+  createLeaseSessionEnd,
+  getLeaseRegistryExecutionLocks,
+} from '../request-execution-scope.ts';
+import type { LeaseEndReason } from '../../sessions/lease-lifecycle.ts';
 import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts';
 import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts';
 import { finalizeDaemonSessionApplicationLifecycle } from '../application-lifecycle-recovery.ts';
@@ -88,6 +92,7 @@ import type {
 import type { HostDiagnostics } from '@agent-device/contracts/host-diagnostics';
 
 const DAEMON_LEASE_RELEASE_TIMEOUT_MS = 1_000;
+const LEASE_EXPIRY_SWEEP_INTERVAL_MS = 1_000;
 const DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS = 1_000;
 const DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS = 2_000;
 // An orphaned `simctl recordVideo` releases the host-wide recording lock only after it finishes
@@ -454,13 +459,22 @@ export async function startDaemonRuntime(
     maxLeaseTtlMs: parseIntegerEnv(env.AGENT_DEVICE_LEASE_MAX_TTL_MS),
     onLeaseExpired: (lease) => {
       void expiredProviderLeaseReleaser.release(lease);
+      void endSessionsForLease(lease.leaseId, 'LEASE_EXPIRED');
     },
   });
+  // The registry only notices an expired lease when something calls it; this sweep ends an
+  // abandoned lease (and, through `onLeaseExpired`, its session and claim) without a later request.
+  const leaseExpirySweep = setInterval(
+    () => leaseRegistry.consumeExpiredLeases(),
+    LEASE_EXPIRY_SWEEP_INTERVAL_MS,
+  );
+  leaseExpirySweep.unref();
   const dispatchRequest = createRequestHandler({
     logPath,
     token,
     sessionStore,
     leaseRegistry,
+    endSessionsForLease: async (leaseId) => await endSessionsForLease(leaseId, 'LEASE_RELEASED'),
     appLogAdmissionLedger,
     audioProbeAdmissionLedger,
     perfCaptureAdmissionLedger,
@@ -558,6 +572,18 @@ export async function startDaemonRuntime(
     // holds the session back to retry. The reaper runs it once the expiry is committed to.
   };
 
+  const leaseSessionEnd = createLeaseSessionEnd({
+    sessionStore,
+    executionLocks: getLeaseRegistryExecutionLocks(leaseRegistry),
+    teardownSession: settleIdleExpiredSession,
+    withinDiagnosticsScope: async (run) => await withDaemonDiagnosticsScope({ logPath }, run),
+  });
+
+  const endSessionsForLease = async (leaseId: string, reason: LeaseEndReason): Promise<void> => {
+    await leaseSessionEnd.end(leaseId, reason);
+    idleReap.noteActivity();
+  };
+
   const sessionIdleExpiry = createSessionIdleExpiry({
     sessionStore,
     idleExpiryMs: resolveSessionIdleExpiryMs(env),
@@ -637,6 +663,8 @@ export async function startDaemonRuntime(
       const httpServer = await createDaemonHttpServer({
         handleRequest,
         leaseRegistry,
+        endSessionsForLease: async (leaseId) =>
+          await endSessionsForLease(leaseId, 'LEASE_RELEASED'),
         token,
         retainArtifacts,
         env,
@@ -810,6 +838,8 @@ export async function startDaemonRuntime(
   const shutdown = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => {
     idleReap.cancel();
     sessionIdleExpiry.cancel();
+    leaseSessionEnd.cancel();
+    clearInterval(leaseExpirySweep);
     if (shuttingDown) return;
     shuttingDown = true;
     stopMetadataLossWatch();
@@ -818,6 +848,7 @@ export async function startDaemonRuntime(
     }
     await closeDaemonServers(servers);
     await Promise.allSettled(inFlightRequests);
+    await leaseSessionEnd.drain();
     // Hand healthy runners off before durable session teardown. The lifecycle gateway later
     // terminates only still-owned generations once all resources have finalized.
     //

packages/daemon/src/server/http/http-server.ts

@@ -70,6 +70,7 @@ const MAX_HTTP_RPC_BODY_BYTES = 1024 * 1024;
 export async function createDaemonHttpServer(options: {
   handleRequest: DaemonInvokeFn;
   leaseRegistry?: LeaseRegistry;
+  endSessionsForLease?: (leaseId: string) => Promise<void>;
   token?: string;
   retainArtifacts?: boolean;
   env?: NodeJS.ProcessEnv;
@@ -126,6 +127,7 @@ export async function createDaemonHttpServer(options: {
         res,
         expectedToken: token,
         registry: options.leaseRegistry,
+        endSessionsForLease: options.endSessionsForLease,
       })
     ) {
       return;