Encode field-observed needs: trace rollup, env-token audit, dangling-ref lint

Three right-layer fixes from the AskRVT field study. The core lint now
catches dangling block-output references (kind block-output) — the
silent class where an API body or agent prompt ships literal template
text while apply reported unresolved: [] and a desk went dark in prod;
pure graph check, every caller, code fields excluded (runtime fails
those loudly), Slack-link/comparison shapes guarded, v2 contract enum
widened. The workflow lint augmentation adds an undeclared {{TOKEN}}
audit (a missing token resolves to empty string at run time — the trap
that masked a production error monitor). New workflow trace <runId>
augmentation rolls up a run: recursive span walk (shallow walks
silently dropped subworkflow blocks), per-type stats, real-error
filtering, slowest blocks. All three live-verified through the agent.

Claude-Session: https://claude.ai/code/session_01CgaxNAaeD3taGdghbXn17w

Author: Sg312

apps/sim/lib/api/contracts/v2/workflows.ts

@@ -2520,7 +2520,7 @@ const v2WorkflowLintSchema = z
             .union([z.string(), z.array(z.string())])
             .describe('The reference, or references, that did not resolve.'),
           kind: z
-            .enum(['credential', 'resource', 'custom-tool', 'mcp-tool', 'skill'])
+            .enum(['credential', 'resource', 'custom-tool', 'mcp-tool', 'skill', 'block-output'])
             .describe('What kind of entity the reference was expected to name.'),
           reason: z.string().describe('Why the reference does not resolve.'),
         })

apps/sim/lib/mothership/tools/handlers/agent-cli/commands/lint.ts

@@ -2,6 +2,7 @@ import type { WorkflowState } from '@sim/workflow-types/workflow'
 import { fetchWorkflowState } from '@/lib/mothership/tools/handlers/agent-cli/commands/workflow-views'
 import {
   type AgentCliCommand,
+  type AgentCliRuntime,
   agentCliFail,
   agentCliOk,
 } from '@/lib/mothership/tools/handlers/agent-cli/types'
@@ -31,9 +32,67 @@ export const workflowLintCommand: AgentCliCommand = {
       workspaceId: runtime.workspaceId,
       subjectUserId: runtime.userId,
     })
+    const undeclaredEnvVars = await collectUndeclaredEnvVars(runtime, graph)
     const summary = hasWorkflowLintIssues(report)
       ? formatWorkflowLintMessage(report)
-      : 'No lint issues found.'
-    return agentCliOk(JSON.stringify({ summary, ...report }, null, 2))
+      : undeclaredEnvVars.length > 0
+        ? `Undeclared environment variables referenced: ${undeclaredEnvVars.map((v) => v.name).join(', ')} — an unresolved {{TOKEN}} resolves to an EMPTY STRING at run time, not an error.`
+        : 'No lint issues found.'
+    return agentCliOk(JSON.stringify({ summary, undeclaredEnvVars, ...report }, null, 2))
   },
 }
+
+const ENV_TOKEN = /\{\{\s*([A-Za-z0-9_]+)\s*\}\}/g
+
+function envTokenNames(value: unknown, out: Map<string, Set<string>>, blockName: string): void {
+  if (typeof value === 'string') {
+    for (const match of value.matchAll(ENV_TOKEN)) {
+      if (!match[1]) continue
+      const blocks = out.get(match[1]) ?? new Set<string>()
+      blocks.add(blockName)
+      out.set(match[1], blocks)
+    }
+  } else if (Array.isArray(value)) {
+    for (const item of value) envTokenNames(item, out, blockName)
+  } else if (typeof value === 'object' && value !== null) {
+    for (const item of Object.values(value)) envTokenNames(item, out, blockName)
+  }
+}
+
+/**
+ * Referenced-but-undeclared {{TOKEN}} audit. Sim resolves a missing token to an
+ * empty string rather than an error, so the failure it causes is silent and
+ * downstream — the exact trap a fleet audit found masking a broken production
+ * error monitor. Declared names come from the same secrets surface the CLI
+ * exposes (workspace + the caller's personal scope).
+ */
+async function collectUndeclaredEnvVars(
+  runtime: AgentCliRuntime,
+  graph: Pick<WorkflowState, 'blocks'>
+): Promise<{ name: string; blocks: string[] }[]> {
+  const referenced = new Map<string, Set<string>>()
+  for (const block of Object.values(graph.blocks ?? {})) {
+    const b = block as { name?: string; subBlocks?: Record<string, { value?: unknown }> }
+    for (const subBlock of Object.values(b.subBlocks ?? {})) {
+      envTokenNames(subBlock?.value, referenced, b.name ?? 'unnamed block')
+    }
+  }
+  if (referenced.size === 0) return []
+  const declared = new Set<string>()
+  let cursor: string | undefined
+  for (let page = 0; page < 10; page++) {
+    const response = await runtime.client.request<{
+      data: { name: string }[]
+      nextCursor: string | null
+    }>('/api/v2/secrets', {
+      query: { workspaceId: runtime.workspaceId, ...(cursor ? { cursor } : {}) },
+    })
+    for (const secret of response.data) declared.add(secret.name)
+    if (!response.nextCursor) break
+    cursor = response.nextCursor
+  }
+  return [...referenced.entries()]
+    .filter(([name]) => !declared.has(name))
+    .map(([name, blocks]) => ({ name, blocks: [...blocks].sort() }))
+    .sort((a, b) => a.name.localeCompare(b.name))
+}

apps/sim/lib/mothership/tools/handlers/agent-cli/commands/trace.ts

@@ -0,0 +1,156 @@
+import {
+  type AgentCliCommand,
+  type AgentCliRuntime,
+  agentCliFail,
+  agentCliOk,
+} from '@/lib/mothership/tools/handlers/agent-cli/types'
+
+/**
+ * `workflow trace <runId>` — a run's trace rolled up for diagnosis, replacing
+ * the hand-written span filters agents otherwise improvise (a field audit found
+ * four divergent jq programs over the same trace, two of which disagreed 7 vs
+ * 21 blocks because a shallow walk silently drops every block inside a child
+ * workflow). The walk here is always recursive, errors come only from status
+ * and error FIELDS (never from schema literals that merely contain the word
+ * "error"), and subworkflow spans keep their nesting depth visible.
+ */
+
+interface TraceSpan {
+  id?: string
+  name?: string
+  type?: string
+  duration?: number
+  durationMs?: number
+  status?: string
+  errorHandled?: boolean
+  errorType?: string
+  errorMessage?: string
+  blockId?: string
+  children?: TraceSpan[]
+}
+
+interface FlatSpan {
+  name: string
+  type: string
+  durationMs: number
+  depth: number
+  status?: string
+  errorMessage?: string
+  errorHandled?: boolean
+}
+
+function flattenSpans(spans: TraceSpan[], depth: number, out: FlatSpan[]): void {
+  for (const span of spans) {
+    out.push({
+      name: span.name ?? span.blockId ?? 'unnamed',
+      type: span.type ?? 'unknown',
+      durationMs: span.durationMs ?? span.duration ?? 0,
+      depth,
+      ...(span.status !== undefined ? { status: span.status } : {}),
+      ...(span.errorMessage !== undefined ? { errorMessage: span.errorMessage } : {}),
+      ...(span.errorHandled !== undefined ? { errorHandled: span.errorHandled } : {}),
+    })
+    if (span.children?.length) flattenSpans(span.children, depth + 1, out)
+  }
+}
+
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0
+  const index = Math.min(sorted.length - 1, Math.floor(p * sorted.length))
+  return sorted[index] ?? 0
+}
+
+export const workflowTraceCommand: AgentCliCommand = {
+  path: ['workflow', 'trace'],
+  summary: 'Roll up one run trace: per-block timings, per-type stats, real errors, slowest path',
+  usage: 'workflow trace <runId>',
+  async execute(rest, runtime: AgentCliRuntime) {
+    const runId = rest[0]
+    if (!runId) return agentCliFail('Usage: sim workflow trace <runId>')
+    const run = await runtime.client.request<{
+      data: {
+        status?: string
+        totalDurationMs?: number
+        trigger?: string
+        workflow?: { id?: string; name?: string }
+        traceSpans?: TraceSpan[]
+      }
+    }>(`/api/v2/logs/${encodeURIComponent(runId)}`)
+    const record = run.data
+    const spans: FlatSpan[] = []
+    flattenSpans(record.traceSpans ?? [], 0, spans)
+    if (spans.length === 0) {
+      return agentCliOk(
+        JSON.stringify(
+          {
+            runId,
+            status: record.status,
+            workflow: record.workflow?.name,
+            note: 'No trace spans (spans age out on their own retention schedule).',
+          },
+          null,
+          2
+        )
+      )
+    }
+
+    const byType = new Map<string, number[]>()
+    for (const span of spans) {
+      const durations = byType.get(span.type) ?? []
+      durations.push(span.durationMs)
+      byType.set(span.type, durations)
+    }
+    const typeStats = [...byType.entries()]
+      .map(([type, durations]) => {
+        const sorted = [...durations].sort((a, b) => a - b)
+        return {
+          type,
+          count: sorted.length,
+          totalMs: sorted.reduce((a, b) => a + b, 0),
+          p50Ms: percentile(sorted, 0.5),
+          maxMs: sorted[sorted.length - 1] ?? 0,
+        }
+      })
+      .sort((a, b) => b.totalMs - a.totalMs)
+
+    const errors = spans
+      .filter((span) => span.errorMessage || (span.status && /^(error|failed)$/i.test(span.status)))
+      .map((span) => ({
+        block: span.name,
+        type: span.type,
+        depth: span.depth,
+        ...(span.status ? { status: span.status } : {}),
+        ...(span.errorMessage ? { message: span.errorMessage.slice(0, 400) } : {}),
+        ...(span.errorHandled !== undefined ? { handled: span.errorHandled } : {}),
+      }))
+
+    const slowest = [...spans]
+      .sort((a, b) => b.durationMs - a.durationMs)
+      .slice(0, 10)
+      .map((span) => ({
+        block: span.name,
+        type: span.type,
+        durationMs: span.durationMs,
+        depth: span.depth,
+      }))
+
+    return agentCliOk(
+      JSON.stringify(
+        {
+          runId,
+          workflow: record.workflow?.name,
+          status: record.status,
+          trigger: record.trigger,
+          totalDurationMs: record.totalDurationMs,
+          blockCount: spans.length,
+          maxDepth: Math.max(...spans.map((span) => span.depth)),
+          errors,
+          typeStats,
+          slowestBlocks: slowest,
+        },
+        null,
+        2
+      )
+    )
+  },
+}

apps/sim/lib/mothership/tools/handlers/agent-cli/index.ts

@@ -5,6 +5,7 @@ import {
   workflowsGrepCommand,
 } from '@/lib/mothership/tools/handlers/agent-cli/commands/grep'
 import { workflowLintCommand } from '@/lib/mothership/tools/handlers/agent-cli/commands/lint'
+import { workflowTraceCommand } from '@/lib/mothership/tools/handlers/agent-cli/commands/trace'
 import {
   workflowBlocksCommand,
   workflowEdgesCommand,
@@ -28,6 +29,7 @@ const AGENT_CLI_COMMANDS: readonly AgentCliCommand[] = [
   workflowEdgesCommand,
   workflowGrepCommand,
   workflowLintCommand,
+  workflowTraceCommand,
   workflowsGrepCommand,
 ]
 

apps/sim/lib/workflows/editing/dangling-refs.test.ts

@@ -0,0 +1,94 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { collectDanglingBlockOutputReferences } from '@/lib/workflows/editing/lint'
+
+function graph(
+  blocks: Record<
+    string,
+    { type?: string; name?: string; subBlocks?: Record<string, { value?: unknown }> }
+  >
+) {
+  return { blocks } as Parameters<typeof collectDanglingBlockOutputReferences>[0]
+}
+
+describe('collectDanglingBlockOutputReferences', () => {
+  it('flags a reference to a deleted block in an API body', () => {
+    const findings = collectDanglingBlockOutputReferences(
+      graph({
+        b1: {
+          type: 'api',
+          name: 'PostBack',
+          subBlocks: { body: { value: '{"spec": "<attachformspec.result>"}' } },
+        },
+      })
+    )
+    expect(findings).toHaveLength(1)
+    expect(findings[0]).toMatchObject({
+      blockId: 'b1',
+      field: 'body',
+      kind: 'block-output',
+      value: ['<attachformspec.result>'],
+    })
+  })
+
+  it('resolves heads by normalized block name, id, and special prefixes', () => {
+    const findings = collectDanglingBlockOutputReferences(
+      graph({
+        b1: { type: 'starter', name: 'Start' },
+        b2: {
+          type: 'api',
+          name: 'Call',
+          subBlocks: {
+            url: { value: 'https://x.test/<start.input>' },
+            body: { value: '<loop.index> and <b1.input> are fine' },
+          },
+        },
+      })
+    )
+    expect(findings).toHaveLength(0)
+  })
+
+  it('ignores non-reference angle text (Slack links, comparisons, bare tags)', () => {
+    const findings = collectDanglingBlockOutputReferences(
+      graph({
+        b1: {
+          type: 'slack',
+          name: 'Notify',
+          subBlocks: {
+            text: { value: 'See <https://sim.ai|the docs> or <b>bold</b>, math a<b.c && d>e' },
+          },
+        },
+      })
+    )
+    expect(findings).toHaveLength(0)
+  })
+
+  it('skips function code fields (the runtime fails those loudly)', () => {
+    const findings = collectDanglingBlockOutputReferences(
+      graph({
+        b1: {
+          type: 'function',
+          name: 'Fn',
+          subBlocks: { code: { value: 'return <ghost.value>' } },
+        },
+      })
+    )
+    expect(findings).toHaveLength(0)
+  })
+
+  it('walks nested values like inputMapping objects', () => {
+    const findings = collectDanglingBlockOutputReferences(
+      graph({
+        b1: {
+          type: 'workflow_input',
+          name: 'Invoke',
+          subBlocks: { inputMapping: { value: { lead: '<ghostblock.output.lead>' } } },
+        },
+      })
+    )
+    expect(findings).toHaveLength(1)
+    expect(findings[0]!.field).toBe('inputMapping')
+  })
+})

apps/sim/lib/workflows/editing/lint-report.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import type { WorkflowState } from '@sim/workflow-types/workflow'
 import {
+  collectDanglingBlockOutputReferences,
   collectWorkflowFieldIssues,
   lintEditedWorkflowState,
   type WorkflowLintReport,
@@ -57,6 +58,11 @@ export async function buildWorkflowLintReport(
 ): Promise<WorkflowLintReport> {
   const unresolvedReferences: WorkflowLintUnresolvedReference[] = []
 
+  // Pure graph check, so it runs for every caller: a dangling block-output
+  // reference passes literal text through at run time on the surfaces that
+  // do not fail loudly (API bodies, agent prompts).
+  unresolvedReferences.push(...collectDanglingBlockOutputReferences(graph))
+
   if (scope.subjectUserId) {
     for (const collect of [collectUnresolvedReferences, collectUnresolvedAgentToolReferences]) {
       try {