Helpthread · zaridan · Jul 19, 2026 · Jul 19, 2026
diff --git a/specs/deploy/gmail-inbound-runbook.md b/specs/deploy/gmail-inbound-runbook.md
@@ -180,8 +180,12 @@ privilege).
 2. `PUBLIC_BASE_URL` = your production URL (e.g. `https://desk.resonantiq.app`),
    matching the OAuth redirect URI (A2.3) and the Pub/Sub push endpoint (A3.4).
    No trailing slash (the composition root strips one defensively either way).
-3. Deploy. `vercel.json` (in the repo) declares the two Vercel Cron jobs:
-   - `*/1 * * * *` → `GET /api/v1/internal/queue/drain` (drain the job queue).
+3. Deploy. `vercel.json` (in the repo) declares three Vercel Cron jobs:
+   - `*/1 * * * *` → `GET /api/v1/internal/queue/drain` (drain the job queue —
+     also delivers webhooks, HT-69: `WEBHOOK_DELIVERY_TOPIC` is handled here).
+   - `*/1 * * * *` → `GET /api/v1/internal/outbox/drain` (HT-69: turn
+     `event_outbox` rows into webhook-delivery queue jobs — a SEPARATE tick
+     from the queue drain above; that one then actually sends them).
    - `0 6 * * *` → `GET /api/v1/internal/cron/watch-maintenance` (daily renewal + sweep; UTC).
    Vercel Cron invokes these as HTTP GETs; the handlers require the
    `CRON_SECRET` (Vercel sends it as a bearer via the `Authorization` header on
@@ -277,9 +281,10 @@ Authorization: Bearer $CRON_SECRET
 
 Answers **`200` when healthy, `503` when any alert is tripped** (body is the
 full JSON report either way: `ok`, `alerts[]`, and per-section detail —
-queue stats, 24h ledger outcome counts, 24h forged-token aggregate, and
-per-mailbox status + Gmail `watch()` expiry). Read-only and cheap — polling
-every minute is fine.
+queue stats, 24h ledger outcome counts, 24h forged-token aggregate,
+per-mailbox status + Gmail `watch()` expiry, and — HT-69 — a `webhooks`
+section: currently `auto_disabled` endpoints and 24h webhook-delivery
+dead-letter count). Read-only and cheap — polling every minute is fine.
 
 **Wiring a monitor:** point any status-code poller that can send one custom
 header (UptimeRobot, Checkly, a `curl -fsS` in a cron you already own) at the
@@ -300,6 +305,8 @@ Each `alerts[]` entry is `<code>: <detail>`. The codes are stable:
 | `forged-token-burst` | ≥ threshold (default 5) stored deliveries in 24h carried reply tokens that FAILED signature verification — someone is guessing/tampering with threading tokens (threading.md §5) | Search Vercel logs for `forged_token_detected` (WARN); review `senderAddress`/`conversationId` across events. The mail itself threaded safely (a forged token never appends) |
 | `mailbox-needs-attention` | A mailbox is `paused` (cursor expired — gmail-push.md §5 rebaseline) or `needs_reconnect` (dead OAuth grant) — **inbound mail is not flowing** | `needs_reconnect`: re-run the Part E consent. `paused`: reconnect to rebaseline the cursor, then check for a gap |
 | `watch-expiring` | An active mailbox's Gmail `watch()` expires in < 72h (or was never armed) — the daily renewal has been failing for days | Function logs for `/internal/cron/watch-maintenance` (`gmail_watch_maintenance` events); a manual `GET` of that endpoint with the cron secret re-arms immediately |
+| `webhook-endpoint-auto-disabled` | HT-69: a webhook endpoint hit 20 consecutive delivery failures and auto-disabled — a module (or an operator's own integration) has silently stopped receiving events | `SELECT id, url, consecutive_failures FROM webhook_endpoints WHERE status = 'auto_disabled'`; fix the receiving side, then `PATCH /api/v1/webhooks/{id}` with `{"status":"active"}` to re-enable (resets the counter) |
+| `webhook-delivery-dead-letter-growth` | HT-69: a webhook delivery exhausted its retries in the last 24h (`WEBHOOK_DELIVERY_TOPIC` on `queue_jobs`) | `SELECT payload, last_error FROM queue_jobs WHERE topic = 'webhook.delivery' AND dead_lettered_at IS NOT NULL ORDER BY dead_lettered_at DESC` — `payload.endpointId` names the endpoint; this can precede (or accompany) an eventual auto-disable |
 
 ### G3. Structured log events (Vercel log search)
 
@@ -308,7 +315,11 @@ ingest outcome: threading decision, append-fallback reason, forgedTokenCount,
 parse size, attachment count, ledger outcome), `forged_token_detected` (WARN
 — the per-message security event behind `forged-token-burst`), `queue_drain`
 (per drain tick that claimed work or fenced a stale worker: claimed/acked/
-retried/deadLettered/staleSkipped; quiet ticks don't log), `gmail_reconcile`
+retried/deadLettered/staleSkipped; quiet ticks don't log — this is also
+where webhook-delivery attempts surface, since `WEBHOOK_DELIVERY_TOPIC` is
+handled by the SAME drain), `outbox_drain` (HT-69: per outbox-drain tick
+that claimed at least one `event_outbox` row — claimed/enqueued/dispatched;
+quiet ticks don't log, same convention as `queue_drain`), `gmail_reconcile`
 (per reconcile job: cursor positions, skip/retry/ack reasons), and
 `gmail_watch_maintenance` (the daily renewal + sweep). Correlate transport
 events to ingest events on `(mailboxId, providerMessageId)`

diff --git a/src/api/agents.test.ts b/src/api/agents.test.ts
@@ -8,6 +8,7 @@
  * directly.
  */
 
+import { randomBytes } from 'node:crypto'
 import { afterEach, describe, expect, it } from 'vitest'
 import { mintInviteToken } from '../auth/invite-token.js'
 import { hashPassword } from '../auth/password-hash.js'
@@ -19,8 +20,13 @@ import type { EmailSender, OutboundEmail } from '../providers/index.js'
 import { type AgentRecord, type AgentStore, createAgentStore } from '../store/agents.js'
 import { createConversationStore } from '../store/conversations.js'
 import { createMailboxStore, type MailboxStore } from '../store/mailboxes.js'
+import { ENCRYPTION_KEY_BYTES } from '../store/token-crypto.js'
+import { createWebhookEndpointStore } from '../store/webhook-endpoints.js'
 import { createInboxApi } from './index.js'
 
+/** None of this suite's tests exercise `/webhooks/*` — a real PGlite-backed store plus a no-op queue is just enough for `createInboxApi` to construct (HT-69's `webhooks` deps are now REQUIRED, mirroring `agents`). */
+const WEBHOOKS_ENC_KEY = randomBytes(ENCRYPTION_KEY_BYTES)
+
 const TOKEN = 'test-token-for-the-agents-and-auth-suite'
 const MAIL_DOMAIN = 'mail.example.test'
 const SUPPORT_ADDRESS = 'support@example.test'
@@ -86,6 +92,10 @@ describe('Agents & Authentication API', () => {
         mailboxStore,
         ...(overrides.uiBaseUrl !== undefined ? { uiBaseUrl: overrides.uiBaseUrl } : {}),
       },
+      webhooks: {
+        store: createWebhookEndpointStore(db, WEBHOOKS_ENC_KEY),
+        queue: { async enqueue() {} },
+      },
     })
     return { db, agentStore, mailboxStore, api, sent }
   }

diff --git a/src/api/index.test.ts b/src/api/index.test.ts
@@ -26,9 +26,11 @@ import { createGmailWatchStateStore } from '../store/gmail-watch-state.js'
 import { createMailboxTokenStore } from '../store/mailbox-tokens.js'
 import { createMailboxStore, type MailboxStore } from '../store/mailboxes.js'
 import { ENCRYPTION_KEY_BYTES } from '../store/token-crypto.js'
+import { createWebhookEndpointStore } from '../store/webhook-endpoints.js'
 import type { AgentsApiDeps } from './agents.js'
 import type { GmailReconcileJob } from './gmail-webhook.js'
 import { createInboxApi, type InboxApiDeps } from './index.js'
+import type { WebhooksApiDeps } from './webhooks.js'
 
 const TOKEN_ENC_KEY = randomBytes(ENCRYPTION_KEY_BYTES)
 
@@ -56,6 +58,21 @@ function testAgentsDeps(db: Db): AgentsApiDeps {
   }
 }
 
+/**
+ * Build the REQUIRED `webhooks` deps (HT-69) for a `createInboxApi` call
+ * wired to `db` — a real PGlite-backed `WebhookEndpointStore` plus a
+ * no-op `QueueProvider` (nothing in this suite exercises delivery; that is
+ * `src/webhooks/*.test.ts`'s and `src/api/webhooks.test.ts`'s job). Just
+ * enough for `createInboxApi` to construct and for the existing routes
+ * this suite covers to behave unchanged.
+ */
+function testWebhooksDeps(db: Db): WebhooksApiDeps {
+  return {
+    store: createWebhookEndpointStore(db, TOKEN_ENC_KEY),
+    queue: { async enqueue() {} },
+  }
+}
+
 /** A fake `EmailSender` that records every `OutboundEmail` it's asked to send, never fails. */
 function createFakeSender(): { sender: EmailSender; sent: OutboundEmail[] } {
   const sent: OutboundEmail[] = []
@@ -253,6 +270,7 @@ describe('createInboxApi', () => {
       mailDomain: MAIL_DOMAIN,
       supportAddress: SUPPORT_ADDRESS,
       agents: agentsDeps,
+      webhooks: testWebhooksDeps(db),
       ...(overrides.openTracking !== undefined ? { openTracking: overrides.openTracking } : {}),
       ...(overrides.gmailPush !== undefined ? { gmailPush: overrides.gmailPush } : {}),
       ...(overrides.gmailConnect !== undefined ? { gmailConnect: overrides.gmailConnect } : {}),
@@ -702,6 +720,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
       })
 
       const res = await api(
@@ -809,6 +828,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
       })
 
       const res = await api(
@@ -1014,6 +1034,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
       })
 
       const res = await api(
@@ -1926,6 +1947,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
         gmailPush: {
           verifySignature: async () => true,
           subscription: SUBSCRIPTION,
@@ -1957,6 +1979,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
         gmailPush: {
           verifySignature: async () => true,
           subscription: SUBSCRIPTION,
@@ -2092,6 +2115,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
         ...(gmailConnect !== undefined ? { gmailConnect } : {}),
       })
     }
@@ -2325,6 +2349,7 @@ describe('createInboxApi', () => {
         mailDomain: MAIL_DOMAIN,
         supportAddress: SUPPORT_ADDRESS,
         agents: testAgentsDeps(db),
+        webhooks: testWebhooksDeps(db),
         ...(gmailDisconnect !== undefined ? { gmailDisconnect } : {}),
       })
     }
@@ -2425,6 +2450,13 @@ describe('createInboxApi — hardening (Codex review)', () => {
       providers: [],
       mailboxStore: {} as unknown as MailboxStore,
     } satisfies AgentsApiDeps,
+    // Same "never invoked in this block" reasoning as `agents` above — these
+    // tests are purely about construction-time validation and the
+    // conversations-route error paths, never /webhooks/*.
+    webhooks: {
+      store: {} as unknown as WebhooksApiDeps['store'],
+      queue: {} as unknown as WebhooksApiDeps['queue'],
+    } satisfies WebhooksApiDeps,
   }
 
   it('throws at construction on an empty apiToken (fail closed — an empty token would authenticate every request)', () => {

diff --git a/src/api/index.ts b/src/api/index.ts
@@ -81,6 +81,14 @@ import {
   matchOpenTrackingPixel,
   matchRoute,
 } from './router.js'
+import {
+  handleCreateWebhook,
+  handleDeleteWebhook,
+  handleListWebhooks,
+  handlePatchWebhook,
+  handleTestWebhook,
+  type WebhooksApiDeps,
+} from './webhooks.js'
 
 /**
  * Minimum length for the service Bearer token. A short/empty token is a
@@ -195,6 +203,14 @@ export interface InboxApiDeps {
    * for the conceded race.
    */
   selfEchoGuard?: SelfEchoGuardDeps
+  /**
+   * The webhooks admin API (HT-69; specs/modules/substrate-v1.md §5) —
+   * REQUIRED, like `agents`: this is core substrate ("free forever", spec
+   * §1), not a deployment-specific optional feature like `openTracking`/
+   * `gmailPush`. See `src/api/webhooks.ts`'s module doc for the full
+   * `POST`/`GET`/`PATCH`/`DELETE`/`.../test` surface this wires up.
+   */
+  webhooks: WebhooksApiDeps
 }
 
 /**
@@ -494,6 +510,43 @@ export function createInboxApi(deps: InboxApiDeps): (request: Request) => Promis
             request,
             deps.agents,
           )
+
+        // --- Webhooks admin API (HT-69; specs/modules/substrate-v1.md §5) ---
+
+        case 'webhooks-list':
+          return await handleListWebhooks(
+            await resolveActingAgent(request, deps.agents.store),
+            deps.webhooks,
+          )
+
+        case 'webhooks-create':
+          return await handleCreateWebhook(
+            await resolveActingAgent(request, deps.agents.store),
+            request,
+            deps.webhooks,
+          )
+
+        case 'webhook-patch':
+          return await handlePatchWebhook(
+            route.id,
+            await resolveActingAgent(request, deps.agents.store),
+            request,
+            deps.webhooks,
+          )
+
+        case 'webhook-delete':
+          return await handleDeleteWebhook(
+            route.id,
+            await resolveActingAgent(request, deps.agents.store),
+            deps.webhooks,
+          )
+
+        case 'webhook-test':
+          return await handleTestWebhook(
+            route.id,
+            await resolveActingAgent(request, deps.agents.store),
+            deps.webhooks,
+          )
       }
     } catch (err) {
       console.error('[inbox-api] unhandled error handling request', err)

diff --git a/src/api/router.ts b/src/api/router.ts
@@ -164,6 +164,30 @@ const AGENT_MAILBOXES: RouteDef = {
   methods: ['GET', 'PUT'],
 }
 
+// --- Webhooks admin API (HT-69; specs/modules/substrate-v1.md §5) -----------
+//
+// Admin-only, acting-Agent header REQUIRED on every route (`src/api/
+// webhooks.ts`'s module doc) — same Bearer-gated-ordinary-route shape as
+// Agents & Authentication above, no pre-auth carve-out.
+
+/** `/api/v1/webhooks` — list (GET) and register (POST), both admin only — spec §5. */
+const WEBHOOKS_LIST: RouteDef = {
+  pattern: /^\/api\/v1\/webhooks$/,
+  methods: ['GET', 'POST'],
+}
+
+/** `/api/v1/webhooks/{id}` — patch/delete (admin only) — spec §5. Anchored `[^/]+$` so it never matches a `.../test` suffix, mirroring `AGENT_ITEM`'s own anchoring against `.../password`/`.../invite`/`.../mailboxes`. */
+const WEBHOOK_ITEM: RouteDef = {
+  pattern: /^\/api\/v1\/webhooks\/(?<id>[^/]+)$/,
+  methods: ['PATCH', 'DELETE'],
+}
+
+/** `/api/v1/webhooks/{id}/test` — fire a synthetic `test.ping` through the real delivery path (admin only) — spec §5, POST only. */
+const WEBHOOK_TEST: RouteDef = {
+  pattern: /^\/api\/v1\/webhooks\/(?<id>[^/]+)\/test$/,
+  methods: ['POST'],
+}
+
 /** Every route this API recognizes, checked in order. */
 const ROUTES: readonly RouteDef[] = [
   CONVERSATIONS_LIST,
@@ -185,6 +209,9 @@ const ROUTES: readonly RouteDef[] = [
   MAILBOXES_LIST,
   AGENT_MAILBOXES,
   AGENT_ITEM,
+  WEBHOOKS_LIST,
+  WEBHOOK_TEST,
+  WEBHOOK_ITEM,
 ]
 
 /** The outcome of matching a `(method, pathname)` pair against {@link ROUTES}. */
@@ -214,6 +241,11 @@ export type RouteMatch =
   | { kind: 'mailboxes-list' }
   | { kind: 'agent-mailboxes-get'; id: string }
   | { kind: 'agent-mailboxes-put'; id: string }
+  | { kind: 'webhooks-list' }
+  | { kind: 'webhooks-create' }
+  | { kind: 'webhook-patch'; id: string }
+  | { kind: 'webhook-delete'; id: string }
+  | { kind: 'webhook-test'; id: string }
   | { kind: 'method-not-allowed'; allow: string[] }
   | { kind: 'not-found' }
 
@@ -339,6 +371,9 @@ export function matchRoute(method: string, pathname: string): RouteMatch {
     if (route === MAILBOXES_LIST) {
       return { kind: 'mailboxes-list' }
     }
+    if (route === WEBHOOKS_LIST) {
+      return method === 'GET' ? { kind: 'webhooks-list' } : { kind: 'webhooks-create' }
+    }
 
     // Every remaining route guarantees a present, non-empty `id` group (per
     // its `[^/]+` pattern) whenever it matched.
@@ -366,6 +401,13 @@ export function matchRoute(method: string, pathname: string): RouteMatch {
       if (method === 'GET') return { kind: 'agent-mailboxes-get', id }
       return { kind: 'agent-mailboxes-put', id }
     }
+    if (route === WEBHOOK_TEST) {
+      return { kind: 'webhook-test', id }
+    }
+    if (route === WEBHOOK_ITEM) {
+      if (method === 'DELETE') return { kind: 'webhook-delete', id }
+      return { kind: 'webhook-patch', id }
+    }
     if (route === AGENT_ITEM) {
       if (method === 'GET') return { kind: 'agent-item', id }
       if (method === 'DELETE') return { kind: 'agent-delete', id }