fix(gateway): normalize transport-closed error matching

rmcp Display strings vary (Transport closed, channel closed, unexpected end of stream). Tokenize instead of listing one more substring.

Signed-off-by: crimsonsunset <jsangio1@gmail.com>

Author: crimsonsunset

crates/mcpmux-gateway/src/pool/routing.rs

@@ -945,27 +945,50 @@ impl RoutingService {
 
     /// Check if an error string indicates authentication is needed
     fn is_auth_error(error_str: &str) -> bool {
+        let lower = error_str.to_ascii_lowercase();
         let indicators = [
             "401",
             "unauthorized",
             "invalid_token",
             "token expired",
             "access token",
         ];
-        indicators.iter().any(|s| error_str.contains(s))
+        indicators.iter().any(|s| lower.contains(s))
     }
 
     /// Check if an error string indicates the backend transport died.
     ///
-    /// Live rmcp stdio kill (Aug 20) stringified as `MCP call failed: Transport closed`.
+    /// Matches after case/punctuation/camelCase normalize so rmcp Display
+    /// variants (`Transport closed`, `connection closed: …`, `Transport channel
+    /// closed`, `unexpected end of stream`) and MCP `-32000` all hit the same
+    /// path. Auth is classified first by [`reconnect_path_for_error`].
     fn is_transport_closed_error(error_str: &str) -> bool {
-        let indicators = [
-            "connection closed",
-            "-32000",
+        let lower = error_str.to_ascii_lowercase();
+        if lower.contains("-32000") {
+            return true;
+        }
+        let tokens = normalize_error_tokens(error_str);
+        const PHRASES: &[&str] = &[
             "transport closed",
-            "transport channel closed",
+            "connection closed",
+            "channel closed",
+            "transport terminated",
+            "handler terminated",
+            "unexpected end of stream",
+            "session expired",
+            "broken pipe",
+            "connection reset",
+            "keep alive timeout",
+            "transport send error",
+            "send message error",
         ];
-        indicators.iter().any(|s| error_str.contains(s))
+        if PHRASES.iter().any(|p| tokens.contains(p)) {
+            return true;
+        }
+        tokens.contains("closed")
+            && ["transport", "connection", "channel", "stream"]
+                .iter()
+                .any(|noun| tokens.contains(noun))
     }
 
     /// Check if tool result content contains authentication error indicators.
@@ -1010,7 +1033,34 @@ fn failure_trigger_label(path: Option<ReconnectPath>) -> &'static str {
         .unwrap_or("unmatched")
 }
 
-/// Classify a lowercased transport error for retry routing.
+/// Collapse camelCase and punctuation so `TransportClosed` / `transport_closed`
+/// / `transport-closed` all become `transport closed`.
+fn normalize_error_tokens(error_str: &str) -> String {
+    let mut with_camel = String::with_capacity(error_str.len() + 8);
+    let mut prev_was_lower = false;
+    for c in error_str.chars() {
+        if c.is_ascii_uppercase() && prev_was_lower {
+            with_camel.push(' ');
+        }
+        with_camel.push(c);
+        prev_was_lower = c.is_ascii_lowercase();
+    }
+
+    let mut out = String::with_capacity(with_camel.len());
+    let mut prev_space = false;
+    for c in with_camel.chars() {
+        if c.is_ascii_alphanumeric() {
+            out.push(c.to_ascii_lowercase());
+            prev_space = false;
+        } else if !prev_space {
+            out.push(' ');
+            prev_space = true;
+        }
+    }
+    out
+}
+
+/// Classify a transport error for retry routing. Case-insensitive.
 fn reconnect_path_for_error(err_str: &str) -> Option<ReconnectPath> {
     if RoutingService::is_auth_error(err_str) {
         Some(ReconnectPath::OAuth)
@@ -1079,11 +1129,35 @@ mod call_failure_classify_tests {
 
     #[test]
     fn rmcp_stdio_transport_closed_is_not_unmatched() {
-        let err = "mcp call failed: transport closed";
+        let err = "MCP call failed: Transport closed";
         assert_eq!(reconnect_path_for_error(err), Some(ReconnectPath::Fresh));
         assert!(!RoutingService::is_auth_error(err));
     }
 
+    #[test]
+    fn transport_closed_shapes_normalize_to_fresh() {
+        let cases = [
+            "TransportClosed",
+            "transport_closed",
+            "transport-closed",
+            "Transport channel closed",
+            "connection closed: initialize response",
+            "unexpected end of stream",
+            "broken pipe",
+            "Session expired (HTTP 404)",
+            "keep alive timeout after 30000ms",
+            "connection reset by peer",
+            "the transport has been closed",
+        ];
+        for err in cases {
+            assert_eq!(
+                reconnect_path_for_error(err),
+                Some(ReconnectPath::Fresh),
+                "{err}"
+            );
+        }
+    }
+
     #[test]
     fn auth_error_still_routes_to_oauth_reconnect() {
         let err = "401 unauthorized: token expired";
@@ -1099,4 +1173,16 @@ mod call_failure_classify_tests {
         assert!(!RoutingService::is_transport_closed_error(err));
         assert!(!RoutingService::is_auth_error(err));
     }
+
+    #[test]
+    fn closed_without_transport_noun_is_unmatched() {
+        assert!(reconnect_path_for_error("file closed").is_none());
+        assert!(reconnect_path_for_error("request timeout after 60s").is_none());
+    }
+
+    #[test]
+    fn auth_wins_when_error_also_looks_closed() {
+        let err = "401 unauthorized: connection closed";
+        assert_eq!(reconnect_path_for_error(err), Some(ReconnectPath::OAuth));
+    }
 }

docs/planning/backend-connection-resilience-test.md

@@ -1,34 +1,48 @@
 # Backend Connection Resilience — Manual Test Playbook
 
-**For:** any agent (or human) verifying `c09e569` / `root-resolution`
+**For:** re-running verification on `root-resolution`
 **Last Updated:** Aug 20, 2026
 **Implements:** [`backend-connection-resilience.md`](./backend-connection-resilience.md)
-**Do not implement code.** Execute the cases below, record pass/fail, stop.
+**Shipped:** `c09e569` (retry + FK guard), `54d0de2` (literal `transport closed`). Matcher now also normalizes rmcp Display variants (camelCase / punctuation).
+
+---
+
+## Results (Aug 20, 2026)
+
+| Case | Result |
+| ---- | ------ |
+| A bind FK | **PASS** — `invalid_argument` + `mcpmux_list_feature_sets`; 38/74 counts unchanged |
+| B stdio reconnect | **PASS** — `wakatime` / `wakatime_wakatime_summaries`. Kill child → invoke succeeded. Log: `trigger=transport_closed`, `reconnect_fresh completed ok=true`. Live error: `MCP call failed: Transport closed` |
+| C HA idle | **SKIPPED** |
+| D unmatched | **SKIPPED** — grant-layer "did you mean", never reached classifier |
+
+Confounders: Tauri-watch rebuild wiped inbound sessions (`POST /mcp` → 404 until MCP reload). After reload, 6 unpinned roots required one `mcpmux_set_workspace_root` to *this* repo before invoke was possible. That pin was session disambiguation, not the reconnect path.
 
 ---
 
 ## Answer first: new sessions in different roots?
 
-**No. Do not open a new Cursor chat, do not switch workspace roots, do not call `mcpmux_set_workspace_root`.**
+**No. Do not open a new Cursor chat, do not switch workspace roots, do not call `mcpmux_set_workspace_root` to "fix" a closed connection.**
 
-That was the old workaround. It opens a fresh *inbound* rmcp session and hides the outbound-pool bug. Using it here makes Case B/C inconclusive.
+That was the old workaround. It opens a fresh *inbound* rmcp session and hides the outbound-pool bug. Using it as the Case B/C recovery makes those cases inconclusive.
 
 | Action | Use for this test? |
 | --- | --- |
 | Same Cursor chat, same workspace (`/Users/joe/Desktop/Repos/Personal/mcp-mux`) | **Yes — required** |
-| Reload Cursor MCP tools (once, before starting, if the gateway just rebuilt) | Yes, once |
-| New chat / different root / `set_workspace_root` | **No** (confounds reconnect) |
+| Reload Cursor MCP tools (once, if the gateway just rebuilt) | Yes, once, *before* Case B/C |
+| `set_workspace_root` to this repo after reload, **only if** `mcpmux_list_servers` says multiple roots are unpinned | Yes — otherwise nothing is `ready` |
+| `set_workspace_root` after a transport-closed invoke | **No** |
 | Bind a real FeatureSet onto another repo | **No** (Case A uses a fake UUID; no write) |
 
-Bindings are keyed by exact `workspace_root` + `machine_id`, not by chat. This repo already has two rows for the same path (Gondor vs Rohan), both on FeatureSet `All`. Stay here.
+Bindings are keyed by exact `workspace_root` + `machine_id`, not by chat. Stay in this repo.
 
 ---
 
 ## Snapshot (live DB, Aug 20 2026)
 
 ```
 DB=~/Library/Application\ Support/com.mcpmux.desktop/mcpmux.db
-LOG=~/Library/Application\ Support/com.mcpmux.desktop/logs/mcpmux.2026-08-20.log
+LOG=~/Library/Application\ Support/com.mcpmux.desktop/logs/mcpmux.$(date +%Y-%m-%d).log
 ```
 
 | Fact | Value |
@@ -37,8 +51,8 @@ LOG=~/Library/Application\ Support/com.mcpmux.desktop/logs/mcpmux.2026-08-20.log
 | This root | `/Users/joe/Desktop/Repos/Personal/mcp-mux` |
 | Binding (Gondor `ec211deb…`) | `5a588b93-5ed1-4ada-b7cb-8e32a9f11058` → FeatureSet `All` (`fs_default_00000000-0000-0000-0000-000000000001`) |
 | Binding (Rohan `5d581ac9…`) | `8e2b36b6-eeff-4818-9259-948c1b9c3b6b` → `All` |
-| HA backend | `home-assistant-new` (HTTP, enabled, 95 features) |
-| Binding row count | 38 parents / 74 junction rows (re-count before Case A) |
+| HA backend | `home-assistant-new` (HTTP) |
+| Stdio used for B | `wakatime` |
 | Pool stats | **in-memory only** — `consecutive_failures` will not appear in SQLite |
 
 Re-count before Case A (numbers drift):
@@ -53,10 +67,11 @@ sqlite3 "$HOME/Library/Application Support/com.mcpmux.desktop/mcpmux.db" \
 
 ## Preconditions
 
-1. Debug gateway is the listener on `:45818` (ancestor = `launchd`, not Cursor Helper). Health: `curl -sf http://127.0.0.1:45818/health` → `{"status":"ok","version":"0.5.0"}`.
-2. Cursor MCP `user-mcpmux` points at `http://localhost:45818/mcp`. Reload tools **once** if the binary was just rebuilt, then do not reload again.
-3. Only call mux via `user-mcpmux` (`mcpmux_search_tools` → `mcpmux_get_tool_schema` if needed → `mcpmux_invoke_tool`). No direct backend MCP servers.
-4. Do not run `pnpm dev:stop` / rebuild mid-test (evicts the pool and invalidates Case B/C).
+1. Debug gateway is the listener on `:45818`. Health: `curl -sf http://127.0.0.1:45818/health` → `{"status":"ok",…}`.
+2. Cursor MCP `user-mcpmux` points at `http://localhost:45818/mcp`. Reload tools **once** if the binary was just rebuilt, then do not reload again mid-case.
+3. Only call mux via `user-mcpmux`. No direct backend MCP servers.
+4. Do not run `pnpm dev:stop` / rebuild mid-test (evicts the pool *and* all inbound sessions).
+5. If `mcpmux_list_servers` reports several unpinned roots, pin `/Users/joe/Desktop/Repos/Personal/mcp-mux` once, then start Case A/B.
 
 ---
 
@@ -65,47 +80,38 @@ sqlite3 "$HOME/Library/Application Support/com.mcpmux.desktop/mcpmux.db" \
 **Goal:** a nonexistent `feature_set_id` returns `invalid_argument`, not `FOREIGN KEY constraint failed`, and the DB does not grow.
 
 1. Snapshot counts (query above). Call them `B0` / `J0`.
-2. Call `mcpmux_bind_current_workspace` with `feature_set_id` = `00000000-0000-0000-0000-00000000dead` (or any other unused UUID). Do **not** approve anything — the guard runs before consent.
+2. Call `mcpmux_bind_current_workspace` with `feature_set_id` = `00000000-0000-0000-0000-00000000dead`. Do **not** approve anything — the guard runs before consent.
 3. **Pass** if the tool error JSON has `"error":"invalid_argument"` and the message contains `mcpmux_list_feature_sets`.
 4. **Fail** if the message contains `FOREIGN KEY`, `internal_error`, or `constraint`.
 5. Re-count. `B0` and `J0` must be unchanged.
-6. Grep the log:
-
-```bash
-rg "bind_current_workspace rejected" \
-  "$HOME/Library/Application Support/com.mcpmux.desktop/logs/mcpmux.$(date +%Y-%m-%d).log"
-```
-
-Expect a `warn` with the fake `feature_set_id` and this Space id.
+6. Grep: `rg "bind_current_workspace rejected" "$LOG"`
 
 ---
 
 ## Case B — reconnect after a killed stdio child (~5 min)
 
 **Goal:** a transport-closed error triggers `reconnect_fresh` (not OAuth `reconnect_instance`) and the retry succeeds. Same session.
 
-Pick a **stdio** server that is already invokable under `All`. Cheap options on this machine: `wakatime`, `markitdown`, `chrome-devtools`. Confirm first:
+Pick a **stdio** server that is `ready` under `All`. Cheap options: `wakatime`, `markitdown`, `chrome-devtools`.
 
 ```
 mcpmux_search_tools({ "server_id": "wakatime", "mode": "browse", "limit": 5 })
 ```
 
-If `server_readiness` is not `ready`, pick another stdio id from that browse, or `mcpmux_list_servers` and take one with `ready`.
+If `server_readiness` is not `ready`, pick another stdio id, or pin this root first (see Preconditions).
 
-1. Invoke a cheap read-only tool once so the instance is live. Example (only if search returned it): `mcpmux_invoke_tool` `server_id=wakatime` `tool=wakatime_wakatime_summaries` with explicit `start`/`end` dates for today. Any successful call is enough.
+1. Invoke a cheap read-only tool once so the instance is live. `wakatime`: `mcpmux_invoke_tool` `server_id=wakatime` `tool=wakatime_wakatime_summaries` with explicit `start`/`end` for today (qualified name; bare `wakatime_summaries` can fail invoke).
 2. Note the time (`date -u +%H:%M:%S`).
 3. Kill **only the child**, not McpMux:
 
 ```bash
-# find the stdio child (example: wakatime). Do not kill target/debug/mcpmux.
 pgrep -lf wakatime
 # then: kill <child-pid>
+# Do not kill target/debug/mcpmux. Do not `pkill -f mcpmux`.
 ```
 
-If you cannot identify a safe child, stop and report INCONCLUSIVE. Do not `pkill -f mcpmux`.
-
 4. In **this same chat**, invoke the same tool again. Do not call `set_workspace_root`.
-5. **Pass** if the invoke succeeds (or returns a normal tool error, not `-32000` / `Connection closed`).
+5. **Pass** if the invoke succeeds (or a normal tool error, not raw `Transport closed` / `-32000` to the agent).
 6. Grep from the timestamp in step 2:
 
 ```bash
@@ -115,23 +121,19 @@ rg "backend call_tool failed|reconnect attempted after call_tool failure|reconne
 
 **Pass logs:**
 
-- `backend call_tool failed` with `trigger=transport_closed` (or `auth` only if the error was actually 401)
+- `backend call_tool failed` with `trigger=transport_closed` (stdio kill has been `error=mcp call failed: transport closed`)
 - `reconnect attempted after call_tool failure` with `ok=true`
-- `reconnect_fresh completed` with `ok=true` (this is the new path; `reconnect_instance` / `Reconnecting instance ... after OAuth` must **not** be the line for this failure)
+- `reconnect_fresh completed` with `ok=true` (`reconnect_instance` / OAuth must **not** be the line for this failure)
 
-**Fail:** invoke returns `MCP error -32000: Connection closed` to the agent, or the log shows OAuth reconnect for a stdio kill.
+**Fail:** invoke returns `Transport closed` / `-32000` / `Connection closed` to the agent, or `trigger=unmatched`, or OAuth reconnect for a stdio kill.
 
 ---
 
 ## Case C — original HA idle (optional, 15–20 min)
 
-**Goal:** reproduce the reported HTTP shape against `home-assistant-new`.
+**Goal:** the reported HTTP shape against `home-assistant-new`.
 
-1. Search: `mcpmux_search_tools({ "server_id": "home-assistant-new", "mode": "browse", "limit": 5 })`.
-2. Invoke one cheap read-only HA tool. Confirm success.
-3. Wait 15–20 minutes. Do not invoke that server. Do not reload MCP. Do not re-pin the workspace. Other mux tools are fine.
-4. Invoke the **same** HA tool again in this chat.
-5. **Pass / fail / log checks** are identical to Case B, except `server_id=home-assistant-new` and the error string historically was `MCP error -32000: Connection closed`.
+Same as B except `server_id=home-assistant-new`, wait 15–20 min with no traffic to that server, and the historical error was `MCP error -32000: Connection closed`.
 
 If you cannot wait, mark Case C SKIPPED and rely on B.
 
@@ -142,7 +144,7 @@ If you cannot wait, mark Case C SKIPPED and rely on B.
 **Goal:** a failure that is neither auth nor transport-closed is not swallowed.
 
 1. `mcpmux_invoke_tool` against a ready server with a tool name that does not exist, e.g. `server_id=wakatime` `tool=definitely_not_a_real_tool`.
-2. **Pass** if the raw error comes back to the caller (not a silent success, not a reconnect).
+2. **Pass** if the raw error comes back (not a silent success, not a reconnect).
 3. If the failure is classified unmatched, the log line is `trigger=unmatched` and there is **no** `reconnect attempted after call_tool failure` for that call.
 
 A permission / not-found error that never hits the backend is also acceptable — note it as "never reached classifier" rather than fail.
@@ -172,7 +174,7 @@ Case B stdio reconnect: PASS | FAIL | INCONCLUSIVE | SKIPPED
   server_id / tool:
   first invoke: ok/err
   child kill: pid / skipped why
-  second invoke: ok / raw -32000 / other
+  second invoke: ok / raw Transport closed / other
   trigger= :
   reconnect_fresh ok= :
   oauth reconnect used: yes/no