feat(kernel): recall-aware matcher scoring + transparency (FT-K1/K3)

scoreEntry blended score max(coverage, matched/ABSOLUTE_K) + patternBonus (ABSOLUTE_K=5): a genuine 2-3 keyword match on a keyword-rich entry is now reachable (was matched/total, which made 30-keyword entries need 7+ hits to clear a 0.3 floor), while single incidental hits stay <=0.2. Additive transparency: optional matchLoadout(task,index,{minScore}), exported DEFAULT_MIN_SCORE, and MatchResult.scoreComponents. Public API back-compat (only an additive optional param). Calibration vs the live 336-entry index: game canon 0.15->0.6-0.8 (recall gap fixed), generic-word noise ~0.4 -> recommended floor re-derives to 0.5 when this ships at republish (live hook unchanged for now, still on published 1.4.3).

Tests: kernel 117->126.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mcp-tool-shop

packages/kernel/src/index.ts

@@ -8,6 +8,7 @@ export type {
   LoadoutIndex,
   Frontmatter,
   MatchResult,
+  ScoreComponents,
   UsageEvent,
   MergeConflict,
   MergedIndex,
@@ -24,7 +25,8 @@ export { parseFrontmatter, serializeFrontmatter } from "./frontmatter.js";
 export { estimateTokens } from "./tokens.js";
 
 // ── Matcher ────────────────────────────────────────────────────
-export { matchLoadout, lookupEntry } from "./match.js";
+export { matchLoadout, lookupEntry, DEFAULT_MIN_SCORE } from "./match.js";
+export type { MatchOptions } from "./match.js";
 
 // ── Validator ──────────────────────────────────────────────────
 export { validateIndex } from "./validate.js";

packages/kernel/src/match.ts

@@ -6,15 +6,65 @@
  *
  * Matching is deterministic:
  * - Tokenizes the task into lowercase words
- * - Scores each entry by keyword and pattern overlap
+ * - Scores each entry by keyword and pattern overlap (recall-aware)
  * - Returns entries above a minimum score threshold
  * - Core entries are always included (score 1.0)
  * - Manual entries are never auto-included (require explicit lookup)
  */
 
-import type { LoadoutIndex, LoadoutEntry, MatchResult, LoadMode } from "./types.js";
+import type {
+  LoadoutIndex,
+  LoadoutEntry,
+  MatchResult,
+  LoadMode,
+  ScoreComponents,
+} from "./types.js";
 
-const MIN_SCORE = 0.1;  // minimum score to include a domain entry
+// Minimum score to include a domain entry. Exported so callers can see
+// (and override via matchLoadout opts) the inclusion threshold.
+export const DEFAULT_MIN_SCORE = 0.1;
+
+/**
+ * Recall-aware absolute-hit denominator (FT-K1).
+ *
+ * The original matcher scored domain entries by pure coverage
+ * (matched / entry.keywords.length). On the live index, entries average
+ * 30.8 keywords each (median 21, p90 71), so a genuine 2-3 keyword
+ * topical match on a keyword-rich entry scored ~0.07-0.10 — below the
+ * 0.1 inclusion floor and effectively unreachable — while the noise we
+ * actually wanted to filter (single incidental hits) sat at 0.1-0.25.
+ *
+ * Pure coverage punishes keyword-rich entries for being thorough. The
+ * fix blends coverage with an *absolute* recall signal: matched / 5.
+ * ABSOLUTE_K = 5 means "5 matched keywords is a full-confidence match
+ * regardless of how many keywords the entry declares." The blended base
+ * is max(coverage, absolute), so:
+ *
+ *   - A keyword-rich entry can no longer be starved by its own breadth
+ *     (a real 2-3 kw match becomes reachable).
+ *   - A tiny entry still benefits from high coverage (max() keeps the
+ *     stronger of the two signals).
+ *
+ * Why 5 (not 3 or 10): with a 0.3 practical "genuine match" expectation
+ * and the existing 0.1 inclusion floor, ABSOLUTE_K = 5 places the
+ * thresholds where we want them:
+ *
+ *   - 1 incidental hit  → absolute 0.20 → stays below a 0.3 floor (noise filtered)
+ *   - 2 genuine hits     → absolute 0.40 → comfortably reachable
+ *   - 3 genuine hits     → absolute 0.60 → clearly a match
+ *   - 5 genuine hits     → absolute 1.00 → full confidence
+ *
+ * A larger K (e.g. 10) would push 2-3 kw matches back below the floor;
+ * a smaller K (e.g. 3) would let single incidental hits (0.33) cross a
+ * 0.3 noise threshold. 5 is the value that keeps genuine multi-keyword
+ * matches reachable while single incidental hits stay quiet.
+ */
+const ABSOLUTE_K = 5;
+
+// ── Options for matchLoadout (FT-K3, additive) ─────────────────
+export interface MatchOptions {
+  minScore?: number;  // inclusion threshold for domain entries (default DEFAULT_MIN_SCORE)
+}
 
 // ── Tokenize a task description into matchable words ───────────
 function tokenize(text: string): Set<string> {
@@ -31,7 +81,12 @@ function tokenize(text: string): Set<string> {
 function scoreEntry(
   entry: LoadoutEntry,
   taskTokens: Set<string>,
-): { score: number; matchedKeywords: string[]; matchedPatterns: string[] } {
+): {
+  score: number;
+  matchedKeywords: string[];
+  matchedPatterns: string[];
+  scoreComponents?: ScoreComponents;
+} {
   // Core entries always match
   if (entry.priority === "core") {
     return { score: 1.0, matchedKeywords: [], matchedPatterns: [] };
@@ -63,38 +118,54 @@ function scoreEntry(
     }
   }
 
-  // Score: proportion of keywords matched + pattern bonus
-  const keywordScore =
-    entry.keywords.length > 0
-      ? matchedKeywords.length / entry.keywords.length
-      : 0;
+  // FT-K1: recall-aware blend.
+  // coverage = the old pure metric (matched / declared keyword count).
+  // absolute = recall signal (matched / ABSOLUTE_K) — independent of how
+  //            many keywords the entry declares.
+  // base     = max(coverage, absolute) — keeps the stronger of the two,
+  //            so keyword-rich entries are not starved by their breadth
+  //            and tiny entries still benefit from high coverage.
+  const matched = matchedKeywords.length;
+  const coverage =
+    entry.keywords.length > 0 ? matched / entry.keywords.length : 0;
+  const absolute = matched / ABSOLUTE_K;
+  const base = Math.max(coverage, absolute);
 
   const patternBonus =
-    entry.patterns.length > 0 && matchedPatterns.length > 0
-      ? 0.2
-      : 0;
+    entry.patterns.length > 0 && matchedPatterns.length > 0 ? 0.2 : 0;
+
+  const score = Math.min(1.0, base + patternBonus);
 
-  const score = Math.min(1.0, keywordScore + patternBonus);
+  const scoreComponents: ScoreComponents = {
+    matched,
+    coverage,
+    absolute,
+    base,
+    patternBonus,
+  };
 
-  return { score, matchedKeywords, matchedPatterns };
+  return { score, matchedKeywords, matchedPatterns, scoreComponents };
 }
 
 // ── Match a task against a loadout index ────────────────────────
 // Returns entries that should be loaded, sorted by score (highest first).
+//
+// FT-K3: `opts.minScore` overrides the inclusion threshold; the 2-arg
+// call form is unchanged and defaults to DEFAULT_MIN_SCORE.
 export function matchLoadout(
   task: string,
   index: LoadoutIndex,
+  opts?: MatchOptions,
 ): MatchResult[] {
+  const minScore = opts?.minScore ?? DEFAULT_MIN_SCORE;
   const taskTokens = tokenize(task);
   const results: MatchResult[] = [];
 
   for (const entry of index.entries) {
-    const { score, matchedKeywords, matchedPatterns } = scoreEntry(
-      entry,
-      taskTokens,
-    );
+    const { score, matchedKeywords, matchedPatterns, scoreComponents } =
+      scoreEntry(entry, taskTokens);
 
-    if (score >= MIN_SCORE) {
+    if (score >= minScore) {
       const mode: LoadMode = entry.priority === "manual"
         ? "manual"
         : entry.priority === "core"
@@ -109,7 +180,18 @@ export function matchLoadout(
             ? `keywords [${matchedKeywords.join(", ")}]`
             : `patterns [${matchedPatterns.join(", ")}]`;
 
-      results.push({ entry, score, matchedKeywords, matchedPatterns, reason, mode });
+      const result: MatchResult = {
+        entry,
+        score,
+        matchedKeywords,
+        matchedPatterns,
+        reason,
+        mode,
+      };
+      // Additive: only domain entries produce a component breakdown.
+      if (scoreComponents) result.scoreComponents = scoreComponents;
+
+      results.push(result);
     }
   }
 

packages/kernel/src/tests/match.test.ts

@@ -1,6 +1,6 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
-import { matchLoadout, lookupEntry } from "../match.js";
+import { matchLoadout, lookupEntry, DEFAULT_MIN_SCORE } from "../match.js";
 import type { LoadoutIndex } from "../types.js";
 import { DEFAULT_TRIGGERS } from "../types.js";
 
@@ -69,15 +69,20 @@ describe("matchLoadout", () => {
     assert.ok(results[0].reason.includes("keywords"));
   });
 
-  it("scores by keyword overlap proportion", () => {
+  it("scores by recall-aware blend (coverage vs absolute)", () => {
     const index = makeIndex(
       { id: "narrow", keywords: ["ci", "workflow", "runner", "matrix", "dependabot"] },
       { id: "broad", keywords: ["ci", "workflow"] },
     );
     const results = matchLoadout("fix the ci workflow", index);
-    // "broad" has 2/2 match (1.0), "narrow" has 2/5 match (0.4)
+    // FT-K1: both match 2 keywords.
+    //   broad:  coverage 2/2 = 1.0, absolute 2/5 = 0.4 → max = 1.0
+    //   narrow: coverage 2/5 = 0.4, absolute 2/5 = 0.4 → max = 0.4
     assert.equal(results[0].entry.id, "broad");
-    assert.ok(results[0].score > results[1].score);
+    assert.equal(results[0].score, 1.0);
+    const narrow = results.find((r) => r.entry.id === "narrow")!;
+    assert.equal(narrow.score, 0.4);
+    assert.ok(results[0].score > narrow.score);
   });
 
   it("gives pattern bonus", () => {
@@ -86,10 +91,13 @@ describe("matchLoadout", () => {
       { id: "with-pattern", keywords: ["ci", "workflow", "runner"], patterns: ["ci_pipeline"] },
       { id: "without-pattern", keywords: ["ci", "workflow", "runner"] },
     );
-    // Task matches 1/3 keywords (0.33) + pattern bonus (0.2) = 0.53 vs 0.33
+    // 1 matched keyword: coverage 1/3 = 0.333, absolute 1/5 = 0.2 → base 0.333.
+    // with-pattern: 0.333 + 0.2 bonus = 0.533; without-pattern: 0.333.
     const results = matchLoadout("fix the ci pipeline", index);
     const withPattern = results.find((r) => r.entry.id === "with-pattern")!;
     const without = results.find((r) => r.entry.id === "without-pattern")!;
+    assert.ok(Math.abs(withPattern.score - 0.5333333333333333) < 1e-9);
+    assert.ok(Math.abs(without.score - 0.3333333333333333) < 1e-9);
     assert.ok(withPattern.score > without.score);
     assert.ok(withPattern.reason.includes("keywords") && withPattern.reason.includes("patterns"));
   });
@@ -112,6 +120,123 @@ describe("matchLoadout", () => {
   });
 });
 
+// Build N filler keywords that will NOT match the test tasks below.
+function filler(n: number): string[] {
+  return Array.from({ length: n }, (_, i) => `filler${i}`);
+}
+
+describe("matchLoadout — recall-aware scoring (FT-K1)", () => {
+  it("(a) a 2-keyword match on a 20+ keyword entry now scores >= 0.4", () => {
+    // 24-keyword entry; task hits exactly 2 of them.
+    const index = makeIndex(
+      { id: "rich", keywords: ["deploy", "release", ...filler(22)] },
+    );
+    const results = matchLoadout("deploy a release", index);
+    assert.equal(results.length, 1);
+    assert.equal(results[0].matchedKeywords.length, 2);
+    // coverage 2/24 = 0.083, absolute 2/5 = 0.4 -> max = 0.4
+    assert.ok(results[0].score >= 0.4, `score was ${results[0].score}`);
+    assert.equal(results[0].score, 0.4);
+    // Transparency: component breakdown is present and explains the score.
+    assert.ok(results[0].scoreComponents);
+    assert.equal(results[0].scoreComponents!.matched, 2);
+    assert.equal(results[0].scoreComponents!.absolute, 0.4);
+    assert.ok(results[0].scoreComponents!.coverage < 0.1);
+    assert.equal(results[0].scoreComponents!.base, 0.4);
+  });
+
+  it("(a') a 3-keyword match on a keyword-rich entry scores 0.6", () => {
+    const index = makeIndex(
+      { id: "rich", keywords: ["deploy", "release", "rollback", ...filler(27)] },
+    );
+    const results = matchLoadout("deploy release rollback now", index);
+    assert.equal(results[0].matchedKeywords.length, 3);
+    // absolute 3/5 = 0.6 dominates coverage 3/30 = 0.1
+    assert.ok(Math.abs(results[0].score - 0.6) < 1e-9, `score was ${results[0].score}`);
+  });
+
+  it("(b) a single incidental hit on a large entry stays <= 0.2", () => {
+    // 30-keyword entry; task hits exactly 1 keyword incidentally.
+    const index = makeIndex(
+      { id: "big", keywords: ["deploy", ...filler(29)] },
+    );
+    const results = matchLoadout("deploy something", index);
+    assert.equal(results.length, 1);
+    assert.equal(results[0].matchedKeywords.length, 1);
+    // coverage 1/30 = 0.033, absolute 1/5 = 0.2 -> max = 0.2
+    assert.ok(results[0].score <= 0.2, `score was ${results[0].score}`);
+    assert.equal(results[0].score, 0.2);
+  });
+
+  it("single hit on a tiny entry keeps high coverage (max picks coverage)", () => {
+    const index = makeIndex(
+      { id: "tiny", keywords: ["deploy", "release", "rollback"] },
+    );
+    const results = matchLoadout("deploy something", index);
+    // coverage 1/3 = 0.333 beats absolute 1/5 = 0.2 -> max = 0.333
+    assert.ok(Math.abs(results[0].score - 0.3333333333333333) < 1e-9, `score was ${results[0].score}`);
+  });
+});
+
+describe("matchLoadout — minScore option (FT-K3)", () => {
+  it("(c) minScore option filters out low scores", () => {
+    // Single incidental hit on a large entry -> score 0.2.
+    const index = makeIndex(
+      { id: "big", keywords: ["deploy", ...filler(29)] },
+    );
+    // Default threshold (0.1): the 0.2 entry is included.
+    const included = matchLoadout("deploy something", index);
+    assert.equal(included.length, 1);
+    // Raised threshold (0.3): the 0.2 entry is filtered out as noise.
+    const filtered = matchLoadout("deploy something", index, { minScore: 0.3 });
+    assert.equal(filtered.length, 0);
+  });
+
+  it("minScore option does not affect a genuine multi-keyword match", () => {
+    const index = makeIndex(
+      { id: "rich", keywords: ["deploy", "release", ...filler(22)] },
+    );
+    // score 0.4 survives a 0.3 threshold.
+    const results = matchLoadout("deploy a release", index, { minScore: 0.3 });
+    assert.equal(results.length, 1);
+    assert.equal(results[0].entry.id, "rich");
+  });
+
+  it("DEFAULT_MIN_SCORE matches the implicit 2-arg behavior", () => {
+    const index = makeIndex(
+      { id: "big", keywords: ["deploy", ...filler(29)] },
+    );
+    const implicit = matchLoadout("deploy something", index);
+    const explicit = matchLoadout("deploy something", index, { minScore: DEFAULT_MIN_SCORE });
+    assert.equal(implicit.length, explicit.length);
+    assert.equal(implicit.length, 1);
+  });
+});
+
+describe("matchLoadout — core/manual unchanged (FT-K1 regression)", () => {
+  it("(d) core entries still score 1.0 with no component breakdown", () => {
+    const index = makeIndex(
+      { id: "core-rule", keywords: ["whatever"], priority: "core" },
+    );
+    const results = matchLoadout("anything at all", index);
+    assert.equal(results.length, 1);
+    assert.equal(results[0].score, 1.0);
+    assert.equal(results[0].mode, "eager");
+    assert.equal(results[0].reason, "core: always loaded");
+    // Core entries do not get a recall-aware component breakdown.
+    assert.equal(results[0].scoreComponents, undefined);
+  });
+
+  it("(d) manual entries are still never auto-included", () => {
+    const index = makeIndex(
+      { id: "manual-rule", keywords: ["deploy", "release"], priority: "manual" },
+    );
+    // Even with a strong keyword match, manual entries do not auto-load.
+    const results = matchLoadout("deploy a release now", index);
+    assert.equal(results.length, 0);
+  });
+});
+
 describe("lookupEntry", () => {
   it("finds entry by id", () => {
     const index = makeIndex(

packages/kernel/src/types.ts

@@ -55,6 +55,17 @@ export interface Frontmatter {
   triggers: Triggers;
 }
 
+// ── Score components (matcher transparency) ────────────────────
+// Additive breakdown of how a domain entry's score was computed,
+// so callers can see WHY an entry scored the way it did.
+export interface ScoreComponents {
+  matched: number;       // count of matched keywords
+  coverage: number;      // matched / entry.keywords.length (the old pure metric)
+  absolute: number;      // matched / ABSOLUTE_K (recall-aware metric)
+  base: number;          // max(coverage, absolute) — the blended floor
+  patternBonus: number;  // 0 or 0.2 if any pattern matched
+}
+
 // ── Match result from the matcher ──────────────────────────────
 export interface MatchResult {
   entry: LoadoutEntry;
@@ -63,6 +74,7 @@ export interface MatchResult {
   matchedPatterns: string[];
   reason: string;        // human-readable explanation of why this matched
   mode: LoadMode;        // how this entry should be loaded
+  scoreComponents?: ScoreComponents;  // additive: breakdown of the score (domain entries only)
 }
 
 // ── Usage event (append-only log) ─────────────────────────────