feat(memories): diagnostics channel + index health summary + dedupe (FT-MR3/4/10)

Structured diagnostics[] ({severity,code,message,refPath?,line?,hint?}) on the analysis result, populated where missing/orphan/unresolved are recorded; flat missingFiles/orphanFiles kept as derived views. cmdIndex now prints a 'N missing / orphan / dup / id-too-long' health summary from validateMemory. Hoisted the 3 copy-pasted resolveRefPath into src/paths.ts (behavior-preserving). All exported from the barrel for uniform rendering under the unified CLI.

Tests: memories 54->65.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mcp-tool-shop

packages/memories/src/analyze.ts

@@ -13,19 +13,8 @@ import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
 import { join, dirname, relative, extname, basename, resolve } from "node:path";
 import { estimateTokens } from "@mcptoolshop/ai-loadout";
 import { parseMemoryMd } from "./parser.js";
-import type { MemoryAnalysis, MemoryRef } from "./types.js";
-
-/**
- * Try to resolve a reference path against multiple base directories.
- * Returns the first path that exists, or null.
- */
-function resolveRefPath(refPath: string, ...baseDirs: string[]): string | null {
-  for (const base of baseDirs) {
-    const full = join(base, refPath);
-    if (existsSync(full)) return full;
-  }
-  return null;
-}
+import { resolveRefPath } from "./paths.js";
+import type { MemoryAnalysis, MemoryRef, Diagnostic } from "./types.js";
 
 /**
  * Analyze a MEMORY.md file and its referenced topic files.
@@ -47,6 +36,12 @@ export function analyzeMemoryMd(filePath: string): MemoryAnalysis {
 
   // Check which referenced files exist
   // Try: relative to MEMORY.md, then relative to parent dir
+  //
+  // FT-MR3: every missing/orphan signal is recorded as a structured Diagnostic
+  // here, at the point of detection — the flat string[] arrays below are derived
+  // views kept for back-compat. Library consumers read `diagnostics`; the CLI
+  // renders it uniformly. Nothing goes to stderr.
+  const diagnostics: Diagnostic[] = [];
   const missingFiles: string[] = [];
   let topicTokens = 0;
 
@@ -58,9 +53,25 @@ export function analyzeMemoryMd(filePath: string): MemoryAnalysis {
         topicTokens += estimateTokens(topicContent);
       } catch {
         missingFiles.push(ref.path);
+        diagnostics.push({
+          severity: "error",
+          code: "MISSING_TOPIC_FILE",
+          message: `Referenced topic file not found: ${ref.path}`,
+          refPath: ref.path,
+          line: ref.line,
+          hint: "Create the file or remove the reference from MEMORY.md",
+        });
       }
     } else {
       missingFiles.push(ref.path);
+      diagnostics.push({
+        severity: "error",
+        code: "MISSING_TOPIC_FILE",
+        message: `Referenced topic file not found: ${ref.path}`,
+        refPath: ref.path,
+        line: ref.line,
+        hint: "Create the file or remove the reference from MEMORY.md",
+      });
     }
   }
 
@@ -85,11 +96,12 @@ export function analyzeMemoryMd(filePath: string): MemoryAnalysis {
           if (stat.isFile() && extname(entry) === ".md" && entry !== "MEMORY.md") {
             if (!referencedBasenames.has(entry)) {
               orphanFiles.push(entry);
+              diagnostics.push(orphanDiagnostic(entry));
             }
           }
           // Also scan subdirectories
           if (stat.isDirectory()) {
-            scanDir(fullPath, fileDir, referencedBasenames, orphanFiles);
+            scanDir(fullPath, fileDir, referencedBasenames, orphanFiles, diagnostics);
           }
         } catch {
           // Skip entries we can't stat (permission denied, broken symlinks)
@@ -106,12 +118,24 @@ export function analyzeMemoryMd(filePath: string): MemoryAnalysis {
     refs,
     orphanFiles,
     missingFiles,
+    diagnostics,
     totalTokens: inlineTokens + topicTokens,
     inlineTokens,
     topicTokens,
   };
 }
 
+/** FT-MR3: build the structured Diagnostic for an orphan topic file. */
+function orphanDiagnostic(path: string): Diagnostic {
+  return {
+    severity: "warning",
+    code: "ORPHAN_TOPIC_FILE",
+    message: `Topic file not referenced in MEMORY.md: ${path}`,
+    refPath: path,
+    hint: "Add a reference in MEMORY.md or delete the file",
+  };
+}
+
 /**
  * Recursively scan a directory for .md files not in the referenced set.
  */
@@ -120,6 +144,7 @@ function scanDir(
   baseDir: string,
   referenced: Set<string>,
   orphans: string[],
+  diagnostics: Diagnostic[],
 ): void {
   let entries: string[];
   try {
@@ -134,11 +159,12 @@ function scanDir(
       const stat = statSync(fullPath);
 
       if (stat.isDirectory()) {
-        scanDir(fullPath, baseDir, referenced, orphans);
+        scanDir(fullPath, baseDir, referenced, orphans, diagnostics);
       } else if (extname(entry) === ".md") {
         const relPath = relative(baseDir, fullPath).replace(/\\/g, "/");
         if (!referenced.has(relPath) && !referenced.has(basename(relPath))) {
           orphans.push(relPath);
+          diagnostics.push(orphanDiagnostic(relPath));
         }
       }
     } catch {

packages/memories/src/cli.ts

@@ -17,6 +17,27 @@ import { analyzeMemoryMd } from "./analyze.js";
 import { generateIndex } from "./index-gen.js";
 import { validateMemory, validateMemoryIndex } from "./validate.js";
 import { generateStats, formatStats } from "./stats.js";
+import type { ValidationIssue } from "./types.js";
+
+/**
+ * FT-MR4: roll up the validateMemory issue list into the four headline counts
+ * the `index` health summary and `validate` both surface. Centralizing this
+ * keeps the `index` summary byte-identical to what `validate` reports in
+ * detail, so the numbers can never drift between the two commands.
+ */
+function healthCounts(issues: ValidationIssue[]): {
+  missing: number;
+  orphan: number;
+  duplicate: number;
+  idTooLong: number;
+} {
+  return {
+    missing: issues.filter((i) => i.code === "MISSING_TOPIC_FILE").length,
+    orphan: issues.filter((i) => i.code === "ORPHAN_TOPIC_FILE").length,
+    duplicate: issues.filter((i) => i.code === "DUPLICATE_REF").length,
+    idTooLong: issues.filter((i) => i.code === "ID_TOO_LONG").length,
+  };
+}
 
 // ── Colors ────────────────────────────────────────────────────
 const BOLD = "\x1b[1m";
@@ -223,6 +244,18 @@ async function cmdIndex(args: string[]) {
     warn(`${n} unresolved ref${n === 1 ? "" : "s"} skipped — run \`validate\` for detail`);
   }
 
+  // FT-MR4: run the richer validateMemory(analysis) rollup and print a one-line
+  // health summary so `index` surfaces the same counts `validate` computes.
+  // Previously `index` only printed validateMemoryIndex issues if non-empty and
+  // never showed the missing/orphan/duplicate/id-too-long picture — pairing it
+  // with the diagnostics channel here means a single `index` run tells the user
+  // whether the memory store is healthy, pointing them at `validate` for detail.
+  const memoryIssues = validateMemory(analysis);
+  const c = healthCounts(memoryIssues);
+  info(
+    `${c.missing} missing · ${c.orphan} orphan · ${c.duplicate} duplicate · ${c.idTooLong} id-too-long — run \`validate\` for detail`,
+  );
+
   // Validate
   const issues = validateMemoryIndex(index);
   if (issues.length > 0) {

packages/memories/src/index-gen.ts

@@ -5,21 +5,14 @@
  * from the parsed memory references and their topic files.
  */
 
-import { readFileSync, existsSync } from "node:fs";
-import { join, dirname, resolve } from "node:path";
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
 import { estimateTokens, parseFrontmatter } from "@mcptoolshop/ai-loadout";
 import type { LoadoutEntry, Budget, Frontmatter } from "@mcptoolshop/ai-loadout";
 import { DEFAULT_TRIGGERS } from "@mcptoolshop/ai-loadout";
 import type { MemoryAnalysis, MemoryIndex, MemoryRef } from "./types.js";
 import { extractKeywords } from "./analyze.js";
-
-function resolveRefPath(refPath: string, ...baseDirs: string[]): string | null {
-  for (const base of baseDirs) {
-    const full = join(base, refPath);
-    if (existsSync(full)) return full;
-  }
-  return null;
-}
+import { resolveRefPath } from "./paths.js";
 
 /**
  * Generate a memory dispatch index from analysis results.
@@ -51,9 +44,25 @@ export function generateIndex(
       // unresolved path already lands in analysis.missingFiles; that is the
       // observability channel. The CLI (cmdIndex) reads missingFiles and prints
       // a one-line summary; the library stays quiet.
+      //
+      // FT-MR3: also record the structured diagnostic so SDK callers get
+      // severity + code + line + hint, not just a bare string. missingFiles
+      // stays a derived view kept in lockstep for back-compat.
       if (!analysis.missingFiles.includes(ref.path)) {
         analysis.missingFiles.push(ref.path);
       }
+      if (!analysis.diagnostics.some(
+        (d) => d.code === "UNRESOLVED_REF" && d.refPath === ref.path,
+      )) {
+        analysis.diagnostics.push({
+          severity: "error",
+          code: "UNRESOLVED_REF",
+          message: `Reference could not be resolved to a file on disk: ${ref.path}`,
+          refPath: ref.path,
+          line: ref.line,
+          hint: "Create the file, fix the path, or remove the reference from MEMORY.md",
+        });
+      }
       continue;
     }
 

packages/memories/src/index.ts

@@ -4,6 +4,7 @@ export type {
   MemoryRef,
   MemoryAnalysis,
   MemoryIndex,
+  Diagnostic,
 } from "./types.js";
 
 // Re-export kernel types
@@ -34,6 +35,12 @@ export { generateIndex } from "./index-gen.js";
 // ── Validator ─────────────────────────────────────────────────
 export { validateMemory, validateMemoryIndex } from "./validate.js";
 
+// ── Path resolution ───────────────────────────────────────────
+// FT-MR10: the single shared ref-path resolver, so the unified loadout-os CLI
+// (and any SDK consumer) resolves topic refs identically to analyze/index-gen/
+// validate.
+export { resolveRefPath } from "./paths.js";
+
 // ── Stats ─────────────────────────────────────────────────────
 export { generateStats, formatStats } from "./stats.js";
 export type { StatsReport } from "./stats.js";

packages/memories/src/paths.ts

@@ -0,0 +1,29 @@
+/**
+ * Shared path-resolution helper for claude-memories.
+ *
+ * FT-MR10: the same `resolveRefPath` logic was copy-pasted in analyze.ts,
+ * index-gen.ts, and (a 2-arg variant) validate.ts. A single implementation
+ * here guarantees analyze / index-gen / validate — and the unified loadout-os
+ * CLI — all resolve a referenced topic path identically.
+ *
+ * Strategy (unchanged, behavior-preserving): try each base directory in order,
+ * joining the ref path onto it, and return the first that exists on disk;
+ * otherwise null. Callers pass MEMORY.md's own directory first, then its
+ * parent (for refs written project-root-relative like "memory/foo.md" when
+ * MEMORY.md itself lives inside memory/).
+ */
+
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * Resolve a reference path against one or more base directories.
+ * Returns the first `join(base, refPath)` that exists, or null.
+ */
+export function resolveRefPath(refPath: string, ...baseDirs: string[]): string | null {
+  for (const base of baseDirs) {
+    const full = join(base, refPath);
+    if (existsSync(full)) return full;
+  }
+  return null;
+}

packages/memories/src/tests/cli.test.ts

@@ -145,3 +145,62 @@ describe("unresolved-ref summary (MEM-B03)", () => {
     }
   });
 });
+
+// ── FT-MR4: `index` surfaces the health-summary counts ────────────────
+describe("index health summary (FT-MR4)", () => {
+  it("prints the missing · orphan · duplicate · id-too-long one-liner", () => {
+    const dir = mkdtempSync(join(tmpdir(), "mem-health-"));
+    try {
+      const out = join(dir, "index.json");
+      // The fixture has missing refs (nullout, xrpl-lab) and an over-long id.
+      const res = runCli(["index", join(FIXTURES, "MEMORY.md"), "--out", out]);
+      assert.equal(res.status, 0);
+      assert.match(res.stdout, /\d+ missing · \d+ orphan · \d+ duplicate · \d+ id-too-long/);
+      assert.match(res.stdout, /run `validate` for detail/);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("the index summary counts match what validate reports", () => {
+    const dir = mkdtempSync(join(tmpdir(), "mem-health-match-"));
+    try {
+      const out = join(dir, "index.json");
+      const idx = runCli(["index", join(FIXTURES, "MEMORY.md"), "--out", out]);
+      const val = runCli(["validate", join(FIXTURES, "MEMORY.md")]);
+
+      // Pull the counts the index summary printed.
+      const m = idx.stdout.match(
+        /(\d+) missing · (\d+) orphan · (\d+) duplicate · (\d+) id-too-long/,
+      );
+      assert.ok(m, "index must print the four headline counts");
+      const [, missing, orphan, duplicate, idTooLong] = m.map(Number);
+
+      // validate prints one line per issue; count occurrences of each code.
+      const count = (code: string) =>
+        (val.stdout.match(new RegExp(`\\[${code}\\]`, "g")) ?? []).length;
+      assert.equal(missing, count("MISSING_TOPIC_FILE"), "missing count agrees with validate");
+      assert.equal(orphan, count("ORPHAN_TOPIC_FILE"), "orphan count agrees with validate");
+      assert.equal(duplicate, count("DUPLICATE_REF"), "duplicate count agrees with validate");
+      assert.equal(idTooLong, count("ID_TOO_LONG"), "id-too-long count agrees with validate");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("a clean MEMORY.md reports all-zero counts", () => {
+    const dir = mkdtempSync(join(tmpdir(), "mem-health-clean-"));
+    try {
+      writeFileSync(join(dir, "topic.md"), "# Topic\nbody\n");
+      writeFileSync(
+        join(dir, "MEMORY.md"),
+        "# Mem\n\n## Active\n\nTopic — a real topic → `topic.md`\n",
+      );
+      const res = runCli(["index", join(dir, "MEMORY.md"), "--out", join(dir, "out.json")]);
+      assert.equal(res.status, 0);
+      assert.match(res.stdout, /0 missing · 0 orphan · 0 duplicate · 0 id-too-long/);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});