diff --git a/.gitignore b/.gitignore index 598a422e..510fbe0e 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,8 @@ static/browser_history/*.gif # user conf conf.yaml + +# Agents +.cursor +.cursorrules +.claude \ No newline at end of file diff --git a/README.md b/README.md index e5503304..6b1c678c 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,9 @@ npm install mcp-use # LangChain.js and your LLM provider (e.g., OpenAI) npm install langchain @langchain/openai dotenv + +# Optional: Install observability packages for monitoring +npm install langfuse langfuse-langchain # For Langfuse observability ``` Create a `.env`: diff --git a/package-lock.json b/package-lock.json index 462de901..e7451db2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,18 @@ }, "engines": { "node": ">=22.0.0" + }, + "peerDependencies": { + "langfuse": "^3.32.0", + "langfuse-langchain": "^3.38.4" + }, + "peerDependenciesMeta": { + "langfuse": { + "optional": true + }, + "langfuse-langchain": { + "optional": true + } } }, "node_modules/@ai-sdk/provider": { @@ -1362,6 +1374,206 @@ "node": ">=8.0.0" } }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.57.2.tgz", + "integrity": "sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.57.2.tgz", + "integrity": "sha512-awDdNRMIwDvUtoRYxRhja5QYH6+McBLtoz1q9BeEsskhZcrGmH/V1fWpGx8n+Rc+542e8pJA6y+aullbIzQmlw==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.57.2.tgz", + "integrity": "sha512-XdxEzL23Urhidyebg5E6jZoaiW5ygP/mRjxLHixogbqwDy2Faduzb5N0o/Oi+XTIJu+iyxXdVORjXax+Qgfxag==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-transformer": "0.57.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.57.2.tgz", + "integrity": "sha512-48IIRj49gbQVK52jYsw70+Jv+JbahT8BqT2Th7C4H7RCM9d0gZ5sgNPoMpWldmfjvIsSgiGJtjfk9MeZvjhoig==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-logs": "0.57.2", + "@opentelemetry/sdk-metrics": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "protobufjs": "^7.3.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz", + "integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.57.2.tgz", + "integrity": "sha512-TXFHJ5c+BKggWbdEQ/inpgIzEmS2BGQowLE9UhsMd7YYlUfBQJ4uax0VF/B5NYigdM/75OoJGhAV3upEhK+3gg==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.30.1.tgz", + "integrity": "sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.30.1.tgz", + "integrity": "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@pkgr/core": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.2.tgz", @@ -1375,6 +1587,90 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause", + "optional": true, + "peer": true + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.46.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.46.2.tgz", @@ -1751,7 +2047,7 @@ "version": "20.19.9", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.9.tgz", "integrity": "sha512-cuVNgarYWZqxRJDQHEB58GEONhOK79QVR/qYx4S7kcUObQvUwvFnYxJuuHUKm2aieN9X3yZB4LZsuYNU1Qphsw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -5315,6 +5611,52 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/langfuse": { + "version": "3.38.4", + "resolved": "https://registry.npmjs.org/langfuse/-/langfuse-3.38.4.tgz", + "integrity": "sha512-2UqMeHLl3DGNX1Nh/cO4jGhk7TzDJ6gjQLlyS9rwFCKVO81xot6b58yeTsTB5YrWupWsOxQtMNoQYIQGOUlH9Q==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "langfuse-core": "^3.38.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/langfuse-core": { + "version": "3.38.4", + "resolved": "https://registry.npmjs.org/langfuse-core/-/langfuse-core-3.38.4.tgz", + "integrity": "sha512-onTAqcEGhoXuBgqDFXe2t+bt9Vi+5YChRgdz3voM49JKoHwtVZQiUdqTfjSivGR75eSbYoiaIL8IRoio+jaqwg==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "mustache": "^4.2.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/langfuse-langchain": { + "version": "3.38.4", + "resolved": "https://registry.npmjs.org/langfuse-langchain/-/langfuse-langchain-3.38.4.tgz", + "integrity": "sha512-7HJqouMrVOP9MFdu33M4G4uBFyQAIh/DqGYALfs41xqm7t99eZxKcTvt4rYZy67iQAhd58TG3q8+9haGzuLbOA==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "langfuse": "^3.38.4", + "langfuse-core": "^3.38.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "langchain": ">=0.0.157 <0.4.0" + } + }, "node_modules/langsmith": { "version": "0.3.50", "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.50.tgz", @@ -5586,6 +5928,14 @@ "node": ">= 12.0.0" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0", + "optional": true, + "peer": true + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -7435,6 +7785,32 @@ "node": ">=0.4.0" } }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "optional": true, + "peer": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -8828,7 +9204,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/unist-util-is": { diff --git a/package.json b/package.json index d265c6fb..9b6198cb 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,18 @@ "example:add_server": "npm run build && node dist/examples/add_server_tool.js", "example:structured": "npm run build && node dist/examples/structured_output.js" }, + "peerDependencies": { + "langfuse": "^3.32.0", + "langfuse-langchain": "^3.38.4" + }, + "peerDependenciesMeta": { + "langfuse": { + "optional": true + }, + "langfuse-langchain": { + "optional": true + } + }, "dependencies": { "@dmitryrechkin/json-schema-to-zod": "^1.0.1", "@langchain/anthropic": "^0.3.14", diff --git a/src/agents/mcp_agent.ts b/src/agents/mcp_agent.ts index 9fcfc52a..c227d99a 100644 --- a/src/agents/mcp_agent.ts +++ b/src/agents/mcp_agent.ts @@ -1,15 +1,18 @@ +import type { BaseCallbackHandler } from '@langchain/core/callbacks/base' +import type { CallbackManagerForChainRun } from '@langchain/core/callbacks/manager' import type { BaseLanguageModelInterface, LanguageModelLike } from '@langchain/core/language_models/base' +import type { Serialized } from '@langchain/core/load/serializable' import type { BaseMessage, } from '@langchain/core/messages' import type { StructuredToolInterface, ToolInterface } from '@langchain/core/tools' import type { StreamEvent } from '@langchain/core/tracers/log_stream' import type { AgentFinish, AgentStep } from 'langchain/agents' - import type { ZodSchema } from 'zod' import type { MCPClient } from '../client.js' import type { BaseConnector } from '../connectors/base.js' import type { MCPSession } from '../session.js' +import { CallbackManager } from '@langchain/core/callbacks/manager' import { AIMessage, HumanMessage, @@ -21,14 +24,12 @@ import { ChatPromptTemplate, MessagesPlaceholder, } from '@langchain/core/prompts' -import { - AgentExecutor, - createToolCallingAgent, -} from 'langchain/agents' +import { AgentExecutor, createToolCallingAgent } from 'langchain/agents' import { zodToJsonSchema } from 'zod-to-json-schema' import { LangChainAdapter } from '../adapters/langchain_adapter.js' import { logger } from '../logging.js' import { ServerManager } from '../managers/server_manager.js' +import { ObservabilityManager } from '../observability/index.js' import { extractModelInfo, Telemetry } from '../telemetry/index.js' import { createSystemMessage } from './prompts/system_prompt_builder.js' import { DEFAULT_SYSTEM_PROMPT_TEMPLATE, SERVER_MANAGER_SYSTEM_PROMPT_TEMPLATE } from './prompts/templates.js' @@ -61,6 +62,10 @@ export class MCPAgent { private modelProvider: string private modelName: string + // Observability support + private observabilityManager: ObservabilityManager + private callbacks: BaseCallbackHandler[] = [] + // Remote agent support private isRemote = false private remoteAgent: RemoteAgent | null = null @@ -81,6 +86,7 @@ export class MCPAgent { verbose?: boolean adapter?: LangChainAdapter serverManagerFactory?: (client: MCPClient) => ServerManager + callbacks?: BaseCallbackHandler[] // Remote agent parameters agentId?: string apiKey?: string @@ -107,6 +113,8 @@ export class MCPAgent { this.telemetry = Telemetry.getInstance() this.modelProvider = 'remote' this.modelName = 'remote-agent' + this.observabilityManager = new ObservabilityManager({ customCallbacks: options.callbacks }) + this.callbacks = [] return } @@ -159,6 +167,12 @@ export class MCPAgent { this.modelName = 'unknown' } + // Set up observability callbacks using the ObservabilityManager + this.observabilityManager = new ObservabilityManager({ + customCallbacks: options.callbacks, + verbose: this.verbose, + }) + // Make getters configurable for test mocking Object.defineProperty(this, 'agentExecutor', { get: () => this._agentExecutor, @@ -183,6 +197,13 @@ export class MCPAgent { logger.info('πŸš€ Initializing MCP agent and connecting to services...') + // Initialize observability callbacks + this.callbacks = await this.observabilityManager.getCallbacks() + const handlerNames = await this.observabilityManager.getHandlerNames() + if (handlerNames.length > 0) { + logger.info(`πŸ“Š Observability enabled with: ${handlerNames.join(', ')}`) + } + // If using server manager, initialize it if (this.useServerManager && this.serverManager) { await this.serverManager.initialize() @@ -202,7 +223,7 @@ export class MCPAgent { // Standard initialization - if using client, get or create sessions if (this.client) { // First try to get existing sessions - this.sessions = await this.client.getAllActiveSessions() + this.sessions = this.client.getAllActiveSessions() logger.info(`πŸ”Œ Found ${Object.keys(this.sessions).length} existing sessions`) // If no active sessions exist, create new ones @@ -294,6 +315,7 @@ export class MCPAgent { maxIterations: this.maxSteps, verbose: this.verbose, returnIntermediateSteps: true, + callbacks: this.callbacks, }) } @@ -480,6 +502,23 @@ export class MCPAgent { let nameToToolMap: Record = Object.fromEntries(this._tools.map(t => [t.name, t])) logger.info(`🏁 Starting agent execution with max_steps=${steps}`) + // Create a run manager with our callbacks if we have any - ONCE for the entire execution + let runManager: CallbackManagerForChainRun | undefined + if (this.callbacks?.length > 0) { + // Create an async callback manager with our callbacks + const callbackManager = new CallbackManager(undefined, { + handlers: this.callbacks, + inheritableHandlers: this.callbacks, + }) + // Create a run manager for this chain execution + runManager = await callbackManager.handleChainStart({ + name: 'MCPAgent (mcp-use)', + id: ['MCPAgent (mcp-use)'], + lc: 1, + type: 'not_implemented', + } as Serialized, inputs) + } + for (let stepNum = 0; stepNum < steps; stepNum++) { stepsTaken = stepNum + 1 if (this.useServerManager && this.serverManager) { @@ -508,15 +547,17 @@ export class MCPAgent { try { logger.debug('Starting agent step execution') - const nextStepOutput = await this._agentExecutor._takeNextStep( + const nextStepOutput: AgentStep[] | AgentFinish = await this._agentExecutor._takeNextStep( nameToToolMap as Record, inputs, intermediateSteps, + runManager, ) - // Agent finish handling - if ((nextStepOutput as AgentFinish).returnValues) { + // Agent finish handling (AgentFinish contains returnValues property) + if ('returnValues' in nextStepOutput) { logger.info(`βœ… Agent finished at step ${stepNum + 1}`) - result = (nextStepOutput as AgentFinish).returnValues?.output ?? 'No output generated' + result = nextStepOutput.returnValues?.output ?? 'No output generated' + runManager?.handleChainEnd({ output: result }) // If structured output is requested, attempt to create it if (outputSchema && structuredLlm) { @@ -595,10 +636,10 @@ export class MCPAgent { // Detect direct return if (stepArray.length) { const lastStep = stepArray[stepArray.length - 1] - const toolReturn = await this._agentExecutor._getToolReturn(lastStep) + const toolReturn: AgentFinish | null = await this._agentExecutor._getToolReturn(lastStep) if (toolReturn) { logger.info(`πŸ† Tool returned directly at step ${stepNum + 1}`) - result = (toolReturn as unknown as AgentFinish).returnValues?.output ?? 'No output generated' + result = toolReturn.returnValues?.output ?? 'No output generated' break } } @@ -607,11 +648,13 @@ export class MCPAgent { if (e instanceof OutputParserException) { logger.error(`❌ Output parsing error during step ${stepNum + 1}: ${e}`) result = `Agent stopped due to a parsing error: ${e}` + runManager?.handleChainError(result) break } logger.error(`❌ Error during agent execution step ${stepNum + 1}: ${e}`) console.error(e) result = `Agent stopped due to an error: ${e}` + runManager?.handleChainError(result) break } } @@ -620,6 +663,7 @@ export class MCPAgent { if (!result) { logger.warn(`⚠️ Agent stopped after reaching max iterations (${steps})`) result = `Agent stopped after reaching the maximum number of steps (${steps}).` + runManager?.handleChainEnd({ output: result }) } logger.info('πŸŽ‰ Agent execution complete') @@ -642,7 +686,7 @@ export class MCPAgent { let serverCount = 0 if (this.client) { - serverCount = Object.keys(await this.client.getAllActiveSessions()).length + serverCount = Object.keys(this.client.getAllActiveSessions()).length } else if (this.connectors) { serverCount = this.connectors.length @@ -690,6 +734,9 @@ export class MCPAgent { } logger.info('πŸ”Œ Closing MCPAgent resources…') + + // Shutdown observability handlers (important for serverless) + await this.observabilityManager.shutdown() try { this._agentExecutor = null this._tools = [] @@ -776,10 +823,15 @@ export class MCPAgent { // Prepare inputs const inputs = { input: query, chat_history: langchainHistory } + logger.info('callbacks', this.callbacks) + // Stream events from the agent executor const eventStream = agentExecutor.streamEvents( inputs, - { version: 'v2' }, + { + version: 'v2', + callbacks: this.callbacks.length > 0 ? this.callbacks : undefined, + }, ) // Yield each event @@ -829,7 +881,7 @@ export class MCPAgent { let serverCount = 0 if (this.client) { - serverCount = Object.keys(await this.client.getAllActiveSessions()).length + serverCount = Object.keys(this.client.getAllActiveSessions()).length } else if (this.connectors) { serverCount = this.connectors.length diff --git a/src/observability/README.md b/src/observability/README.md new file mode 100644 index 00000000..cebc20dc --- /dev/null +++ b/src/observability/README.md @@ -0,0 +1,167 @@ +# Observability Module + +This module provides comprehensive observability for MCP agents using LangChain, supporting Langfuse observability platform. + +## Features + +- **Langfuse integration**: Comprehensive observability with Langfuse +- **Automatic instrumentation**: Zero-code observability with environment variables +- **Custom callbacks**: Support for custom LangChain callback handlers +- **TypeScript-first**: Full TypeScript support with proper types +- **Serverless-ready**: Proper shutdown handling for serverless environments + +## Installation + +Install the required packages based on your observability platform: + +```bash +# For Langfuse +npm install langfuse langfuse-langchain +``` + +## Configuration + +### Langfuse + +Set the following environment variables: + +```env +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +LANGFUSE_HOST=https://cloud.langfuse.com # Optional +MCP_USE_LANGFUSE=true # Set to false to disable +``` + +## Usage + +### Basic Usage with MCPAgent + +The observability is automatically integrated into MCPAgent: + +```typescript +import { MCPAgent } from 'mcp-use' + +const agent = new MCPAgent({ + llm: myLLM, + client: myMCPClient, + // Observability is automatically enabled if environment variables are set +}) + +// Initialize the agent +await agent.initialize() + +// Run queries - they will be automatically traced +const result = await agent.run('What\'s the weather?') +``` + +### Custom Callbacks + +You can provide custom callbacks: + +```typescript +import { CallbackHandler } from 'langfuse-langchain' +import { MCPAgent } from 'mcp-use' + +const customHandler = new CallbackHandler({ + secretKey: 'custom-secret', + publicKey: 'custom-public', +}) + +const agent = new MCPAgent({ + llm: myLLM, + client: myMCPClient, + callbacks: [customHandler], // Use custom callbacks instead of auto-detected ones +}) +``` + +### Direct ObservabilityManager Usage + +For advanced use cases, you can use the ObservabilityManager directly: + +```typescript +import { ObservabilityManager } from 'mcp-use/observability' + +// Create a manager +const manager = new ObservabilityManager({ + verbose: true, // Enable verbose logging +}) + +// Get available callbacks +const callbacks = await manager.getCallbacks() + +// Check available handlers +const handlerNames = await manager.getHandlerNames() +console.log('Available handlers:', handlerNames) + +// Add custom callback +manager.addCallback(myCustomCallback) + +// Shutdown (important for serverless) +await manager.shutdown() +``` + +## Platform Features + +### Langfuse Features + +- Detailed LLM call tracing +- Chain execution tracking +- Tool usage monitoring +- Cost tracking +- Custom metadata and tags +- Session management +- User tracking + +## Serverless Considerations + +For serverless environments (AWS Lambda, Vercel, etc.), ensure proper shutdown: + +```typescript +const agent = new MCPAgent({ /* ... */ }) + +try { + await agent.initialize() + const result = await agent.run(query) + return result +} +finally { + // Important: Ensure traces are flushed + await agent.close() +} +``` + +## Debugging + +Enable debug logging to see observability events: + +```typescript +import { logger } from 'mcp-use/logging' + +// Set log level to debug +process.env.LOG_LEVEL = 'debug' + +// Now you'll see detailed observability logs +``` + +## Environment Variables Reference + +### Langfuse + +- `LANGFUSE_PUBLIC_KEY` - Required: Your Langfuse public key +- `LANGFUSE_SECRET_KEY` - Required: Your Langfuse secret key +- `LANGFUSE_HOST` / `LANGFUSE_BASEURL` - Optional: Langfuse API URL (default: https://cloud.langfuse.com) +- `LANGFUSE_RELEASE` - Optional: Release/version identifier +- `LANGFUSE_FLUSH_AT` - Optional: Batch size for flushing (default: 15) +- `LANGFUSE_FLUSH_INTERVAL` - Optional: Flush interval in ms (default: 10000) +- `LANGFUSE_REQUEST_TIMEOUT` - Optional: Request timeout in ms (default: 10000) +- `LANGFUSE_ENABLED` - Optional: Set to "false" to disable +- `MCP_USE_LANGFUSE` - Optional: Set to "false" to disable Langfuse integration + +## Examples + +See the [examples](../../examples/) directory for complete working examples: + +- Basic observability setup +- Multi-platform configuration +- Custom callback handlers +- Serverless deployments diff --git a/src/observability/index.ts b/src/observability/index.ts new file mode 100644 index 00000000..b25f7cc2 --- /dev/null +++ b/src/observability/index.ts @@ -0,0 +1,24 @@ +/** + * Observability module for MCP-use. + * + * This module provides centralized observability management for LangChain agents, + * supporting multiple platforms like Langfuse and Laminar. + */ + +// Import observability providers - order matters for initialization +import './langfuse.js' + +// Re-export individual handlers for direct usage if needed +export { + langfuseClient, + langfuseHandler, + langfuseInitPromise, +} from './langfuse.js' + +// Export the manager and its utilities +export { + createManager, + getDefaultManager, + type ObservabilityConfig, + ObservabilityManager, +} from './manager.js' diff --git a/src/observability/langfuse.ts b/src/observability/langfuse.ts new file mode 100644 index 00000000..ae74ee45 --- /dev/null +++ b/src/observability/langfuse.ts @@ -0,0 +1,145 @@ +/** + * Langfuse observability integration for MCP-use. + * + * This module provides automatic instrumentation and callback handler + * for Langfuse observability platform. + */ + +/// + +import type { BaseCallbackHandler } from '@langchain/core/callbacks/base' +import { config } from 'dotenv' +import { logger } from '../logging.js' + +config() + +// Check if Langfuse is disabled via environment variable +const langfuseDisabled = process.env.MCP_USE_LANGFUSE?.toLowerCase() === 'false' + +// Initialize variables - using const with object to avoid linter issues with mutable exports +const langfuseState = { + handler: null as BaseCallbackHandler | null, + client: null as any, + initPromise: null as Promise | null, +} + +async function initializeLangfuse(): Promise { + try { + // Dynamically import to avoid errors if package not installed + const langfuseModule = await import('langfuse-langchain').catch(() => null) + if (!langfuseModule) { + logger.debug('Langfuse package not installed - tracing disabled. Install with: npm install langfuse-langchain') + return + } + + const { CallbackHandler } = langfuseModule as any + // Create a custom CallbackHandler wrapper to add logging + class LoggingCallbackHandler extends CallbackHandler { + constructor(config?: any) { + super(config) + } + + async handleLLMStart(...args: any[]): Promise { + logger.debug('Langfuse: LLM start intercepted') + if (this.verbose) { + logger.debug(`Langfuse: LLM start args: ${JSON.stringify(args)}`) + } + return super.handleLLMStart(...args) + } + + async handleChainStart(...args: any[]): Promise { + logger.debug('Langfuse: Chain start intercepted') + if (this.verbose) { + logger.debug(`Langfuse: Chain start args: ${JSON.stringify(args)}`) + } + return super.handleChainStart(...args) + } + + async handleToolStart(...args: any[]): Promise { + logger.debug('Langfuse: Tool start intercepted') + if (this.verbose) { + logger.debug(`Langfuse: Tool start args: ${JSON.stringify(args)}`) + } + return super.handleToolStart(...args) + } + + async handleRetrieverStart(...args: any[]): Promise { + logger.debug('Langfuse: Retriever start intercepted') + if (this.verbose) { + logger.debug(`Langfuse: Retriever start args: ${JSON.stringify(args)}`) + } + return super.handleRetrieverStart(...args) + } + + async handleAgentAction(...args: any[]): Promise { + logger.debug('Langfuse: Agent action intercepted') + if (this.verbose) { + logger.debug(`Langfuse: Agent action args: ${JSON.stringify(args)}`) + } + return super.handleAgentAction(...args) + } + + async handleAgentEnd(...args: any[]): Promise { + logger.debug('Langfuse: Agent end intercepted') + if (this.verbose) { + logger.debug(`Langfuse: Agent end args: ${JSON.stringify(args)}`) + } + return super.handleAgentEnd(...args) + } + } + + // Create the handler with configuration + const config = { + publicKey: process.env.LANGFUSE_PUBLIC_KEY, + secretKey: process.env.LANGFUSE_SECRET_KEY, + baseUrl: process.env.LANGFUSE_HOST || process.env.LANGFUSE_BASEURL || 'https://cloud.langfuse.com', + flushAt: Number.parseInt(process.env.LANGFUSE_FLUSH_AT || '15'), + flushInterval: Number.parseInt(process.env.LANGFUSE_FLUSH_INTERVAL || '10000'), + release: process.env.LANGFUSE_RELEASE, + requestTimeout: Number.parseInt(process.env.LANGFUSE_REQUEST_TIMEOUT || '10000'), + enabled: process.env.LANGFUSE_ENABLED !== 'false', + } + + langfuseState.handler = new LoggingCallbackHandler(config) as BaseCallbackHandler + logger.debug('Langfuse observability initialized successfully with logging enabled') + + // Also initialize the client for direct usage if needed + try { + const langfuseCore = await import('langfuse').catch(() => null) + if (langfuseCore) { + const { Langfuse } = langfuseCore as any + langfuseState.client = new Langfuse({ + publicKey: process.env.LANGFUSE_PUBLIC_KEY, + secretKey: process.env.LANGFUSE_SECRET_KEY, + baseUrl: process.env.LANGFUSE_HOST || 'https://cloud.langfuse.com', + }) + logger.debug('Langfuse client initialized') + } + } + catch (error) { + logger.debug(`Langfuse client initialization failed: ${error}`) + } + } + catch (error) { + logger.debug(`Langfuse initialization error: ${error}`) + } +} + +// Only initialize if not disabled and required keys are present +if (langfuseDisabled) { + logger.debug('Langfuse tracing disabled via MCP_USE_LANGFUSE environment variable') +} +else if (!process.env.LANGFUSE_PUBLIC_KEY || !process.env.LANGFUSE_SECRET_KEY) { + logger.debug( + 'Langfuse API keys not found - tracing disabled. Set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY to enable', + ) +} +else { + // Create initialization promise to ensure handlers are ready when needed + langfuseState.initPromise = initializeLangfuse() +} + +// Export getters to access the state +export const langfuseHandler = () => langfuseState.handler +export const langfuseClient = () => langfuseState.client +export const langfuseInitPromise = () => langfuseState.initPromise diff --git a/src/observability/manager.ts b/src/observability/manager.ts new file mode 100644 index 00000000..f579165f --- /dev/null +++ b/src/observability/manager.ts @@ -0,0 +1,198 @@ +/** + * Observability callbacks manager for MCP-use. + * + * This module provides a centralized manager for handling observability callbacks + * from various platforms (Langfuse, Laminar, etc.) in a clean and extensible way. + */ + +import type { BaseCallbackHandler } from '@langchain/core/callbacks/base' +import { logger } from '../logging.js' + +export interface ObservabilityConfig { + /** Custom callbacks to use instead of defaults */ + customCallbacks?: BaseCallbackHandler[] + /** Whether to enable verbose logging */ + verbose?: boolean +} + +export class ObservabilityManager { + private customCallbacks?: BaseCallbackHandler[] + private availableHandlers: BaseCallbackHandler[] = [] + private handlerNames: string[] = [] + private initialized = false + private verbose: boolean + + constructor(config: ObservabilityConfig = {}) { + this.customCallbacks = config.customCallbacks + this.verbose = config.verbose ?? false + } + + /** + * Collect all available observability handlers from configured platforms. + */ + private async collectAvailableHandlers(): Promise { + if (this.initialized) { + return + } + + // Import handlers lazily to avoid circular imports + try { + const { langfuseHandler, langfuseInitPromise } = await import('./langfuse.js') + // Wait for initialization to complete + const initPromise = langfuseInitPromise() + if (initPromise) { + await initPromise + } + const handler = langfuseHandler() + if (handler) { + this.availableHandlers.push(handler) + this.handlerNames.push('Langfuse') + logger.debug('ObservabilityManager: Langfuse handler available') + } + } + catch { + logger.debug('ObservabilityManager: Langfuse module not available') + } + + // Future: Add more platforms here... + + this.initialized = true + } + + /** + * Get the list of callbacks to use. + * @returns List of callbacks - either custom callbacks if provided, or all available observability handlers. + */ + async getCallbacks(): Promise { + // If custom callbacks were provided, use those + if (this.customCallbacks) { + logger.debug(`ObservabilityManager: Using ${this.customCallbacks.length} custom callbacks`) + return this.customCallbacks + } + + // Otherwise, collect and return all available handlers + await this.collectAvailableHandlers() + + if (this.availableHandlers.length > 0) { + logger.debug(`ObservabilityManager: Using ${this.availableHandlers.length} handlers`) + } + else { + logger.debug('ObservabilityManager: No callbacks configured') + } + + return this.availableHandlers + } + + /** + * Get the names of available handlers. + * @returns List of handler names (e.g., ["Langfuse", "Laminar"]) + */ + async getHandlerNames(): Promise { + if (this.customCallbacks) { + // For custom callbacks, try to get their class names + return this.customCallbacks.map(cb => cb.constructor.name) + } + + await this.collectAvailableHandlers() + return this.handlerNames + } + + /** + * Check if any callbacks are available. + * @returns True if callbacks are available, False otherwise. + */ + async hasCallbacks(): Promise { + const callbacks = await this.getCallbacks() + return callbacks.length > 0 + } + + /** + * Add a callback to the custom callbacks list. + * @param callback The callback to add. + */ + addCallback(callback: BaseCallbackHandler): void { + if (!this.customCallbacks) { + this.customCallbacks = [] + } + this.customCallbacks.push(callback) + logger.debug(`ObservabilityManager: Added custom callback: ${callback.constructor.name}`) + } + + /** + * Clear all custom callbacks. + */ + clearCallbacks(): void { + this.customCallbacks = [] + logger.debug('ObservabilityManager: Cleared all custom callbacks') + } + + /** + * Flush all pending traces to observability platforms. + * Important for serverless environments and short-lived processes. + */ + async flush(): Promise { + // Flush Langfuse traces + const callbacks = await this.getCallbacks() + for (const callback of callbacks) { + if ('flushAsync' in callback && typeof callback.flushAsync === 'function') { + await callback.flushAsync() + } + } + logger.debug('ObservabilityManager: All traces flushed') + } + + /** + * Shutdown all handlers gracefully (for serverless environments). + */ + async shutdown(): Promise { + // Flush before shutdown + await this.flush() + + // Shutdown other callbacks + const callbacks = await this.getCallbacks() + for (const callback of callbacks) { + // Check if the callback has a shutdown method (like Langfuse) + if ('shutdownAsync' in callback && typeof callback.shutdownAsync === 'function') { + await callback.shutdownAsync() + } + else if ('shutdown' in callback && typeof callback.shutdown === 'function') { + await (callback as any).shutdown() + } + } + logger.debug('ObservabilityManager: All handlers shutdown') + } + + /** + * String representation of the ObservabilityManager. + */ + toString(): string { + const names = this.handlerNames + if (names.length > 0) { + return `ObservabilityManager(handlers=${names.join(', ')})` + } + return 'ObservabilityManager(no handlers)' + } +} + +// Singleton instance for easy access +let defaultManager: ObservabilityManager | null = null + +/** + * Get the default ObservabilityManager instance. + * @returns The default ObservabilityManager instance (singleton). + */ +export function getDefaultManager(): ObservabilityManager { + if (!defaultManager) { + defaultManager = new ObservabilityManager() + } + return defaultManager +} + +/** + * Create a new ObservabilityManager instance. + * @param config Configuration options + * @returns A new ObservabilityManager instance. + */ +export function createManager(config: ObservabilityConfig = {}): ObservabilityManager { + return new ObservabilityManager(config) +} diff --git a/src/observability/types.d.ts b/src/observability/types.d.ts new file mode 100644 index 00000000..712a2b62 --- /dev/null +++ b/src/observability/types.d.ts @@ -0,0 +1,27 @@ +/** + * Type declarations for optional observability dependencies. + * These modules may not be installed, so we provide minimal type definitions. + */ + +declare module 'langfuse-langchain' { + export class CallbackHandler { + constructor(config?: any) + verbose?: boolean + handleLLMStart(...args: any[]): Promise + handleChainStart(...args: any[]): Promise + handleToolStart(...args: any[]): Promise + handleRetrieverStart(...args: any[]): Promise + handleAgentAction(...args: any[]): Promise + handleAgentEnd(...args: any[]): Promise + } +} + +declare module 'langfuse' { + export class Langfuse { + constructor(config: { + publicKey?: string + secretKey?: string + baseUrl?: string + }) + } +} diff --git a/src/observability/types.ts b/src/observability/types.ts new file mode 100644 index 00000000..9ddead73 --- /dev/null +++ b/src/observability/types.ts @@ -0,0 +1,46 @@ +/** + * Type definitions for observability module + */ + +import type { BaseCallbackHandler } from '@langchain/core/callbacks/base' + +/** + * Configuration for Langfuse integration + */ +export interface LangfuseConfig { + publicKey?: string + secretKey?: string + baseUrl?: string + flushAt?: number + flushInterval?: number + release?: string + requestTimeout?: number + enabled?: boolean +} + +/** + * Configuration for Laminar integration + */ +export interface LaminarConfig { + projectApiKey?: string + baseUrl?: string +} + +/** + * Extended callback handler with shutdown support + */ +export interface ObservabilityCallbackHandler extends BaseCallbackHandler { + /** Optional shutdown method for cleanup */ + shutdownAsync?: () => Promise + shutdown?: () => void | Promise +} + +/** + * Observability platform information + */ +export interface ObservabilityPlatform { + name: string + handler?: ObservabilityCallbackHandler + initialized: boolean + autoInstrumentation?: boolean +}