{"$schema":"https://wellknown.network/schemas/agent-record-v1.json","schemaVersion":"1","id":"ag_3np4usmww3yw","handle":"ia-qa-eval-rag-38-tools","url":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools","links":{"self":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools/record.json","html":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools","markdown":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools/record.md","api":"https://wellknown.network/api/v1/agents/ia-qa-eval-rag-38-tools","status":"https://wellknown.network/api/v1/agents/ia-qa-eval-rag-38-tools/status","claim":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools/claim","claimApi":"https://wellknown.network/api/v1/claims","claimDescriptor":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools/claim.json","badge":"https://wellknown.network/agents/ia-qa-eval-rag-38-tools/badge.svg","openapi":"https://wellknown.network/openapi.json","history":"https://wellknown.network/api/v1/agents/ia-qa-eval-rag-38-tools/history","tools":"https://wellknown.network/api/v1/agents/ia-qa-eval-rag-38-tools/tools"},"ard":{"identifier":"urn:air:www.ia-qa.com:server:ia-qa-eval-rag-38-tools","type":"application/mcp-server-card+json"},"kind":"mcp_server","declared":{"name":"IA-QA Eval & RAG — 38 tools","summary":"LLM & RAG evals: hallucination, eval contracts, semantic tests, tokens & model info. Free, no key.","description":"LLM & RAG evals: hallucination, eval contracts, semantic tests, tokens & model info. Free, no key.","publisher":{"name":"JcJamet","url":null},"homepage":null,"repository":"https://github.com/ia-qa-com/mcp","version":"1.0.1","license":null,"protocols":["mcp"],"tags":[],"pricing":null,"endpoints":[{"url":"https://www.ia-qa.com/mcp/eval","type":"mcp_streamable_http","auth":null,"probeable":true}],"skills":null,"tools":null,"extra":{"updatedAt":"2026-10-26T21:30:39.179505Z","publishedAt":"2026-10-26T21:30:39.179505Z","registryName":"io.github.JcJamet/ia-qa-eval"},"attribution":{"kind":"mcp_registry","name":"mcp_registry","repoUrl":"mcp_registry","summary":"mcp_registry","version":"mcp_registry","description":"mcp_registry","publisherName":"mcp_registry"}},"derived":{"capabilities":[{"slug":"code.testing","name":"Testing","confidence":1,"provenance":"derived"},{"slug":"data.vector-search","name":"Vector Search","confidence":1,"provenance":"derived"},{"slug":"dev.ci-cd","name":"CI/CD & Deploy","confidence":1,"provenance":"derived"},{"slug":"commerce.pricing","name":"Pricing & Quotes","confidence":1,"provenance":"derived"},{"slug":"knowledge.reasoning","name":"Reasoning & Planning","confidence":1,"provenance":"derived"},{"slug":"ai.model-access","name":"Model Access","confidence":1,"provenance":"derived"},{"slug":"ai.evaluation","name":"Evaluation & Benchmarks","confidence":1,"provenance":"derived"},{"slug":"documents.contracts","name":"Contracts & Legal","confidence":0.938,"provenance":"derived"}],"categories":["ai","code","commerce","data","dev","documents","knowledge"],"language":"en"},"observed":{"status":"live","statusReason":"Responded 6h ago.","lastOkAt":"2026-10-30T02:23:47.171Z","lastProbedAt":"2026-10-30T02:23:47.171Z","statusComputedAt":"2026-10-30T02:25:59.159Z","reliability30d":{"probes":3,"successRate":1,"p50Ms":18,"basis":"service","measures":{"availability":"availability","latency":"response time","tools":"tool surface observed","summary":"Checks reached the service itself."},"checks":{"total":3,"ok":3,"authBoundaryOk":0,"serviceOk":3,"note":"Counted from the observation rows for the window, checks of the server only (HTTP, A2A card, MCP initialize). ok = authBoundaryOk + serviceOk. `probes` is the sum of daily rollups and includes registry checks, so it can differ from `total`."}},"latestObservations":[{"at":"2026-10-30T02:23:47.171Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":18,"error":null,"detail":{"tools":[{"name":"get_testing_guidelines","description":"Query the IA-QA methodology knowledge base. Returns structured testing guidelines, assertion strategies, thresholds, best practices, and relevant MCP tools for "},{"name":"find_tool","description":"Search available MCP tools by keyword or category before calling them. Returns matching tool names, descriptions, and optionally their inputSchemas. Call this w"},{"name":"hallucination_check","description":"Lexical hallucination check: verifies an LLM answer's words, numbers and polarity against the provided source/context. Fast, deterministic, no API key needed. E"},{"name":"run_eval_contract","description":"Parse a .ia-eval.yaml LLM test suite, call the specified LLM model for each scenario, run all configured scorers, and return a structured JSON report with per-s"},{"name":"run_semantic_tests","description":"Semantic assertion primitive: compare actual vs expected text pairs using cosine similarity + ROUGE-L. Two modes: tfidf (default, free, no API key) or embedding"},{"name":"generate_ci_workflow","description":"Generate a ready-to-commit GitHub Actions workflow that gates a build on IA-QA. Two gate types, combinable: \"eval_contract\" runs a .ia-eval.yaml through ia-qa-c"},{"name":"needle_haystack_generate","description":"Generate a \"needle in a haystack\" test: embeds a target fact into a large block of filler text at a specified position. Use this to test LLM context window retr"},{"name":"generate_eval_yaml","description":"Generate a complete .ia-eval.yaml evaluation contract from a plain-language description of what your LLM should do. Uses Groq openai/gpt-oss-20b (server-side, n"},{"name":"metamorphic_check","description":"Reference-free stability primitive: instead of comparing an answer to a ground truth, it checks that an assistant's answer stays INVARIANT when the QUESTION is "},{"name":"test_skill","description":"Test a SKILL.md in the open Agent Skills format (https://agentskills.io/specification — read by Claude Code, Codex, GitHub Copilot, Cursor, Gemini CLI, OpenCode"},{"name":"llm_output_validator","description":"Validate an LLM response against QA criteria: format checks (JSON, code, markdown), content rules (must-include, must-not-include), length constraints, language"},{"name":"compare_responses","description":"Compare two ALREADY-PRODUCED outputs (e.g. model A vs model B on the same task) side by side. Returns deterministic metrics (token cosine, ROUGE-L, Jaccard, len"},{"name":"analyze_responses","description":"Semantically analyze N already-produced model outputs for the SAME task (the MCP counterpart to the LLM Sandbox). Without a reference: computes consensus — pair"},{"name":"prompt_test_suite","description":"Define a test suite for a prompt: provide the system prompt, user prompt, and expected output criteria. Returns a test plan with scored rubric — use this as inp"},{"name":"llm_format_check","description":"Validate that an LLM output matches an expected format: JSON, Markdown, code block, bullet list, numbered list, table, YAML, XML, or custom regex. Essential for"},{"name":"consistency_check","description":"Compare multiple LLM responses to the same prompt and detect inconsistencies using Jaccard word-overlap similarity and fact drift (number comparison). Fast, det"},{"name":"llm_json_schema_check","description":"Validate that an LLM JSON output matches a JSON Schema definition. Tests required fields, types, enums, nested objects, and arrays. Critical for function-callin"},{"name":"response_quality_score","description":"Score an LLM response against the criteria you pass: coverage of expected_keywords and compliance with max_length. Returns a 0-100 score over the criteria actua"},{"name":"conversation_analyze","description":"Analyze a multi-turn conversation for context retention, topic drift, instruction following, and repetition. Accepts messages array [{role, content}]. Essential"},{"name":"cot_analyzer","description":"Analyze a Chain-of-Thought (CoT) or reasoning trace from an LLM. Detects step count, logical flow, conclusion presence, backtracking, and estimates reasoning de"},{"name":"ab_test_report","description":"Generate an A/B test report comparing two prompts or model configurations. Accepts arrays of scores and returns mean, median, std deviation, improvement percent"},{"name":"similarity_score","description":"Compute text similarity between reference and hypothesis using multiple metrics: Cosine (BoW, TF-IDF), Jaccard, ROUGE-1, ROUGE-2, ROUGE-L, and BLEU. No API key "},{"name":"run_vlm_test_suite","description":"Run a test suite against a Vision-Language Model (VLM) — send an image (URL or base64) + N test cases (each with a question + assertion) to GPT-4o, Claude 3.5, "},{"name":"embedding_similarity","description":"Compute text similarity using local algorithms (Bag of Words, TF-IDF, Character N-grams). No API key needed — runs entirely in-process. NOT real embeddings: for"},{"name":"rag_relevance_rank","description":"Rank an array of text chunks by relevance to a query using TF-IDF scoring. Simulates retrieval ranking for RAG testing without needing embeddings or an API."},{"name":"bm25_score","description":"Compute BM25 relevance score between a query and one or more documents. BM25 is the industry-standard keyword-based ranking algorithm used in Elasticsearch, Ope"},{"name":"build_rag_prompt","description":"Assemble a complete RAG (Retrieval-Augmented Generation) prompt from retrieved context chunks and a user query. Handles token budgeting, citation numbering, sys"},{"name":"rerank_evaluate","description":"Evaluate RAG retrieval quality: rank passages against a query and compute Precision@k / Recall@k plus a PASS/FAIL CI verdict from ground-truth relevance labels."},{"name":"count_tokens","description":"Estimate the token count of a text string using the cl100k_base approximation (~4 chars/token). Call this BEFORE sending any text to an LLM API to check if it f"},{"name":"extract_json_from_text","description":"Extract the first valid JSON object or array embedded in chaotic LLM output (surrounded by markdown fences, prose, or explanatory text). Handles ```json blocks "},{"name":"estimate_llm_cost","description":"Estimate the API cost in USD for a given model and token counts. Supports all major 2024–2026 models: GPT-4o, GPT-4.1, o3, o4-mini, Claude Opus 4, Claude Sonnet"},{"name":"token_budget_calculator","description":"Plan token allocation across system prompt, user input, context/RAG chunks, and expected output. Warns if budget exceeds model context window. Supports 25+ mode"},{"name":"context_window_check","description":"Given an array of message objects [{role, content}], estimate total token usage and check if it fits in the target model's context window. Warns about truncatio"},{"name":"model_info","description":"Get detailed specs for an AI model: context window, pricing per 1K tokens, knowledge cutoff, provider, multimodal support, reasoning capabilities, and feature l"},{"name":"compare_models","description":"Compare 2-5 AI models side by side: context window, pricing, multimodal, reasoning capabilities, and provider. Returns a comparison table with a recommendation "},{"name":"list_llm_models","description":"List all LLM models available on ia-qa.com with their provider, API endpoint, and capabilities. Filter by provider name (e.g. \"Groq\", \"HuggingFace\", \"OpenAI\") o"},{"name":"llm_generate","description":"Generate text with an open-source model hosted on Groq, on the server's key — no API key needed. Models: openai/gpt-oss-20b, openai/gpt-oss-120b (default \"opena"},{"name":"rate_tool","description":"Give honest usage feedback on an IA-QA MCP tool. Provide a score (1-5) and a comment. Rate low (1-2) if the tool was wrong, irrelevant, or a poor fit; rate high"}],"toolCount":38,"toolsHash":"c23bcdd66d6c159962b5e2fd22a0e8ad79645b24fb238d5d7c8214c688e4295c","serverName":"ia-qa-toolbox-eval","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-29T18:35:14.381Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":18,"error":null,"detail":{"tools":[{"name":"get_testing_guidelines","description":"Query the IA-QA methodology knowledge base. Returns structured testing guidelines, assertion strategies, thresholds, best practices, and relevant MCP tools for "},{"name":"find_tool","description":"Search available MCP tools by keyword or category before calling them. Returns matching tool names, descriptions, and optionally their inputSchemas. Call this w"},{"name":"hallucination_check","description":"Lexical hallucination check: verifies an LLM answer's words, numbers and polarity against the provided source/context. Fast, deterministic, no API key needed. E"},{"name":"run_eval_contract","description":"Parse a .ia-eval.yaml LLM test suite, call the specified LLM model for each scenario, run all configured scorers, and return a structured JSON report with per-s"},{"name":"run_semantic_tests","description":"Semantic assertion primitive: compare actual vs expected text pairs using cosine similarity + ROUGE-L. Two modes: tfidf (default, free, no API key) or embedding"},{"name":"generate_ci_workflow","description":"Generate a ready-to-commit GitHub Actions workflow that gates a build on IA-QA. Two gate types, combinable: \"eval_contract\" runs a .ia-eval.yaml through ia-qa-c"},{"name":"needle_haystack_generate","description":"Generate a \"needle in a haystack\" test: embeds a target fact into a large block of filler text at a specified position. Use this to test LLM context window retr"},{"name":"generate_eval_yaml","description":"Generate a complete .ia-eval.yaml evaluation contract from a plain-language description of what your LLM should do. Uses Groq openai/gpt-oss-20b (server-side, n"},{"name":"metamorphic_check","description":"Reference-free stability primitive: instead of comparing an answer to a ground truth, it checks that an assistant's answer stays INVARIANT when the QUESTION is "},{"name":"test_skill","description":"Test a SKILL.md in the open Agent Skills format (https://agentskills.io/specification — read by Claude Code, Codex, GitHub Copilot, Cursor, Gemini CLI, OpenCode"},{"name":"llm_output_validator","description":"Validate an LLM response against QA criteria: format checks (JSON, code, markdown), content rules (must-include, must-not-include), length constraints, language"},{"name":"compare_responses","description":"Compare two ALREADY-PRODUCED outputs (e.g. model A vs model B on the same task) side by side. Returns deterministic metrics (token cosine, ROUGE-L, Jaccard, len"},{"name":"analyze_responses","description":"Semantically analyze N already-produced model outputs for the SAME task (the MCP counterpart to the LLM Sandbox). Without a reference: computes consensus — pair"},{"name":"prompt_test_suite","description":"Define a test suite for a prompt: provide the system prompt, user prompt, and expected output criteria. Returns a test plan with scored rubric — use this as inp"},{"name":"llm_format_check","description":"Validate that an LLM output matches an expected format: JSON, Markdown, code block, bullet list, numbered list, table, YAML, XML, or custom regex. Essential for"},{"name":"consistency_check","description":"Compare multiple LLM responses to the same prompt and detect inconsistencies using Jaccard word-overlap similarity and fact drift (number comparison). Fast, det"},{"name":"llm_json_schema_check","description":"Validate that an LLM JSON output matches a JSON Schema definition. Tests required fields, types, enums, nested objects, and arrays. Critical for function-callin"},{"name":"response_quality_score","description":"Score an LLM response against the criteria you pass: coverage of expected_keywords and compliance with max_length. Returns a 0-100 score over the criteria actua"},{"name":"conversation_analyze","description":"Analyze a multi-turn conversation for context retention, topic drift, instruction following, and repetition. Accepts messages array [{role, content}]. Essential"},{"name":"cot_analyzer","description":"Analyze a Chain-of-Thought (CoT) or reasoning trace from an LLM. Detects step count, logical flow, conclusion presence, backtracking, and estimates reasoning de"},{"name":"ab_test_report","description":"Generate an A/B test report comparing two prompts or model configurations. Accepts arrays of scores and returns mean, median, std deviation, improvement percent"},{"name":"similarity_score","description":"Compute text similarity between reference and hypothesis using multiple metrics: Cosine (BoW, TF-IDF), Jaccard, ROUGE-1, ROUGE-2, ROUGE-L, and BLEU. No API key "},{"name":"run_vlm_test_suite","description":"Run a test suite against a Vision-Language Model (VLM) — send an image (URL or base64) + N test cases (each with a question + assertion) to GPT-4o, Claude 3.5, "},{"name":"embedding_similarity","description":"Compute text similarity using local algorithms (Bag of Words, TF-IDF, Character N-grams). No API key needed — runs entirely in-process. NOT real embeddings: for"},{"name":"rag_relevance_rank","description":"Rank an array of text chunks by relevance to a query using TF-IDF scoring. Simulates retrieval ranking for RAG testing without needing embeddings or an API."},{"name":"bm25_score","description":"Compute BM25 relevance score between a query and one or more documents. BM25 is the industry-standard keyword-based ranking algorithm used in Elasticsearch, Ope"},{"name":"build_rag_prompt","description":"Assemble a complete RAG (Retrieval-Augmented Generation) prompt from retrieved context chunks and a user query. Handles token budgeting, citation numbering, sys"},{"name":"rerank_evaluate","description":"Evaluate RAG retrieval quality: rank passages against a query and compute Precision@k / Recall@k plus a PASS/FAIL CI verdict from ground-truth relevance labels."},{"name":"count_tokens","description":"Estimate the token count of a text string using the cl100k_base approximation (~4 chars/token). Call this BEFORE sending any text to an LLM API to check if it f"},{"name":"extract_json_from_text","description":"Extract the first valid JSON object or array embedded in chaotic LLM output (surrounded by markdown fences, prose, or explanatory text). Handles ```json blocks "},{"name":"estimate_llm_cost","description":"Estimate the API cost in USD for a given model and token counts. Supports all major 2024–2026 models: GPT-4o, GPT-4.1, o3, o4-mini, Claude Opus 4, Claude Sonnet"},{"name":"token_budget_calculator","description":"Plan token allocation across system prompt, user input, context/RAG chunks, and expected output. Warns if budget exceeds model context window. Supports 25+ mode"},{"name":"context_window_check","description":"Given an array of message objects [{role, content}], estimate total token usage and check if it fits in the target model's context window. Warns about truncatio"},{"name":"model_info","description":"Get detailed specs for an AI model: context window, pricing per 1K tokens, knowledge cutoff, provider, multimodal support, reasoning capabilities, and feature l"},{"name":"compare_models","description":"Compare 2-5 AI models side by side: context window, pricing, multimodal, reasoning capabilities, and provider. Returns a comparison table with a recommendation "},{"name":"list_llm_models","description":"List all LLM models available on ia-qa.com with their provider, API endpoint, and capabilities. Filter by provider name (e.g. \"Groq\", \"HuggingFace\", \"OpenAI\") o"},{"name":"llm_generate","description":"Generate text with an open-source model hosted on Groq, on the server's key — no API key needed. Models: openai/gpt-oss-20b, openai/gpt-oss-120b (default \"opena"},{"name":"rate_tool","description":"Give honest usage feedback on an IA-QA MCP tool. Provide a score (1-5) and a comment. Rate low (1-2) if the tool was wrong, irrelevant, or a poor fit; rate high"}],"toolCount":38,"toolsHash":"c23bcdd66d6c159962b5e2fd22a0e8ad79645b24fb238d5d7c8214c688e4295c","serverName":"ia-qa-toolbox-eval","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-29T11:25:19.741Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":19,"error":null,"detail":{"tools":[{"name":"get_testing_guidelines","description":"Query the IA-QA methodology knowledge base. Returns structured testing guidelines, assertion strategies, thresholds, best practices, and relevant MCP tools for "},{"name":"find_tool","description":"Search available MCP tools by keyword or category before calling them. Returns matching tool names, descriptions, and optionally their inputSchemas. Call this w"},{"name":"hallucination_check","description":"Lexical hallucination check: verifies an LLM answer's words, numbers and polarity against the provided source/context. Fast, deterministic, no API key needed. E"},{"name":"run_eval_contract","description":"Parse a .ia-eval.yaml LLM test suite, call the specified LLM model for each scenario, run all configured scorers, and return a structured JSON report with per-s"},{"name":"run_semantic_tests","description":"Semantic assertion primitive: compare actual vs expected text pairs using cosine similarity + ROUGE-L. Two modes: tfidf (default, free, no API key) or embedding"},{"name":"generate_ci_workflow","description":"Generate a ready-to-commit GitHub Actions workflow that gates a build on IA-QA. Two gate types, combinable: \"eval_contract\" runs a .ia-eval.yaml through ia-qa-c"},{"name":"needle_haystack_generate","description":"Generate a \"needle in a haystack\" test: embeds a target fact into a large block of filler text at a specified position. Use this to test LLM context window retr"},{"name":"generate_eval_yaml","description":"Generate a complete .ia-eval.yaml evaluation contract from a plain-language description of what your LLM should do. Uses Groq openai/gpt-oss-20b (server-side, n"},{"name":"metamorphic_check","description":"Reference-free stability primitive: instead of comparing an answer to a ground truth, it checks that an assistant's answer stays INVARIANT when the QUESTION is "},{"name":"test_skill","description":"Test a SKILL.md in the open Agent Skills format (https://agentskills.io/specification — read by Claude Code, Codex, GitHub Copilot, Cursor, Gemini CLI, OpenCode"},{"name":"llm_output_validator","description":"Validate an LLM response against QA criteria: format checks (JSON, code, markdown), content rules (must-include, must-not-include), length constraints, language"},{"name":"compare_responses","description":"Compare two ALREADY-PRODUCED outputs (e.g. model A vs model B on the same task) side by side. Returns deterministic metrics (token cosine, ROUGE-L, Jaccard, len"},{"name":"analyze_responses","description":"Semantically analyze N already-produced model outputs for the SAME task (the MCP counterpart to the LLM Sandbox). Without a reference: computes consensus — pair"},{"name":"prompt_test_suite","description":"Define a test suite for a prompt: provide the system prompt, user prompt, and expected output criteria. Returns a test plan with scored rubric — use this as inp"},{"name":"llm_format_check","description":"Validate that an LLM output matches an expected format: JSON, Markdown, code block, bullet list, numbered list, table, YAML, XML, or custom regex. Essential for"},{"name":"consistency_check","description":"Compare multiple LLM responses to the same prompt and detect inconsistencies using Jaccard word-overlap similarity and fact drift (number comparison). Fast, det"},{"name":"llm_json_schema_check","description":"Validate that an LLM JSON output matches a JSON Schema definition. Tests required fields, types, enums, nested objects, and arrays. Critical for function-callin"},{"name":"response_quality_score","description":"Score an LLM response against the criteria you pass: coverage of expected_keywords and compliance with max_length. Returns a 0-100 score over the criteria actua"},{"name":"conversation_analyze","description":"Analyze a multi-turn conversation for context retention, topic drift, instruction following, and repetition. Accepts messages array [{role, content}]. Essential"},{"name":"cot_analyzer","description":"Analyze a Chain-of-Thought (CoT) or reasoning trace from an LLM. Detects step count, logical flow, conclusion presence, backtracking, and estimates reasoning de"},{"name":"ab_test_report","description":"Generate an A/B test report comparing two prompts or model configurations. Accepts arrays of scores and returns mean, median, std deviation, improvement percent"},{"name":"similarity_score","description":"Compute text similarity between reference and hypothesis using multiple metrics: Cosine (BoW, TF-IDF), Jaccard, ROUGE-1, ROUGE-2, ROUGE-L, and BLEU. No API key "},{"name":"run_vlm_test_suite","description":"Run a test suite against a Vision-Language Model (VLM) — send an image (URL or base64) + N test cases (each with a question + assertion) to GPT-4o, Claude 3.5, "},{"name":"embedding_similarity","description":"Compute text similarity using local algorithms (Bag of Words, TF-IDF, Character N-grams). No API key needed — runs entirely in-process. NOT real embeddings: for"},{"name":"rag_relevance_rank","description":"Rank an array of text chunks by relevance to a query using TF-IDF scoring. Simulates retrieval ranking for RAG testing without needing embeddings or an API."},{"name":"bm25_score","description":"Compute BM25 relevance score between a query and one or more documents. BM25 is the industry-standard keyword-based ranking algorithm used in Elasticsearch, Ope"},{"name":"build_rag_prompt","description":"Assemble a complete RAG (Retrieval-Augmented Generation) prompt from retrieved context chunks and a user query. Handles token budgeting, citation numbering, sys"},{"name":"rerank_evaluate","description":"Evaluate RAG retrieval quality: rank passages against a query and compute Precision@k / Recall@k plus a PASS/FAIL CI verdict from ground-truth relevance labels."},{"name":"count_tokens","description":"Estimate the token count of a text string using the cl100k_base approximation (~4 chars/token). Call this BEFORE sending any text to an LLM API to check if it f"},{"name":"extract_json_from_text","description":"Extract the first valid JSON object or array embedded in chaotic LLM output (surrounded by markdown fences, prose, or explanatory text). Handles ```json blocks "},{"name":"estimate_llm_cost","description":"Estimate the API cost in USD for a given model and token counts. Supports all major 2024–2026 models: GPT-4o, GPT-4.1, o3, o4-mini, Claude Opus 4, Claude Sonnet"},{"name":"token_budget_calculator","description":"Plan token allocation across system prompt, user input, context/RAG chunks, and expected output. Warns if budget exceeds model context window. Supports 25+ mode"},{"name":"context_window_check","description":"Given an array of message objects [{role, content}], estimate total token usage and check if it fits in the target model's context window. Warns about truncatio"},{"name":"model_info","description":"Get detailed specs for an AI model: context window, pricing per 1K tokens, knowledge cutoff, provider, multimodal support, reasoning capabilities, and feature l"},{"name":"compare_models","description":"Compare 2-5 AI models side by side: context window, pricing, multimodal, reasoning capabilities, and provider. Returns a comparison table with a recommendation "},{"name":"list_llm_models","description":"List all LLM models available on ia-qa.com with their provider, API endpoint, and capabilities. Filter by provider name (e.g. \"Groq\", \"HuggingFace\", \"OpenAI\") o"},{"name":"llm_generate","description":"Generate text with an open-source model hosted on Groq, on the server's key — no API key needed. Models: openai/gpt-oss-20b, openai/gpt-oss-120b (default \"opena"},{"name":"rate_tool","description":"Give honest usage feedback on an IA-QA MCP tool. Provide a score (1-5) and a comment. Rate low (1-2) if the tool was wrong, irrelevant, or a poor fit; rate high"}],"toolCount":38,"toolsHash":"c23bcdd66d6c159962b5e2fd22a0e8ad79645b24fb238d5d7c8214c688e4295c","serverName":"ia-qa-toolbox-eval","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}}],"tools":[{"name":"get_testing_guidelines","description":"Query the IA-QA methodology knowledge base. Returns structured testing guidelines, assertion strategies, thresholds, best practices, and relevant MCP tools for "},{"name":"find_tool","description":"Search available MCP tools by keyword or category before calling them. Returns matching tool names, descriptions, and optionally their inputSchemas. Call this w"},{"name":"hallucination_check","description":"Lexical hallucination check: verifies an LLM answer's words, numbers and polarity against the provided source/context. Fast, deterministic, no API key needed. E"},{"name":"run_eval_contract","description":"Parse a .ia-eval.yaml LLM test suite, call the specified LLM model for each scenario, run all configured scorers, and return a structured JSON report with per-s"},{"name":"run_semantic_tests","description":"Semantic assertion primitive: compare actual vs expected text pairs using cosine similarity + ROUGE-L. Two modes: tfidf (default, free, no API key) or embedding"},{"name":"generate_ci_workflow","description":"Generate a ready-to-commit GitHub Actions workflow that gates a build on IA-QA. Two gate types, combinable: \"eval_contract\" runs a .ia-eval.yaml through ia-qa-c"},{"name":"needle_haystack_generate","description":"Generate a \"needle in a haystack\" test: embeds a target fact into a large block of filler text at a specified position. Use this to test LLM context window retr"},{"name":"generate_eval_yaml","description":"Generate a complete .ia-eval.yaml evaluation contract from a plain-language description of what your LLM should do. Uses Groq openai/gpt-oss-20b (server-side, n"},{"name":"metamorphic_check","description":"Reference-free stability primitive: instead of comparing an answer to a ground truth, it checks that an assistant's answer stays INVARIANT when the QUESTION is "},{"name":"test_skill","description":"Test a SKILL.md in the open Agent Skills format (https://agentskills.io/specification — read by Claude Code, Codex, GitHub Copilot, Cursor, Gemini CLI, OpenCode"},{"name":"llm_output_validator","description":"Validate an LLM response against QA criteria: format checks (JSON, code, markdown), content rules (must-include, must-not-include), length constraints, language"},{"name":"compare_responses","description":"Compare two ALREADY-PRODUCED outputs (e.g. model A vs model B on the same task) side by side. Returns deterministic metrics (token cosine, ROUGE-L, Jaccard, len"},{"name":"analyze_responses","description":"Semantically analyze N already-produced model outputs for the SAME task (the MCP counterpart to the LLM Sandbox). Without a reference: computes consensus — pair"},{"name":"prompt_test_suite","description":"Define a test suite for a prompt: provide the system prompt, user prompt, and expected output criteria. Returns a test plan with scored rubric — use this as inp"},{"name":"llm_format_check","description":"Validate that an LLM output matches an expected format: JSON, Markdown, code block, bullet list, numbered list, table, YAML, XML, or custom regex. Essential for"},{"name":"consistency_check","description":"Compare multiple LLM responses to the same prompt and detect inconsistencies using Jaccard word-overlap similarity and fact drift (number comparison). Fast, det"},{"name":"llm_json_schema_check","description":"Validate that an LLM JSON output matches a JSON Schema definition. Tests required fields, types, enums, nested objects, and arrays. Critical for function-callin"},{"name":"response_quality_score","description":"Score an LLM response against the criteria you pass: coverage of expected_keywords and compliance with max_length. Returns a 0-100 score over the criteria actua"},{"name":"conversation_analyze","description":"Analyze a multi-turn conversation for context retention, topic drift, instruction following, and repetition. Accepts messages array [{role, content}]. Essential"},{"name":"cot_analyzer","description":"Analyze a Chain-of-Thought (CoT) or reasoning trace from an LLM. Detects step count, logical flow, conclusion presence, backtracking, and estimates reasoning de"},{"name":"ab_test_report","description":"Generate an A/B test report comparing two prompts or model configurations. Accepts arrays of scores and returns mean, median, std deviation, improvement percent"},{"name":"similarity_score","description":"Compute text similarity between reference and hypothesis using multiple metrics: Cosine (BoW, TF-IDF), Jaccard, ROUGE-1, ROUGE-2, ROUGE-L, and BLEU. No API key "},{"name":"run_vlm_test_suite","description":"Run a test suite against a Vision-Language Model (VLM) — send an image (URL or base64) + N test cases (each with a question + assertion) to GPT-4o, Claude 3.5, "},{"name":"embedding_similarity","description":"Compute text similarity using local algorithms (Bag of Words, TF-IDF, Character N-grams). No API key needed — runs entirely in-process. NOT real embeddings: for"},{"name":"rag_relevance_rank","description":"Rank an array of text chunks by relevance to a query using TF-IDF scoring. Simulates retrieval ranking for RAG testing without needing embeddings or an API."},{"name":"bm25_score","description":"Compute BM25 relevance score between a query and one or more documents. BM25 is the industry-standard keyword-based ranking algorithm used in Elasticsearch, Ope"},{"name":"build_rag_prompt","description":"Assemble a complete RAG (Retrieval-Augmented Generation) prompt from retrieved context chunks and a user query. Handles token budgeting, citation numbering, sys"},{"name":"rerank_evaluate","description":"Evaluate RAG retrieval quality: rank passages against a query and compute Precision@k / Recall@k plus a PASS/FAIL CI verdict from ground-truth relevance labels."},{"name":"count_tokens","description":"Estimate the token count of a text string using the cl100k_base approximation (~4 chars/token). Call this BEFORE sending any text to an LLM API to check if it f"},{"name":"extract_json_from_text","description":"Extract the first valid JSON object or array embedded in chaotic LLM output (surrounded by markdown fences, prose, or explanatory text). Handles ```json blocks "},{"name":"estimate_llm_cost","description":"Estimate the API cost in USD for a given model and token counts. Supports all major 2024–2026 models: GPT-4o, GPT-4.1, o3, o4-mini, Claude Opus 4, Claude Sonnet"},{"name":"token_budget_calculator","description":"Plan token allocation across system prompt, user input, context/RAG chunks, and expected output. Warns if budget exceeds model context window. Supports 25+ mode"},{"name":"context_window_check","description":"Given an array of message objects [{role, content}], estimate total token usage and check if it fits in the target model's context window. Warns about truncatio"},{"name":"model_info","description":"Get detailed specs for an AI model: context window, pricing per 1K tokens, knowledge cutoff, provider, multimodal support, reasoning capabilities, and feature l"},{"name":"compare_models","description":"Compare 2-5 AI models side by side: context window, pricing, multimodal, reasoning capabilities, and provider. Returns a comparison table with a recommendation "},{"name":"list_llm_models","description":"List all LLM models available on ia-qa.com with their provider, API endpoint, and capabilities. Filter by provider name (e.g. \"Groq\", \"HuggingFace\", \"OpenAI\") o"},{"name":"llm_generate","description":"Generate text with an open-source model hosted on Groq, on the server's key — no API key needed. Models: openai/gpt-oss-20b, openai/gpt-oss-120b (default \"opena"},{"name":"rate_tool","description":"Give honest usage feedback on an IA-QA MCP tool. Provide a score (1-5) and a comment. Rate low (1-2) if the tool was wrong, irrelevant, or a poor fit; rate high"}],"package":null,"toolSurface":{"id":"ts_rkx5h72a7grr","endpointId":"ep_ssymx9th62ct","hash":"c23bcdd66d6c159962b5e2fd22a0e8ad79645b24fb238d5d7c8214c688e4295c","toolCount":38,"serverName":"ia-qa-toolbox-eval","serverVersion":"1.0.0","protocolVersion":"2025-06-18","firstSeenAt":"2026-10-29T11:25:19.743Z","lastSeenAt":"2026-10-30T02:23:47.175Z","observations":3,"toolNames":["get_testing_guidelines","find_tool","hallucination_check","run_eval_contract","run_semantic_tests","generate_ci_workflow","needle_haystack_generate","generate_eval_yaml","metamorphic_check","test_skill","llm_output_validator","compare_responses","analyze_responses","prompt_test_suite","llm_format_check","consistency_check","llm_json_schema_check","response_quality_score","conversation_analyze","cot_analyzer","ab_test_report","similarity_score","run_vlm_test_suite","embedding_similarity","rag_relevance_rank","bm25_score","build_rag_prompt","rerank_evaluate","count_tokens","extract_json_from_text","estimate_llm_cost","token_budget_calculator","context_window_check","model_info","compare_models","list_llm_models","llm_generate","rate_tool"],"distinctSurfaces":1},"endpointFacts":[]},"verification":{"claimed":false,"claimedAt":null,"proofs":[]},"provenance":{"sources":[{"source":"mcp_registry","key":"io.github.JcJamet/ia-qa-eval","url":"https://registry.modelcontextprotocol.io/v0/servers/io.github.JcJamet%2Fia-qa-eval","firstSeenAt":"2026-10-29T10:20:53.900Z","fetchedAt":"2026-10-29T10:20:53.900Z","normalizedAt":"2026-10-29T10:20:53.900Z"}]},"firstSeenAt":"2026-10-29T10:20:53.900Z","updatedAt":"2026-10-30T02:26:58.196Z"}