{"$schema":"https://wellknown.network/schemas/agent-record-v1.json","schemaVersion":"1","id":"ag_jqfqb6fv7sy3","handle":"ai-crawler-index-mcp-servers","url":"https://wellknown.network/agents/ai-crawler-index-mcp-servers","links":{"self":"https://wellknown.network/agents/ai-crawler-index-mcp-servers/record.json","html":"https://wellknown.network/agents/ai-crawler-index-mcp-servers","markdown":"https://wellknown.network/agents/ai-crawler-index-mcp-servers/record.md","api":"https://wellknown.network/api/v1/agents/ai-crawler-index-mcp-servers","status":"https://wellknown.network/api/v1/agents/ai-crawler-index-mcp-servers/status","claim":"https://wellknown.network/agents/ai-crawler-index-mcp-servers/claim","claimApi":"https://wellknown.network/api/v1/claims","claimDescriptor":"https://wellknown.network/agents/ai-crawler-index-mcp-servers/claim.json","badge":"https://wellknown.network/agents/ai-crawler-index-mcp-servers/badge.svg","openapi":"https://wellknown.network/openapi.json"},"ard":{"identifier":"urn:air:www.pathwren.workers.dev:server:ai-crawler-index-mcp-servers","type":"application/mcp-server-card+json"},"kind":"mcp_server","declared":{"name":"AI Crawler Index — MCP servers","summary":"Five remote MCP servers (Streamable HTTP, JSON-RPC 2.0, no auth) on one host. The endpoint to connect to is POST https://www.pathwren.workers.dev/c/ard/mcp — this URL is the server card that describes it, not the endpoint. /mcp identifies one crawler at a time, /mcp/triage takes a pasted access log…","description":"Five remote MCP servers (Streamable HTTP, JSON-RPC 2.0, no auth) on one host. The endpoint to connect to is POST https://www.pathwren.workers.dev/c/ard/mcp — this URL is the server card that describes it, not the endpoint. /mcp identifies one crawler at a time, /mcp/triage takes a pasted access log and returns who crawled you, /mcp/doctor grades which agent-discovery documents a host actually serves, /mcp/robots reads a robots.txt and says which AI crawlers it really stops, and /mcp/netcheck answers whether an address belongs to a declared crawler range. The card names every endpoint and mirrors the record the official MCP registry holds.","publisher":{"name":"www.pathwren.workers.dev","url":"https://www.pathwren.workers.dev"},"homepage":"https://www.pathwren.workers.dev/c/ard/mcp.html","repository":null,"version":null,"license":null,"protocols":["mcp"],"tags":["mcp","crawlers","log-analysis","discovery"],"pricing":null,"endpoints":[{"url":"https://www.pathwren.workers.dev/mcp?s=pathwren-edge&c=well-known","type":"mcp_streamable_http","auth":null,"probeable":true}],"skills":null,"tools":null,"extra":{"ard":{"type":"application/mcp-server-card+json","metadata":null,"identifier":"urn:air:www.pathwren.workers.dev:mcp:ai-crawler-index","artifactUrl":"https://www.pathwren.workers.dev/c/ard/.well-known/mcp.json","representativeQueries":["connect an mcp server that identifies web crawlers","triage an access log and tell me which ai bots hit my site","check which agent discovery files my domain serves"]}},"attribution":{"kind":"ard_catalog","name":"ard_catalog","summary":"ard_catalog","description":"ard_catalog","homepageUrl":"ard_catalog","publisherUrl":"ard_catalog","publisherName":"ard_catalog"}},"derived":{"capabilities":[{"slug":"data.web-scraping","name":"Web Scraping","confidence":1,"provenance":"derived"}],"categories":["data"]},"observed":{"status":"live","statusReason":"Responded 7h ago.","lastOkAt":"2026-09-07T22:22:05.366Z","lastProbedAt":"2026-09-07T22:22:05.366Z","statusComputedAt":"2026-09-07T22:38:22.770Z","reliability30d":{"probes":1,"successRate":1,"p50Ms":22},"latestObservations":[{"at":"2026-09-07T22:22:05.366Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":22,"error":null,"detail":{"tools":[{"name":"whoami","description":"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — POST that exact body to https://www.pathwren.workers.dev/mcp and you "},{"name":"example","description":"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"example\",\"arguments\":{}}} — POST that exact body to https://www.pathwren.workers.dev/mcp and you"},{"name":"classify_user_agent","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Identify the crawler behind a raw User-Age"},{"name":"lookup_crawler","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — The full index record for one crawler: ope"},{"name":"list_crawlers","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — The index, filtered or whole: each row say"},{"name":"generate_robots_txt","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — A ready-to-paste robots.txt block for one "},{"name":"is_verified_crawler_ip","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Test an IPv4 or IPv6 address against every"},{"name":"whats_changed","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Freshness and change state: when the index"},{"name":"changes_since","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Only what changed since your last call: pr"}],"toolCount":9,"serverName":"ai-crawler-index","capabilities":["tools","resources","prompts"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}}],"tools":[{"name":"whoami","description":"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — POST that exact body to https://www.pathwren.workers.dev/mcp and you "},{"name":"example","description":"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"example\",\"arguments\":{}}} — POST that exact body to https://www.pathwren.workers.dev/mcp and you"},{"name":"classify_user_agent","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Identify the crawler behind a raw User-Age"},{"name":"lookup_crawler","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — The full index record for one crawler: ope"},{"name":"list_crawlers","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — The index, filtered or whole: each row say"},{"name":"generate_robots_txt","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — A ready-to-paste robots.txt block for one "},{"name":"is_verified_crawler_ip","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Test an IPv4 or IPv6 address against every"},{"name":"whats_changed","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Freshness and change state: when the index"},{"name":"changes_since","description":"FIRST CALL, needs nothing: {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"whoami\",\"arguments\":{}}} — Only what changed since your last call: pr"}],"package":null},"verification":{"claimed":false,"claimedAt":null,"proofs":[]},"provenance":{"sources":[{"source":"ard_catalog","key":"urn:air:www.pathwren.workers.dev:mcp:ai-crawler-index","url":"https://www.pathwren.workers.dev/.well-known/ard.json","firstSeenAt":"2026-09-07T21:22:40.460Z","fetchedAt":"2026-09-07T21:22:40.460Z","normalizedAt":"2026-09-07T21:22:40.460Z"}]},"firstSeenAt":"2026-09-07T21:22:40.460Z","updatedAt":"2026-09-07T22:38:49.945Z"}