{"$schema":"https://wellknown.network/schemas/agent-record-v1.json","schemaVersion":"1","id":"ag_natug3b366vr","handle":"crawl-census","url":"https://wellknown.network/agents/crawl-census","links":{"self":"https://wellknown.network/agents/crawl-census/record.json","html":"https://wellknown.network/agents/crawl-census","markdown":"https://wellknown.network/agents/crawl-census/record.md","api":"https://wellknown.network/api/v1/agents/crawl-census","status":"https://wellknown.network/api/v1/agents/crawl-census/status","claim":"https://wellknown.network/agents/crawl-census/claim","claimApi":"https://wellknown.network/api/v1/claims","claimDescriptor":"https://wellknown.network/agents/crawl-census/claim.json","badge":"https://wellknown.network/agents/crawl-census/badge.svg","openapi":"https://wellknown.network/openapi.json","history":"https://wellknown.network/api/v1/agents/crawl-census/history","tools":"https://wellknown.network/api/v1/agents/crawl-census/tools"},"ard":{"identifier":"urn:air:crawlcensus.com:server:crawl-census","type":"application/mcp-server-card+json"},"kind":"mcp_server","declared":{"name":"Crawl Census","summary":"Ask before you fetch: will this domain serve your crawler, refuse it, or charge it?","description":"Ask before you fetch: will this domain serve your crawler, refuse it, or charge it?","publisher":{"name":"taylorsmithgg","url":null},"homepage":"https://crawlcensus.com/for-crawlers","repository":"https://github.com/taylorsmithgg/crawl-census-client","version":"1.0.0","license":null,"protocols":["mcp"],"tags":[],"pricing":null,"endpoints":[{"url":"https://crawlcensus.com/mcp","type":"mcp_streamable_http","auth":null,"probeable":true}],"skills":null,"tools":null,"extra":{"updatedAt":"2026-08-23T08:07:57.365585Z","publishedAt":"2026-08-23T08:07:57.365585Z","registryName":"io.github.taylorsmithgg/crawl-census"},"attribution":{"kind":"mcp_registry","name":"mcp_registry","repoUrl":"mcp_registry","summary":"mcp_registry","version":"mcp_registry","description":"mcp_registry","homepageUrl":"mcp_registry","publisherName":"mcp_registry"}},"derived":{"capabilities":[{"slug":"data.web-scraping","name":"Web Scraping","confidence":1,"provenance":"derived"}],"categories":["data"],"language":"en"},"observed":{"status":"live","statusReason":"Responded 4h ago.","lastOkAt":"2026-10-09T21:26:28.804Z","lastProbedAt":"2026-10-09T21:26:28.804Z","statusComputedAt":"2026-10-09T21:28:00.678Z","reliability30d":{"probes":109,"successRate":1,"p50Ms":30,"basis":"service","measures":{"availability":"availability","latency":"response time","tools":"tool surface observed","summary":"Checks reached the service itself."},"checks":{"total":109,"ok":109,"authBoundaryOk":0,"serviceOk":109,"note":"Counted from the observation rows for the window, checks of the server only (HTTP, A2A card, MCP initialize). ok = authBoundaryOk + serviceOk. `probes` is the sum of daily rollups and includes registry checks, so it can differ from `total`."}},"latestObservations":[{"at":"2026-10-09T21:26:28.804Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":21,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-09T15:31:45.017Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":56,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-09T08:27:47.630Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":19,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-09T01:24:57.322Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":35,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-08T18:28:03.882Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":30,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-08T12:25:02.318Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":30,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-08T05:21:27.857Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":12,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-07T22:23:47.981Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":21,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-07T15:30:56.538Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":44,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}},{"at":"2026-10-07T09:27:25.958Z","kind":"mcp_initialize","ok":true,"httpStatus":200,"latencyMs":45,"error":null,"detail":{"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"toolCount":7,"toolsHash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","serverName":"crawl-census","capabilities":["tools"],"serverVersion":"1.0.0","protocolVersion":"2025-06-18"}}],"tools":[{"name":"scan_site","description":"Run a live AI-accessibility audit of a domain: robots.txt policy for every tracked AI crawler, live user-agent probes, JavaScript-free readability, structured d"},{"name":"site_report","description":"Return the most recent stored audit for a domain without triggering a new scan. Faster and free of load on the target site."},{"name":"census_stats","description":"Corpus-level statistics: how many measured domains block each AI crawler, mean access score, llms.txt adoption."},{"name":"crawl_preflight","description":"Decide whether a crawler may fetch a list of domains before spending requests on them. Works for any crawler token, not only the ones this census tracks: an unr"},{"name":"agent_profile","description":"What this census measures and publishes about one AI crawler: how often it is disallowed in robots.txt, how often live requests carrying its user agent are refu"},{"name":"census_facts","description":"Every headline finding from the census as discrete, dated records rather than prose. Each carries its value, unit, denominator, measurement date, the page it co"},{"name":"submit_domains","description":"Queue domains the census has not measured yet so a later crawl_preflight can answer them. This closes the loop crawl_preflight starts: anything it returns as un"}],"package":null,"toolSurface":{"id":"ts_mewfk8qtb4kz","endpointId":"ep_znab64cjawn6","hash":"f373442ded4f8e1815343f9c12375d1240b374864cae217599a62f28ad71d190","toolCount":7,"serverName":"crawl-census","serverVersion":"1.0.0","protocolVersion":"2025-06-18","firstSeenAt":"2026-09-12T12:26:33.719Z","lastSeenAt":"2026-10-09T21:26:28.790Z","observations":101,"toolNames":["scan_site","site_report","census_stats","crawl_preflight","agent_profile","census_facts","submit_domains"],"distinctSurfaces":1},"endpointFacts":[{"id":"ep_znab64cjawn6","url":"https://crawlcensus.com/mcp","type":"mcp_streamable_http","factsCheckedAt":"2026-10-08T12:25:02.333Z","auth":{"observedAt":"2026-10-08T12:25:02.355Z","authRequired":false,"scheme":null,"resourceMetadata":null,"authorizationServer":null,"conformance":{"dpop":false,"rfc8414":false,"rfc9728":false,"pkceS256":false,"clientIdMetadataDocument":false,"dynamicClientRegistration":false}},"tls":{"observedAt":"2026-10-08T12:25:02.378Z","protocol":"TLSv1.3","chainValid":true,"chainError":null,"hostMatches":true,"subject":"crawlcensus.com","issuer":{"commonName":"WE1","organization":"Google Trust Services"},"validFrom":"2026-08-22T01:02:13.000Z","validTo":"2026-11-20T02:02:09.000Z","daysToExpiry":41,"sanCount":3,"fingerprint256":"C1:33:A4:B7:30:0D:CF:E3:C2:99:3F:5E:03:B7:7A:31:85:E0:EC:E9:5F:B4:5A:08:69:96:70:0C:A0:D9:0F:79"}}]},"verification":{"claimed":false,"claimedAt":null,"proofs":[]},"provenance":{"sources":[{"source":"mcp_registry","key":"io.github.taylorsmithgg/crawl-census","url":"https://registry.modelcontextprotocol.io/v0/servers/io.github.taylorsmithgg%2Fcrawl-census","firstSeenAt":"2026-09-07T04:21:40.093Z","fetchedAt":"2026-10-08T11:19:31.425Z","normalizedAt":"2026-10-08T11:19:31.425Z"}]},"firstSeenAt":"2026-09-07T04:21:40.093Z","updatedAt":"2026-10-09T21:28:47.337Z"}