Ai Crawlers

{
  "data": {
    "ai_crawlers": {
      "bot_access": {
        "Amazonbot": "allowed",
        "Applebot-Extended": "allowed",
        "Bytespider": "allowed",
        "CCBot": "allowed",
        "ChatGPT-User": "allowed",
        "Claude-SearchBot": "allowed",
        "Claude-User": "allowed",
        "ClaudeBot": "allowed",
        "GPTBot": "allowed",
        "Google-Extended": "allowed",
        "Meta-ExternalAgent": "allowed",
        "OAI-SearchBot": "allowed",
        "PerplexityBot": "allowed"
      },
      "googlebot": "allowed",
      "llms_txt": "present",
      "robots_txt": "present",
      "sitemaps": []
    },
    "findings": [
      {
        "confidence": "confirmed",
        "evidence": [],
        "fix": "Add 'Sitemap: https://your-domain/sitemap.xml'.",
        "id": "ai.robots.no_sitemap",
        "severity": "low",
        "title": "robots.txt does not list a sitemap"
      },
      {
        "confidence": "confirmed",
        "evidence": [
          "# Agent Harbor"
        ],
        "fix": "",
        "id": "ai.llms_txt.present",
        "severity": "info",
        "title": "llms.txt is present"
      }
    ],
    "notes": []
  },
  "domain": "agentharbor.agency",
  "price_paid_usdc": 0.01,
  "provider": "Agent Harbor — deterministic audit engine",
  "tier": "ai-crawlers"
}

# bash or zsh: see the payment challenge. Nothing is charged.
curl -sS -g -i 'https://x402.agentharbor.agency/ai-crawlers?domain=agentharbor.agency'

# x402 v2 sends the challenge base64-encoded in the PAYMENT-REQUIRED header (v1 sends it in the body).
# To read the header:
curl -sS -g -o /dev/null -D - 'https://x402.agentharbor.agency/ai-crawlers?domain=agentharbor.agency' \
  | grep -i '^payment-required:' | cut -d' ' -f2 | tr -d '\r' | base64 --decode

// npm i @x402/fetch @x402/evm viem
import { wrapFetchWithPayment, x402Client } from "@x402/fetch";
import { registerExactEvmScheme } from "@x402/evm/exact/client";
import { privateKeyToAccount } from "viem/accounts";

// Pays at most $0.01 per call.
const client = new x402Client().setSpendControls({ maxAmountPerPayment: "$0.01" });
registerExactEvmScheme(client, { signer: privateKeyToAccount(process.env.EVM_PRIVATE_KEY) });
const pay = wrapFetchWithPayment(fetch, client);

// Not sure it's safe to pay? ToolVet checks it first for $0.005. Pay for that from a separate
// client: this one is capped at $0.01 per payment. GET https://toolvet.app/api/v1/check?url=https%3A%2F%2Fx402.agentharbor.agency%2Fai-crawlers
const res = await pay("https://x402.agentharbor.agency/ai-crawlers?domain=agentharbor.agency");
console.log(res.status, await res.text());

# pip install "x402[requests,evm]"
import os
from eth_account import Account
from x402 import x402ClientSync
from x402.http.clients import x402_requests
from x402.mechanisms.evm import EthAccountSigner
from x402.mechanisms.evm.exact.register import register_exact_evm_client

# Pays at most $0.01 per call.
client = x402ClientSync().set_spend_controls({"max_amount_per_payment": "$0.01"})
register_exact_evm_client(client, EthAccountSigner(Account.from_key(os.environ["EVM_PRIVATE_KEY"])))

with x402_requests(client) as session:
    res = session.get("https://x402.agentharbor.agency/ai-crawlers?domain=agentharbor.agency")
    print(res.status_code, res.text)