Robots Txt

{
  "ai_bots": {
    "Amazonbot": "permitido",
    "Applebot-Extended": "bloqueado",
    "Bytespider": "bloqueado",
    "CCBot": "bloqueado",
    "ChatGPT-User": "bloqueado",
    "Claude-SearchBot": "bloqueado",
    "Claude-User": "bloqueado",
    "ClaudeBot": "bloqueado",
    "Diffbot": "bloqueado",
    "DuckAssistBot": "bloqueado",
    "GPTBot": "bloqueado",
    "Google-Extended": "bloqueado",
    "MistralAI-User": "permitido",
    "OAI-SearchBot": "bloqueado",
    "Perplexity-User": "bloqueado",
    "PerplexityBot": "bloqueado",
    "YouBot": "bloqueado",
    "anthropic-ai": "bloqueado",
    "cohere-ai": "bloqueado",
    "meta-externalagent": "bloqueado"
  },
  "blocked_ai_bots": 18,
  "groups": 63,
  "query": {
    "agent": "GPTBot",
    "allowed": false,
    "applied_group": [
      "gptbot"
    ],
    "route": "/section/world"
  },
  "robots_txt": true,
  "site": "https://www.nytimes.com",
  "sitemaps": [
    "https://www.nytimes.com/sitemaps/new/news.xml.gz",
    "https://www.nytimes.com/sitemaps/new/sitemap.xml.gz"
  ]
}

# bash or zsh: see the payment challenge. Nothing is charged.
curl -sS -g -i -X POST 'https://api.vextorium.com/v1/robots-txt' \
  -H 'content-type: application/json' \
  --data-raw '{"agent":"GPTBot","route":"/section/world","url":"https://www.nytimes.com"}'

# x402 v2 sends the challenge base64-encoded in the PAYMENT-REQUIRED header (v1 sends it in the body).
# To read the header:
curl -sS -g -o /dev/null -D - -X POST 'https://api.vextorium.com/v1/robots-txt' \
  -H 'content-type: application/json' \
  --data-raw '{"agent":"GPTBot","route":"/section/world","url":"https://www.nytimes.com"}' \
  | grep -i '^payment-required:' | cut -d' ' -f2 | tr -d '\r' | base64 --decode

// npm i @x402/fetch @x402/evm viem
import { wrapFetchWithPayment, x402Client } from "@x402/fetch";
import { registerExactEvmScheme } from "@x402/evm/exact/client";
import { privateKeyToAccount } from "viem/accounts";

// Pays at most $0.003 per call.
const client = new x402Client().setSpendControls({ maxAmountPerPayment: "$0.003" });
registerExactEvmScheme(client, { signer: privateKeyToAccount(process.env.EVM_PRIVATE_KEY) });
const pay = wrapFetchWithPayment(fetch, client);

// Not sure it's safe to pay? ToolVet checks it first for $0.005. Pay for that from a separate
// client: this one is capped at $0.003 per payment. GET https://toolvet.app/api/v1/check?url=https%3A%2F%2Fapi.vextorium.com%2Fv1%2Frobots-txt&method=POST
const res = await pay("https://api.vextorium.com/v1/robots-txt", {
  method: "POST",
  headers: { "content-type": "application/json" },
  body: JSON.stringify({
    "agent": "GPTBot",
    "route": "/section/world",
    "url": "https://www.nytimes.com"
  }),
});
console.log(res.status, await res.text());

# pip install "x402[requests,evm]"
import os
from eth_account import Account
from x402 import x402ClientSync
from x402.http.clients import x402_requests
from x402.mechanisms.evm import EthAccountSigner
from x402.mechanisms.evm.exact.register import register_exact_evm_client

# Pays at most $0.003 per call.
client = x402ClientSync().set_spend_controls({"max_amount_per_payment": "$0.003"})
register_exact_evm_client(client, EthAccountSigner(Account.from_key(os.environ["EVM_PRIVATE_KEY"])))

with x402_requests(client) as session:
    res = session.post(
        "https://api.vextorium.com/v1/robots-txt",
        json={
            "agent": "GPTBot",
            "route": "/section/world",
            "url": "https://www.nytimes.com",
        },
    )
    print(res.status_code, res.text)