Robots Lint

{
  "applies_to": {
    "match": "exact",
    "merged_from": 1,
    "rules": 2,
    "user_agent": "foobot"
  },
  "checks": [
    {
      "findings": 0,
      "key": "product_token_charset",
      "rfc": "§2.2.1",
      "status": "ok"
    }
  ],
  "decisions": [
    {
      "decision": "allowed",
      "normalized": "/example/page.html",
      "octets": 18,
      "reason": "most_specific_match",
      "rule": "allow: /example/page.html",
      "rule_type": "allow",
      "tiebreak": null,
      "url": "/example/page.html"
    }
  ],
  "findings": [
    {
      "check": "path_leading_slash",
      "detail": "…",
      "key": "path_starts_with_wildcard",
      "line": 2,
      "rfc": "§2.2 (ABNF) vs §5.1",
      "severity": "warning"
    }
  ],
  "groups": [
    {
      "line": 8,
      "rules": [
        {
          "effect": "matching",
          "line": 9,
          "pattern": "/example/page.html",
          "type": "allow"
        }
      ],
      "user_agents": [
        "foobot"
      ]
    }
  ],
  "input": {
    "bytes": 180,
    "groups": 2,
    "lines": 10,
    "other_records": 1,
    "rules": 5
  },
  "interpretations": [
    {
      "choice": "…",
      "rfc": "§2.2.2",
      "topic": "octet_counting",
      "why": "…"
    }
  ],
  "limits": {
    "bytes": 180,
    "rfc_parsing_limit_bytes": 512000,
    "within_limit": true
  },
  "linted_at": "2026-07-26T09:00:00.000Z",
  "note": "…",
  "other_records": [
    {
      "key": "sitemap",
      "line": 5,
      "standard": false,
      "value": "https://example.de/sitemap.xml"
    }
  ],
  "summary": {
    "errors": 0,
    "infos": 0,
    "warnings": 1
  }
}

# bash or zsh: see the payment challenge. Nothing is charged.
curl -sS -g -i -X POST 'https://api.ozdreamtools.de/api/robots/lint' \
  -H 'content-type: application/json' \
  --data-raw '{"robots_txt":"User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\nSitemap: https://example.de/sitemap.xml\n\nUser-Agent: foobot\nDisallow: /\nAllow: /example/page.html\n","urls":["/example/page.html","/example/other"],"user_agent":"foobot"}'

# x402 v2 sends the challenge base64-encoded in the PAYMENT-REQUIRED header (v1 sends it in the body).
# To read the header:
curl -sS -g -o /dev/null -D - -X POST 'https://api.ozdreamtools.de/api/robots/lint' \
  -H 'content-type: application/json' \
  --data-raw '{"robots_txt":"User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\nSitemap: https://example.de/sitemap.xml\n\nUser-Agent: foobot\nDisallow: /\nAllow: /example/page.html\n","urls":["/example/page.html","/example/other"],"user_agent":"foobot"}' \
  | grep -i '^payment-required:' | cut -d' ' -f2 | tr -d '\r' | base64 --decode

// npm i @x402/fetch @x402/evm viem
import { wrapFetchWithPayment, x402Client } from "@x402/fetch";
import { registerExactEvmScheme } from "@x402/evm/exact/client";
import { privateKeyToAccount } from "viem/accounts";

// Pays at most $0.05 per call.
const client = new x402Client().setSpendControls({ maxAmountPerPayment: "$0.05" });
registerExactEvmScheme(client, { signer: privateKeyToAccount(process.env.EVM_PRIVATE_KEY) });
const pay = wrapFetchWithPayment(fetch, client);

// Not sure it's safe to pay? ToolVet checks it first for $0.005. Pay for that from a separate
// client: this one is capped at $0.05 per payment. GET https://toolvet.app/api/v1/check?url=https%3A%2F%2Fapi.ozdreamtools.de%2Fapi%2Frobots%2Flint&method=POST
const res = await pay("https://api.ozdreamtools.de/api/robots/lint", {
  method: "POST",
  headers: { "content-type": "application/json" },
  body: JSON.stringify({
    "robots_txt": "User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\nSitemap: https://example.de/sitemap.xml\n\nUser-Agent: foobot\nDisallow: /\nAllow: /example/page.html\n",
    "urls": [
      "/example/page.html",
      "/example/other"
    ],
    "user_agent": "foobot"
  }),
});
console.log(res.status, await res.text());

# pip install "x402[requests,evm]"
import os
from eth_account import Account
from x402 import x402ClientSync
from x402.http.clients import x402_requests
from x402.mechanisms.evm import EthAccountSigner
from x402.mechanisms.evm.exact.register import register_exact_evm_client

# Pays at most $0.05 per call.
client = x402ClientSync().set_spend_controls({"max_amount_per_payment": "$0.05"})
register_exact_evm_client(client, EthAccountSigner(Account.from_key(os.environ["EVM_PRIVATE_KEY"])))

with x402_requests(client) as session:
    res = session.post(
        "https://api.ozdreamtools.de/api/robots/lint",
        json={
            "robots_txt": "User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\nSitemap: https://example.de/sitemap.xml\n\nUser-Agent: foobot\nDisallow: /\nAllow: /example/page.html\n",
            "urls": [
                "/example/page.html",
                "/example/other",
            ],
            "user_agent": "foobot",
        },
    )
    print(res.status_code, res.text)