Crawler Diligence Logs

{
  "ai_crawler_hits": 9,
  "by_crawler": [
    {
      "crawler": "GPTBot",
      "distinct_ips": 3,
      "hits": 3,
      "kind": "training",
      "operator": "OpenAI",
      "spoofed_ips": 1,
      "unverified_ips": 2,
      "verified_ips": 0
    },
    {
      "crawler": "ClaudeBot",
      "distinct_ips": 2,
      "hits": 2,
      "kind": "training",
      "operator": "Anthropic",
      "spoofed_ips": 0,
      "unverified_ips": 2,
      "verified_ips": 0
    },
    {
      "crawler": "PerplexityBot",
      "distinct_ips": 2,
      "hits": 2,
      "kind": "search",
      "operator": "Perplexity",
      "spoofed_ips": 0,
      "unverified_ips": 2,
      "verified_ips": 0
    }
  ],
  "caveats": [
    "Verification is by reverse DNS on the source IP, not Web Bot Auth signatures. Signature verification requires headers present only on a live request; access logs do not record them, so it cannot be done retrospectively.",
    "At most 40 distinct IPs are reverse-resolved per call, and IPv6 addresses are reported as unverified rather than guessed at. \"unverified\" means not checked, not suspicious.",
    "At most 20000 log lines are read per call.",
    "A User-Agent is self-declared. Crawlers that do not identify themselves are invisible to this analysis and are not counted."
  ],
  "lines_parsed": 11,
  "lines_unrecognized": 2,
  "revenue_estimate": {
    "assumed_monthly_crawls": 9,
    "assumed_rate_usd_per_crawl": 0.01,
    "basis": "caller-supplied",
    "high_usd": 0.09,
    "low_usd": 0.02,
    "method": "your rate x observed crawler hits in the submitted log, scaled 25%/60%/100% for partial crawler cooperation. The hit count is measured from your log; the rate is yours. Neither is supplied by this service.",
    "mid_usd": 0.05
  },
  "spoofing_summary": "1 distinct IP(s) claimed an AI crawler User-Agent but their reverse DNS does not belong to that operator. Treat those hits as unverified traffic, not crawler demand."
}

# bash or zsh: see the payment challenge. Nothing is charged.
curl -sS -g -i -X POST 'https://tariffmonkee.com/paid/crawler-diligence-logs' \
  -H 'content-type: application/json' \
  --data-raw '{"log":"20.171.206.70 - - [18/Oct/2026:01:03:10 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n20.171.206.71 - - [18/Oct/2026:01:04:11 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n107.23.91.1 - - [18/Oct/2026:11:30:02 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n160.79.106.10 - - [18/Oct/2026:04:20:05 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n160.79.106.11 - - [18/Oct/2026:04:21:06 +0000] \"GET /faq HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n198.51.100.20 - - [18/Oct/2026:07:11:40 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n198.51.100.21 - - [18/Oct/2026:07:12:41 +0000] \"GET /docs/auth HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n52.70.240.171 - - [18/Oct/2026:09:02:15 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n52.70.240.171 - - [18/Oct/2026:09:03:16 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n203.0.113.10 - - [18/Oct/2026:12:00:33 +0000] \"GET / HTTP/1.1\" 200 5100 \"https://www.google.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"\n203.0.113.11 - - [18/Oct/2026:12:01:33 +0000] \"GET /about HTTP/1.1\" 200 5100 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"","rate_per_crawl":0.01}'

# x402 v2 sends the challenge base64-encoded in the PAYMENT-REQUIRED header (v1 sends it in the body).
# To read the header:
curl -sS -g -o /dev/null -D - -X POST 'https://tariffmonkee.com/paid/crawler-diligence-logs' \
  -H 'content-type: application/json' \
  --data-raw '{"log":"20.171.206.70 - - [18/Oct/2026:01:03:10 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n20.171.206.71 - - [18/Oct/2026:01:04:11 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n107.23.91.1 - - [18/Oct/2026:11:30:02 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n160.79.106.10 - - [18/Oct/2026:04:20:05 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n160.79.106.11 - - [18/Oct/2026:04:21:06 +0000] \"GET /faq HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n198.51.100.20 - - [18/Oct/2026:07:11:40 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n198.51.100.21 - - [18/Oct/2026:07:12:41 +0000] \"GET /docs/auth HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n52.70.240.171 - - [18/Oct/2026:09:02:15 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n52.70.240.171 - - [18/Oct/2026:09:03:16 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n203.0.113.10 - - [18/Oct/2026:12:00:33 +0000] \"GET / HTTP/1.1\" 200 5100 \"https://www.google.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"\n203.0.113.11 - - [18/Oct/2026:12:01:33 +0000] \"GET /about HTTP/1.1\" 200 5100 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"","rate_per_crawl":0.01}' \
  | grep -i '^payment-required:' | cut -d' ' -f2 | tr -d '\r' | base64 --decode

// npm i @x402/fetch @x402/evm viem
import { wrapFetchWithPayment, x402Client } from "@x402/fetch";
import { registerExactEvmScheme } from "@x402/evm/exact/client";
import { privateKeyToAccount } from "viem/accounts";

// Pays at most $0.3 per call.
const client = new x402Client().setSpendControls({ maxAmountPerPayment: "$0.3" });
registerExactEvmScheme(client, { signer: privateKeyToAccount(process.env.EVM_PRIVATE_KEY) });
const pay = wrapFetchWithPayment(fetch, client);

// Not sure it's safe to pay? ToolVet checks it first for $0.005. Pay for that from a separate
// client: this one is capped at $0.3 per payment. GET https://toolvet.app/api/v1/check?url=https%3A%2F%2Ftariffmonkee.com%2Fpaid%2Fcrawler-diligence-logs&method=POST
const res = await pay("https://tariffmonkee.com/paid/crawler-diligence-logs", {
  method: "POST",
  headers: { "content-type": "application/json" },
  body: JSON.stringify({
    "log": "20.171.206.70 - - [18/Oct/2026:01:03:10 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n20.171.206.71 - - [18/Oct/2026:01:04:11 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n107.23.91.1 - - [18/Oct/2026:11:30:02 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n160.79.106.10 - - [18/Oct/2026:04:20:05 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n160.79.106.11 - - [18/Oct/2026:04:21:06 +0000] \"GET /faq HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n198.51.100.20 - - [18/Oct/2026:07:11:40 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n198.51.100.21 - - [18/Oct/2026:07:12:41 +0000] \"GET /docs/auth HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n52.70.240.171 - - [18/Oct/2026:09:02:15 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n52.70.240.171 - - [18/Oct/2026:09:03:16 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n203.0.113.10 - - [18/Oct/2026:12:00:33 +0000] \"GET / HTTP/1.1\" 200 5100 \"https://www.google.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"\n203.0.113.11 - - [18/Oct/2026:12:01:33 +0000] \"GET /about HTTP/1.1\" 200 5100 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"",
    "rate_per_crawl": 0.01
  }),
});
console.log(res.status, await res.text());

# pip install "x402[requests,evm]"
import os
from eth_account import Account
from x402 import x402ClientSync
from x402.http.clients import x402_requests
from x402.mechanisms.evm import EthAccountSigner
from x402.mechanisms.evm.exact.register import register_exact_evm_client

# Pays at most $0.3 per call.
client = x402ClientSync().set_spend_controls({"max_amount_per_payment": "$0.3"})
register_exact_evm_client(client, EthAccountSigner(Account.from_key(os.environ["EVM_PRIVATE_KEY"])))

with x402_requests(client) as session:
    res = session.post(
        "https://tariffmonkee.com/paid/crawler-diligence-logs",
        json={
            "log": "20.171.206.70 - - [18/Oct/2026:01:03:10 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n20.171.206.71 - - [18/Oct/2026:01:04:11 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n107.23.91.1 - - [18/Oct/2026:11:30:02 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n160.79.106.10 - - [18/Oct/2026:04:20:05 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n160.79.106.11 - - [18/Oct/2026:04:21:06 +0000] \"GET /faq HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])\"\n198.51.100.20 - - [18/Oct/2026:07:11:40 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n198.51.100.21 - - [18/Oct/2026:07:12:41 +0000] \"GET /docs/auth HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n52.70.240.171 - - [18/Oct/2026:09:02:15 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n52.70.240.171 - - [18/Oct/2026:09:03:16 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n203.0.113.10 - - [18/Oct/2026:12:00:33 +0000] \"GET / HTTP/1.1\" 200 5100 \"https://www.google.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"\n203.0.113.11 - - [18/Oct/2026:12:01:33 +0000] \"GET /about HTTP/1.1\" 200 5100 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"",
            "rate_per_crawl": 0.01,
        },
    )
    print(res.status_code, res.text)