Keep an AI crawler off your bill

// Makes the text of a Cloudflare custom rule that blocks AI crawlers by name, and refuses a name that you must not block.
// Usage: node rule.mjs <crawler name> [<crawler name> ...]        for example: node rule.mjs meta-externalagent
// Prints the text for "Edit expression" (exit 0). A refused name prints nothing on stdout, says why on stderr and exits with 2.
// A name matches when the user agent CONTAINS it, in any case: so a name must not be a word that a browser also sends.

/** Names that stay allowed, and why. A name is refused when it contains one of these or one of these contains it. */
const LEAVE = {
  facebookexternalhit: "it makes the picture of a link that someone shares on Facebook or Instagram",
  googlebot: "it is a search engine",
  bingbot: "it is a search engine",
  duckduckbot: "it is a search engine",
  applebot: "it is a search engine",
  "oai-searchbot": "it lets ChatGPT search find your site",
  "chatgpt-user": "it reads a page when a person asks ChatGPT about it",
  perplexitybot: "it lets Perplexity find your site",
  "perplexity-user": "it reads a page when a person asks Perplexity about it",
  "claude-searchbot": "it lets Claude search find your site",
  "claude-user": "it reads a page when a person asks Claude about it",
};
/** Words that browsers send in their user agent: a rule on one of these would block real visitors. */
const BROWSER_WORDS = ["mozilla", "chrome", "safari", "firefox", "edge", "gecko", "khtml", "applewebkit", "windows", "macintosh", "android", "iphone", "linux", "compatible", "mobile"];

const names = process.argv.slice(2).map((n) => n.trim());
const refuse = (why) => {
  console.error(why);
  process.exit(2);
};
if (!names.length) refuse("give the crawler's name, for example: node rule.mjs meta-externalagent");
for (const name of names) {
  if (!/^[A-Za-z][A-Za-z0-9._-]{3,}$/.test(name)) refuse(`"${name}" is not a crawler name: letters, digits, dots and dashes, at least 4 characters, no spaces and no quotes.`);
  const low = name.toLowerCase();
  const word = BROWSER_WORDS.find((w) => low.includes(w));
  if (word) refuse(`"${name}" holds "${word}", a word that browsers send. A rule on it would block real visitors.`);
  for (const [leave, why] of Object.entries(LEAVE)) {
    if (low.includes(leave) || leave.includes(low)) refuse(`"${name}" stays allowed: ${why}. Never block it.`);
  }
}
console.log(names.map((n) => `(http.user_agent contains "${n}")`).join(" or "));

// For a Worker on a workers.dev address (no zone, so no Cloudflare rule): answers 403 to a request whose user agent contains the name of
// a crawler, before your code reads the database. Use the same names that rule.mjs accepts: it refuses the ones that must stay allowed.
//
// In your Worker:
//   import { guard } from "./guard.mjs";
//   export default { async fetch(request, env) { const no = guard(request, ["meta-externalagent"]); if (no) return no; /* the rest */ } };
//
// A check by hand:  node guard.mjs "<user agent>" <name> [<name> ...]      prints 403 or 200
import { pathToFileURL } from "node:url";

/** A 403 Response when the request's user agent contains one of `names` (in any case), else null. */
export function guard(request, names) {
  const agent = (request.headers.get("user-agent") ?? "").toLowerCase();
  return names.some((n) => n && agent.includes(n.toLowerCase())) ? new Response("Forbidden", { status: 403 }) : null;
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const [agent, ...names] = process.argv.slice(2);
  const no = guard(new Request("https://example.test/", { headers: { "user-agent": agent ?? "" } }), names);
  console.log(no ? no.status : 200);
}