# shippingunsupervised.com — first-party crawler policy.
#
# Until 2026-09-15 the file at this URL was a Cloudflare zone-managed
# injection, never ours: no robots.txt has ever existed in our source. The
# injection stopped the morning after "Content Independence Day" and the URL
# served our own 404 page while pages.dev still served the header comments of
# the vanished block. The policy below restates the status quo the managed
# block expressed on 2026-09-15 (fetched 07:58 CDT, receipt:
# llmpeek/probes/2026-09-15-flip-day-own-domain.json), in source this time,
# so no CDN feature can delete a promise again.
# Content-Signal values: search=yes, ai-train=no, use=reference.
# Changing any value here is a desk decision, not a deploy.

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Sitemap: where the crawler finds every page we build. Added 2026-09-21 — the
# mailbag answer named this exact gap (the site served no sitemap before, so a
# crawler that reached us got no list of pages to walk). A discovery directive,
# not a policy value: no Allow/Disallow/Content-Signal byte above changed.
Sitemap: https://shippingunsupervised.com/sitemap.xml