# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.
# use:      how AI systems may consume the content (immediate, reference, or full).

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

# AI-training crawlers — extras NOT already covered by Cloudflare's
# managed robots.txt block.
#
# Cloudflare's edge prepends a managed block (driven by the
# dcmetromap.com zone's "Manage your robots.txt" + "Block AI
# training bots" settings) that already disallows GPTBot, ClaudeBot,
# CCBot, Google-Extended, Applebot-Extended, Bytespider, Amazonbot,
# CloudflareBrowserRenderingCrawler, and meta-externalagent — plus
# emits the Article 4 EU DSM "Content-Signal" rights reservation
# preamble. Anything in that managed list is intentionally NOT
# duplicated here. The bots below are ones the managed block doesn't
# cover today.
#
# Search-time AI crawlers (OAI-SearchBot, ChatGPT-User, PerplexityBot,
# ClaudeBot-User, DuckAssistBot, etc.) are NOT listed below on purpose
# — they fall through to `User-agent: *` and stay allowed because they
# drive citation traffic back to the site.
#
# Order: user-agent-specific blocks come BEFORE `User-agent: *` so
# legacy/strict parsers honour the most-specific rule first. Modern
# parsers (Google, Bing) match by most-specific UA regardless of file
# position.

User-agent: Anthropic-AI
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Everyone else — search engines, AI search/agentic crawlers, RSS
# readers, social-share unfurl bots, etc. — gets the standard rules
# below.
#
# No `/wp-*` or `/xmlrpc.php` disallows here on purpose. The new site
# isn't WordPress, and crucially the hero-image SEO migration relies on
# a chain that passes THROUGH `/wp-content/uploads/...` on this domain:
#   metromapdc.com/wp-content/.../dc-metro-map.png
#     → 301 (Cloudflare Redirect Rule, path-preserving)
#     → dcmetromap.com/wp-content/.../dc-metro-map.png
#     → 301 (public/_redirects)
#     → dcmetromap.com/images/dc-metro-map.png
# Disallowing `/wp-content/` would stop Googlebot from fetching the
# intermediate hop, which means it never sees the second 301 and the
# image equity doesn't reach the canonical /images/ URL. So leave the
# WP paths crawlable.
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
# Live-map SUB-pages — the ~100 per-station pages plus the trip
# (`{from}-to-{to}`) and tab (`/search/`, `/planner/`, `/settings/`)
# URLs. These are near-duplicate / thin templated pages that read as
# low-value content to a quality reviewer (this tripped AdSense's
# "low value content" review twice). The pattern blocks every
# `/live-map/<slug>/` sub-path while leaving the canonical `/live-map/`
# hub itself fully crawlable + indexable.
Disallow: /live-map/*/

# Why Disallow here (and not only noindex):
# These sub-pages already carry `X-Robots-Tag: noindex` (set in
# functions/_middleware.ts) AND are excluded from the sitemap. Adding
# the Disallow is safe because none of them was ever indexed (all show
# "N/A — not crawled" in Search Console), so the usual "noindex +
# Disallow strands a page as Indexed-though-blocked" anti-pattern does
# not apply — there is nothing in the index to strand. Blocking the
# crawl additionally keeps these thin pages out of AdSense's site
# evaluation. Re-allow them only once each station page carries
# genuinely unique per-station content.

# Note: this file is served only on the production custom domain.
# On the *.pages.dev staging URL, functions/_middleware.ts overrides
# this with a hard "Disallow: /" via the Basic Auth gate.

Sitemap: https://dcmetromap.com/sitemap.xml