# ---------------------------------------------------------------------------
# Deep-path crawler trap
#
# CMS pages whose parent had an empty slug were saved without a leading "/"
# (e.g. "alumni/" instead of "/about-us/alumni/"), so navigation rendered them
# as RELATIVE links. Browsers and crawlers resolve those against the current
# path, appending one segment per hop. The eventdetails/ and news-item/ routes
# match on their first three segments and absorb anything deeper as request
# parameters, so every one of those URLs still returned HTTP 200 with a full
# page render. Crawlers walked the loop and produced ~200k duplicate URLs,
# which grew to roughly 80% of all platform requests.
#
# The generating bug is fixed in code (CmsPage.getNavHref / generateSlug), so
# no new trap URLs are created. These rules stop crawlers re-walking the URLs
# already sitting in their index.
#
# Canonical /eventdetails/<id>/<slug>/ and /news-item/<id>/<slug>/ stay
# crawlable -- the patterns need a FOURTH segment to match, because robots.txt
# "*" spans "/" and only the literal slashes in the pattern constrain depth.
#
# NOTE: robots.txt groups do not inherit. A crawler with its own User-agent
# block ignores "User-agent: *" entirely, so the trap rules must be repeated in
# every group below.
# ---------------------------------------------------------------------------

User-agent: *
# Disallow everything by default
Disallow: /*
Disallow: /setup*
Disallow: /eventdetails/*/*/*/
Disallow: /news-item/*/*/*/
# Allow favicon.ico with wildcard for directory segment
Allow: /public/*/website/favicon.ico

User-agent: Googlebot
Disallow: /public/
Disallow: /setup*
Disallow: /eventdetails/*/*/*/
Disallow: /news-item/*/*/*/
Allow: /public/*/website/favicon.ico
Allow: /
Crawl-delay: 10

User-agent: BingBot
Disallow: /public/
Disallow: /setup*
Disallow: /eventdetails/*/*/*/
Disallow: /news-item/*/*/*/
Allow: /public/*/website/favicon.ico
Allow: /
Crawl-delay: 10

User-agent: PowerMapper
Disallow: /public/
Disallow: /setup*
Disallow: /eventdetails/*/*/*/
Disallow: /news-item/*/*/*/
Allow: /public/*/website/favicon.ico
Allow: /
Crawl-delay: 10

User-agent: SiteGuruCrawler
Disallow: /public/
Disallow: /setup*
Disallow: /eventdetails/*/*/*/
Disallow: /news-item/*/*/*/
Allow: /public/*/website/favicon.ico
Allow: /
Crawl-delay: 10