# ==========================================
# For all robots (default rules)
# ==========================================

User-agent: *

# Block access to specific groups of pages
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

# Block course material URLs for all bots by default.
# Only explicitly allowed bots below may access these paths.
Disallow: /course-material
Disallow: /course-material-listing-page
Disallow: /course-material-caching

#Request-rate: 1/10              # maximum rate is one page every 10 seconds
#Crawl-delay: 10                 # 10 seconds between page requests
#Visit-time: 0400-0845           # only visit between 04:00 and 08:45 UTC

# ==========================================
# Allowed bots: grant access to course material URLs
# while still blocking sensitive paths (cart / checkout / my-account).
# ==========================================

User-agent: Googlebot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: Bingbot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: ClaudeBot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: FacebookExternalHit
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: Amazonbot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: Barkrowler
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: SearchEngine-Crawler
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: YouBot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: SerpstatBot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: AwarioBot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: DotBot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

User-agent: MonsidoBot
Allow: /course-material
Allow: /course-material-listing-page
Allow: /course-material-caching
Disallow: /cart
Disallow: /checkout
Disallow: /my-account

# ==========================================
# Explicitly blocked bots (no access to anything)
# ==========================================

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DuckDuckBot/1.1
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: GeedoProductSearch
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Applebot
Disallow: /

# ==========================================
# Sitemap (global - applies to all crawlers)
# ==========================================

# Allow search crawlers to discover the sitemap
Sitemap: /sitemap.xml