User-agent: *
Disallow: /cms/
Disallow: /admin/
Disallow: /educators/signup/office-hours/
Disallow: /educators/signup/officehours/
Disallow: /contact/
Disallow: /forgot/
Disallow: /ask/?*user=
Disallow: /ask/?*status=
Disallow: /ask/?next=*
Disallow: /login/?next=*
Disallow: /class/
Disallow: /referral/
Disallow: /payments/plans/
Disallow: /questions/userevent/
Disallow: /search/pretype_books/
Disallow: /search/whiletype_database/
Disallow: */ai-video/
Disallow: /api/
Disallow: /amp/
Disallow: /signup/?next=*
Disallow: /courses/coursevideos/.*

# Allow AI search and agent use
User-agent: OAI-SearchBot
User-agent: ChatGPT-User  
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
Allow: /

# Disallow AI training data collection
User-agent: GPTBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: ClaudeBot
Disallow: /

Sitemap: https://www.numerade.com/sitemap.xml