# robots.txt for 247Rep
# This file controls which pages search engines can crawl and index

# ==================== GLOBAL RULES ====================
# Allow all search engines by default
User-agent: *
Allow: /

# ==================== DISALLOW AUTHENTICATED AREAS ====================
# Prevent indexing of user dashboards and admin areas
Disallow: /dashboard/
Disallow: /onboarding/
Disallow: /api/

# ==================== DISALLOW AUTH PAGES ====================
# Authentication pages should not appear in search results
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password

# ==================== DISALLOW UTILITY PAGES ====================
# These pages exist but shouldn't be indexed
Disallow: /widget/
Disallow: /search-widget/

# ==================== ALLOW IMPORTANT PAGES ====================
# Explicitly allow important pages (best practice)
Allow: /
Allow: /pricing
Allow: /features
Allow: /how-it-works
Allow: /about
Allow: /contact
Allow: /blog
Allow: /docs
Allow: /integrations

# Allow tool pages (important for SEO traffic)
Allow: /whatsapp-link-generator
Allow: /whatsapp-widget-for-website
Allow: /bio-generator-for-instagram
Allow: /whatsapp-marketing-automation
Allow: /qr-code-for-whatsapp-group

# Other public resources
Allow: /cookies
Allow: /help

# ==================== SITEMAPS ====================
# Primary sitemap location
Sitemap: https://www.247rep.app/sitemap.xml

# ==================== CRAWL DELAY ====================
# No crawl delay for major search engines (they self-regulate)
# Uncomment below if you experience server load issues
# Crawl-delay: 1

# ==================== SPECIFIC BOT RULES ====================
# Google bot - allow everything not explicitly disallowed above
User-agent: Googlebot
Allow: /

# Bing bot - allow everything not explicitly disallowed above
User-agent: Bingbot
Allow: /

# ==================== AI/LLM CRAWLERS ====================
# Explicitly allow AI/LLM crawlers for training and indexing
# This makes our content available to AI assistants and search tools

# OpenAI GPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /

# Anthropic Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# ==================== BLOCK AGGRESSIVE SCRAPERS ====================
# Block SEO tools and aggressive scrapers that don't respect rate limits
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /