# =============================================================================
# Robots.txt for SEOEngine.ai
# Optimized for Google Search, favicon visibility & Next.js
# Last Updated: January 2026
# =============================================================================


# -----------------------------------------------------------------------------
# UNIVERSAL RULES
# -----------------------------------------------------------------------------
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/
Disallow: /cdn-cgi/

# Do NOT block Next.js assets (important for favicon & rendering)
Allow: /_next/static/
Allow: /_next/image/

# Allow essential assets
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$
Allow: /*.woff$
Allow: /*.woff2$

# Explicit favicon access (critical for Google)
Allow: /favicon.ico
Allow: /favicon.png
Allow: /favicon.svg

# Content
Allow: /blog/
Allow: /blog/*

Crawl-delay: 1


# -----------------------------------------------------------------------------
# GOOGLE
# -----------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-News
Allow: /blog/


# -----------------------------------------------------------------------------
# OTHER SEARCH ENGINES
# -----------------------------------------------------------------------------
User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: Brave-Search-Bot
Allow: /


# -----------------------------------------------------------------------------
# AI / LLM BOTS
# -----------------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /


# -----------------------------------------------------------------------------
# SEO TOOLS
# -----------------------------------------------------------------------------
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Allow: /
Crawl-delay: 2

User-agent: rogerbot
Allow: /


# -----------------------------------------------------------------------------
# SOCIAL MEDIA PREVIEW BOTS
# -----------------------------------------------------------------------------
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /


# -----------------------------------------------------------------------------
# BAD BOTS (BLOCK)
# -----------------------------------------------------------------------------
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: ChinaClaw
Disallow: /

User-agent: Grabber
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: Wget
Disallow: /


# -----------------------------------------------------------------------------
# CANONICAL HOST (for Yandex and other search engines)
# -----------------------------------------------------------------------------
Host: https://seoengine.ai


# -----------------------------------------------------------------------------
# CLEAN PARAMS (strip tracking parameters from URLs)
# -----------------------------------------------------------------------------
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&ref&fbclid&gclid&msclkid


# -----------------------------------------------------------------------------
# SITEMAPS & FEEDS
# -----------------------------------------------------------------------------
Sitemap: https://seoengine.ai/sitemap.xml
Sitemap: https://seoengine.ai/sitemap-posts.xml

# RSS Feed
# https://seoengine.ai/feed.xml
# https://seoengine.ai/rss.xml

# llms
Allow: /llms/
Allow: /llms/*
# http://seoengine.ai/llms.txt
# =============================================================================
# END
# =============================================================================