# Franke Motorsport - Optimized Robots.txt
# Website: https://franke-motorsport.de
# Last Updated: September 11, 2025

# Default rule for all crawlers
User-agent: *
Allow: /

# Crawl-delay for general bots to prevent overload
Crawl-delay: 1

# ALLOW RULES - Priority Content
# Homepage and main sections
Allow: /
Allow: /#news
Allow: /#streams

# API endpoints for content discovery
Allow: /api/news

# Legal and important pages
Allow: /impressum
Allow: /datenschutz

# Static assets and images
Allow: /franke-motorsport-banner.PNG
Allow: /franke-motorsport-logo.PNG
Allow: /favicon.ico
Allow: /site.webmanifest
Allow: /browserconfig.xml

# DISALLOW RULES - Block unnecessary crawling
# Development and build files
Disallow: /_next/
Disallow: /.next/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.vscode/
Disallow: /.well-known/

# Internal API routes not meant for public crawling
Disallow: /api/cron/
Disallow: /api/redirect

# Config and development files
Disallow: /*.json$
Disallow: /*.config.*
Disallow: /package.json
Disallow: /tsconfig.json
Disallow: /next.config.*
Disallow: /tailwind.config.*
Disallow: /postcss.config.*
Disallow: /eslint.config.*

# SPECIFIC BOT CONFIGURATIONS

# Google Bot - Full access with optimizations
User-agent: Googlebot
Allow: /
Crawl-delay: 0
# Allow Google to access images and videos
Allow: /franke-motorsport-banner.PNG
Allow: /franke-motorsport-logo.PNG

# Google Images Bot
User-agent: Googlebot-Image
Allow: /
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$

# Google Video Bot
User-agent: Googlebot-Video
Allow: /
Allow: /#streams

# Bing Bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo/Verizon Bot
User-agent: Slurp
Allow: /
Crawl-delay: 2

# Social Media Bots - Important for sharing
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

User-agent: Twitterbot
Allow: /
Crawl-delay: 0

User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

User-agent: WhatsApp
Allow: /
Crawl-delay: 0

User-agent: TelegramBot
Allow: /
Crawl-delay: 0

User-agent: DiscordBot
Allow: /
Crawl-delay: 0

# Search Engine Bots
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Crawl-delay: 3

# SEO and Analysis Tools
User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Crawl-delay: 5

# Block problematic bots
User-agent: SemrushBot-SA
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Claude-Web
Disallow: /

# Block AI training bots
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# SITEMAPS
Sitemap: https://franke-motorsport.de/sitemap.xml

# HOST (preferred domain)
Host: franke-motorsport.de

# Contact information for webmaster
# Contact: webmaster@franke-motorsport.de