# ========================================
# robots.txt for thinktank-owl.de
# TYPO3 optimized crawler control
# ========================================

# -----------------------
# AI / LLM Crawlers
# -----------------------

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

# -----------------------
# SEO Crawlers
# -----------------------

User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: ScreamingFrogSEO
Disallow: /

User-agent: Sitebulb
Disallow: /

# -----------------------
# Security / Scanner Bots
# -----------------------

User-agent: AliyunSecBot
Disallow: /

User-agent: PetalBot
Disallow: /

# -----------------------
# Default rules
# -----------------------

User-agent: *

# TYPO3 backend / system directories
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /typo3_src/
Disallow: /fileadmin/_processed_/

# TYPO3 parameter crawling prevention
Disallow: /*tx_news
Disallow: /*tx_news_pi1
Disallow: /*overwriteDemand
Disallow: /*cHash
Disallow: /*?type=

# TYPO3 typical duplicate / print URLs
Disallow: /index.php
Disallow: /*print=
Disallow: /*&print=

# -----------------------
# Allow important assets
# -----------------------

Allow: /build/
Allow: /fileadmin/

# -----------------------
# Sitemap
# -----------------------

Sitemap: https://www.thinktank-owl.de/sitemap.xml