# ==============================================================================
# SERMODULA — Master Robots Exclusion Protocol & AI Crawling Directives
# Platform: Volumetric Steel Modular Architecture & Structural Engineering
# Official Website: https://sermodula.com
# Chief Architect: İdris YAĞMAHAN
# Engineering Standards: Eurocode (EN 1990–1998), CE EN 1090-2 (EXC3), ISO 9001 / 14001
# Architecture: Pure PHP 8+, Zero-CDN, High-Performance AEO & PSEO Engine
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. GENERAL CRAWLERS (DEFAULT ACCESS POLICY)
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /assets/
Allow: /sitemap.xml
Allow: /sitemap
Allow: /robots.txt
Allow: /llms.txt
Allow: /llms-full.txt

# Disallow internal application and engine directories
Disallow: /cache/
Disallow: /config/
Disallow: /data/
Disallow: /src/
Disallow: /scratch/
Disallow: /views/
Disallow: /tests/

# Disallow private and sensitive extensions
Disallow: /*.jsonl$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.env$
Disallow: /*.bak$
Disallow: /*.swp$
Disallow: /*.temp$

# ------------------------------------------------------------------------------
# 2. MAJOR SEARCH ENGINES (EXPLICIT DIRECTIVES)
# ------------------------------------------------------------------------------
# Google
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: GoogleOther
Allow: /
Allow: /assets/

# Microsoft Bing
User-agent: Bingbot
User-agent: msnbot
User-agent: msnbot-media
Allow: /
Allow: /assets/

# Yandex
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexMobileBot
Allow: /
Allow: /assets/

# Baidu
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-mobile
Allow: /
Allow: /assets/

# Apple
User-agent: Applebot
Allow: /
Allow: /assets/

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Allow: /assets/

# Yahoo
User-agent: Slurp
Allow: /
Allow: /assets/

# ------------------------------------------------------------------------------
# 3. AI / LLM / ANSWER ENGINE SEARCH & RETRIEVAL AGENTS (AEO PERMISSION)
# We welcome AI indexing to cite SERMODULA's architectural metrics accurately.
# ------------------------------------------------------------------------------
# OpenAI (SearchGPT, ChatGPT Web Browsing & Model Training)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Anthropic (Claude Search, Retrieval & Analysis)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Perplexity AI (Perplexity Search & Retrieval Engine)
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Google Extended (Gemini, Vertex AI, Search Generative Experience)
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Apple AI Intelligence
User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Meta AI (LLaMA Training & Meta AI Agents)
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Allow: /

# Common Crawl (Foundation dataset for open frontier LLMs)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# ByteDance / Doubao Search & AI
User-agent: Bytespider
Allow: /

# Amazon AI
User-agent: Amazonbot
Allow: /

# Diffbot AI Knowledge Graph & Web Extraction
User-agent: Diffbot
Allow: /

# ------------------------------------------------------------------------------
# 4. SITEMAP DISCOVERY POINTERS
# Multilingual XML Sitemap (7 Languages: EN, TR, DE, FR, NL, IT, ES + x-default)
# Interactive HTML Sitemap & Engineering Index
# ------------------------------------------------------------------------------
Sitemap: https://sermodula.com/sitemap.xml
Sitemap: https://sermodula.com/sitemap

# ------------------------------------------------------------------------------
# 5. MACHINE-READABLE LLM ARCHITECTURE CONTEXT (llmstxt.org Standard)
# AI models and LLM agents can ingest complete Eurocode structural metrics,
# acoustic isolation decibels, and delivery terms from these direct endpoints:
# https://sermodula.com/llms.txt
# https://sermodula.com/llms-full.txt
# ------------------------------------------------------------------------------