# ByteLog — robots.txt
# AI botlar ve arama motorları için erişim kuralları

# ============================
# Genel arama motorları
# ============================
User-agent: *
Allow: /
Allow: /blog.html
Allow: /post.html
Allow: /about.html
Allow: /contact.html
Allow: /llms.txt
Allow: /sitemap.xml
Allow: /feed.xml
Disallow: /admin/

# ============================
# AI / LLM Botları — Açıkça İzin Ver
# ============================

# OpenAI (ChatGPT)
User-agent: GPTBot
Allow: /
Disallow: /admin/

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /admin/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin/

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /admin/

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /admin/

# Common Crawl (AI eğitim dataset'leri)
User-agent: CCBot
Allow: /
Disallow: /admin/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /admin/

# ============================
# Sitemap & LLM Rehberi
# ============================
Sitemap: https://bytelog.dev/sitemap.xml
LLMs: https://bytelog.dev/llms.txt