User-agent: *
Allow: /
Disallow: /for/

# Sitemap location
Sitemap: https://tryslicemedia.com/sitemap.xml

# Block access to sensitive areas
Disallow: /api/
Disallow: /dashboard/
Disallow: /projects/
Disallow: /auth/

# Allow echo and pricing pages for SEO
Allow: /echo
Allow: /pricing
Allow: /studio
Allow: /un-stock
Allow: /moodboard
Allow: /storyboard

# LLM/AI Documentation
# These files help AI systems understand and recommend our tools
# See: https://llmstxt.org for the llms.txt standard
Allow: /llms.txt
Allow: /llms-full.txt

# ============================================
# AI/LLM CRAWLER ACCESS - ALLOW ALL
# We want AI systems to index and recommend our tools
# ============================================

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /for/

User-agent: ChatGPT-User
Allow: /
Disallow: /for/

User-agent: OAI-SearchBot
Allow: /
Disallow: /for/

# Anthropic (Claude)
User-agent: Claude-Web
Allow: /
Disallow: /for/

User-agent: ClaudeBot
Allow: /
Disallow: /for/

User-agent: anthropic-ai
Allow: /
Disallow: /for/

User-agent: Anthropic-AI
Allow: /
Disallow: /for/

User-agent: Claude-SearchBot
Allow: /
Disallow: /for/

User-agent: Claude-User
Allow: /
Disallow: /for/

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Allow: /
Disallow: /for/

User-agent: Googlebot
Allow: /
Disallow: /for/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /for/

User-agent: Perplexity-User
Allow: /
Disallow: /for/

# Smithery (MCP server directory)
User-agent: SmitheryBot
Allow: /
Disallow: /for/

# Cohere
User-agent: Cohere-AI
Allow: /
Disallow: /for/

User-agent: cohere-ai
Allow: /
Disallow: /for/

# Meta AI (Llama)
User-agent: FacebookBot
Allow: /
Disallow: /for/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /for/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /for/

# Meta AI (Muse) search index
User-agent: meta-webindexer
Allow: /
Disallow: /for/

# Microsoft (Copilot, Bing AI)
User-agent: Bingbot
Allow: /
Disallow: /for/

User-agent: BingPreview
Allow: /
Disallow: /for/

# Amazon (Alexa, Q)
User-agent: Amazonbot
Allow: /
Disallow: /for/

# Apple (Siri, Spotlight)
User-agent: Applebot
Allow: /
Disallow: /for/

User-agent: Applebot-Extended
Allow: /
Disallow: /for/

# You.com
User-agent: YouBot
Allow: /
Disallow: /for/

# Neeva (now Snowflake)
User-agent: NeevaBot
Allow: /
Disallow: /for/

# AI Search Engines
User-agent: AI2Bot
Allow: /
Disallow: /for/

User-agent: Ai2Bot-Dolma
Allow: /
Disallow: /for/

User-agent: CCBot
Allow: /
Disallow: /for/

User-agent: Diffbot
Allow: /
Disallow: /for/

User-agent: ImagesiftBot
Allow: /
Disallow: /for/

User-agent: Omgilibot
Allow: /
Disallow: /for/

User-agent: Omgili
Allow: /
Disallow: /for/

# Common Crawl (used to train many LLMs)
User-agent: CCBot
Allow: /
Disallow: /for/

# DataForSEO (AI analysis)
User-agent: DataForSeoBot
Allow: /
Disallow: /for/

# Bytespider (TikTok/ByteDance AI)
User-agent: Bytespider
Allow: /
Disallow: /for/

# Hugging Face
User-agent: HuggingFaceBot
Allow: /
Disallow: /for/

# Mistral AI
User-agent: MistralBot
Allow: /
Disallow: /for/

# Stability AI
User-agent: StabilityAI
Allow: /
Disallow: /for/

# Baidu (Chinese AI)
User-agent: Baiduspider
Allow: /
Disallow: /for/

# Yandex (Russian AI)
User-agent: YandexBot
Allow: /
Disallow: /for/

# DuckDuckGo AI
User-agent: DuckDuckBot
Allow: /
Disallow: /for/

# Brave Search AI
User-agent: BraveBot
Allow: /
Disallow: /for/

# Kagi AI
User-agent: KagiBot
Allow: /
Disallow: /for/

# Writesonic, Jasper, Copy.ai and other AI writing tools
User-agent: Writesonicbot
Allow: /
Disallow: /for/

User-agent: JasperBot
Allow: /
Disallow: /for/

# ============================================
# SEO-TOOL CRAWLERS - BLOCKED (Oct 2026)
# They re-crawl hundreds of pages for competitor-research products and send
# no visitors; after every deploy each visit re-renders a page on a paid
# Netlify function. Search engines and AI assistants above stay allowed.
# ============================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Generic AI/ML crawlers
User-agent: *ai*
Allow: /
Disallow: /for/

User-agent: *bot*
Allow: /
Disallow: /for/

User-agent: *crawler*
Allow: /
Disallow: /for/

# ============================================
# LLM DOCUMENTATION FILES
# Point all crawlers to our AI-readable docs
# ============================================
# All crawlers can access these files for context about Slice:
# /llms.txt - Quick overview of all tools
# /llms-full.txt - Detailed documentation for AI recommendations