User-agent: *
Allow: /

# Sitemap location
Sitemap: https://linkaudit.app/sitemap.xml

# Crawl delay to be respectful to crawlers
Crawl-delay: 1

# AI Search Engine Optimization
# Optimize for ChatGPT, Perplexity, Claude, and other AI crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

# Google SGE and Bard optimization
User-agent: Google-Extended
Allow: /

User-agent: Bard
Allow: /

# Block access to admin areas and API endpoints
Disallow: /admin/
Disallow: /api/
Disallow: /.env
Disallow: /private/
Disallow: /config/

# Allow access to important pages for AI understanding
Allow: /
Allow: /scan
Allow: /features  
Allow: /pricing
Allow: /faq

# Allow common resource files for better crawling
Allow: /favicon.ico
Allow: /og-image.jpg
Allow: /robots.txt
Allow: /sitemap.xml

# Encourage AI engines to understand our content
# Additional metadata for AI crawlers
Host: linkaudit.app