# Mudpot South Indian Restaurant - Robots.txt
# Website: https://www.mudpotrestaurant.ca/
# Last Updated: 2025-01-10

# Main directives for all web crawlers
User-agent: *
Allow: /
Allow: /index.html
Allow: /menu.html
Allow: /about.html
Allow: /our-food.html
Allow: /contact-us.html
Allow: /catering.html
Allow: /mudpot-south-indian-restaurant-edmonton.html
Allow: /tags/
Disallow: /components/
Disallow: /assets/icons/
Disallow: /assets/favicon/
Disallow: /*.js$
Disallow: /*.css$
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html
Crawl-delay: 1

# Google Bot - Full access
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
Allow: /
Disallow: /components/
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# AI Bots - Allow access to content pages
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: Perplexitybot
Allow: /
Allow: /llms.txt
Disallow: /components/
Disallow: /assets/icons/
Disallow: /assets/favicon/
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Google Extended (Bard/Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /components/
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Social Media Bots
User-agent: FacebookBot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TwitterBot
Allow: /
Allow: /assets/logo/
Allow: /assets/banner/
Allow: /assets/backgrounds/
Disallow: /components/

# Bing Bot
User-agent: Bingbot
User-agent: BingPreview
Allow: /
Disallow: /components/
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Google Ads Bot - Full access for advertising
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
Allow: /

# SEO Analysis Bots
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
Crawl-delay: 10
Allow: /
Disallow: /components/

# Slow down aggressive bots
User-agent: Baiduspider
User-agent: YandexBot
User-agent: SeznamBot
Crawl-delay: 20
Allow: /
Disallow: /components/

# Block malicious/unwanted bots
User-agent: MegaIndex
User-agent: BLEXBot
User-agent: PetalBot
Disallow: /

# Sitemap location
Sitemap: https://www.mudpotrestaurant.ca/sitemap.xml

# LLMs.txt location for AI agents
# AI agents should reference: https://www.mudpotrestaurant.ca/llms.txt