# Cyprus Info AI - Robots.txt
# This file tells search engine crawlers which pages or files they can or can't request from your site.
# For more information about the robots.txt standard, see: https://www.robotstxt.org/

# Default rules for all crawlers
User-agent: *
Allow: /

# Allow business pages (new slug-only format: /business-slug)
# Note: Individual business pages are now accessible via /{business-slug}
Allow: /*/

# Allow legacy business URLs for backward compatibility
Allow: /businesses/

# Allow category and blog pages
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/

# Allow static pages
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap

# Disallow private and administrative areas
Disallow: /admin/
# Disallow: /api/
Disallow: /business-portal/
Disallow: /profile/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /claim-business
Disallow: /add-business

# Disallow temporary and cache files
Disallow: /tmp/
Disallow: /cache/
Disallow: /*.tmp$
Disallow: /*.cache$

# Allow important static resources for better rendering
Allow: /assets/
Allow: /images/
Allow: /uploads/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.eot$

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Disallow: /admin/
# Disallow: /api/
Disallow: /business-portal/
Disallow: /profile/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /claim-business
Disallow: /add-business
# Allow Google to access all static resources
Allow: /assets/
Allow: /images/
Allow: /uploads/

User-agent: Bingbot
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Disallow: /admin/
# Disallow: /api/
Disallow: /business-portal/
Disallow: /profile/
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /claim-business
Disallow: /add-business
# Allow Bing to access all static resources
Allow: /assets/
Allow: /images/
Allow: /uploads/

# Social media crawlers
User-agent: Twitterbot
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Allow: /images/
Allow: /uploads/

User-agent: facebookexternalhit
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Allow: /images/
Allow: /uploads/

User-agent: LinkedInBot
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Allow: /images/
Allow: /uploads/

# WhatsApp crawler
User-agent: WhatsApp
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Allow: /images/
Allow: /uploads/

# Telegram crawler
User-agent: TelegramBot
Allow: /
Allow: /*/  # Allow business slug URLs
Allow: /businesses/  # Allow legacy business URLs
Allow: /categories/
Allow: /blog-categories/
Allow: /blog/
Allow: /about
Allow: /advertise
Allow: /contact
Allow: /faq-guides
Allow: /career
Allow: /the-ai-powered-marketer-business-adviser
Allow: /interactive-sitemap
Allow: /images/
Allow: /uploads/

# Crawl delay for general bots (be respectful to server resources)
Crawl-delay: 1

# Sitemap locations - helps search engines discover all pages
# Main sitemap index (references all other sitemaps)
Sitemap: https://cyprusinfo.ai/sitemap.xml

# Individual sitemaps (optional - main index already references these)
Sitemap: https://cyprusinfo.ai/sitemap-static.xml
Sitemap: https://cyprusinfo.ai/sitemap-businesses.xml
Sitemap: https://cyprusinfo.ai/sitemap-blog.xml
Sitemap: https://cyprusinfo.ai/sitemap-categories.xml
Sitemap: https://cyprusinfo.ai/sitemap-blog-categories.xml