# All search engines
User-agent: *

# Admin section
Disallow: /*rss
Disallow: /auth.php
Disallow: /auth/*
Disallow: /bitrix/*
Disallow: /email/*
Disallow: /include/
Disallow: /merge/
Disallow: /register/*
Disallow: /search/*

# Custom pages
Disallow: /merge/
Disallow: /register/*
Disallow: /search/*
Disallow: /subscription-management.php
Disallow: /prices/vacation.php
Disallow: /my/prices/vacation.php
Disallow: /th/prices/vacation.php
Disallow: /self-hosted/trial-key-request.php
Disallow: /tc/*
Disallow: /hi/*
Disallow: /report/*
Disallow: /prices/space.php*
Disallow: /promo/x2-x3/*
Disallow: /promo/save-big-with-bitrix24/*

# Tmp files and Cache
Disallow: /*.pdf$
Disallow: /*.xls$
Disallow: /*.doc$
Disallow: /*.ppt$
Disallow: /*.txt$
Disallow: /*.docx$
Disallow: /*.xlsx$
Disallow: /*.pptx$
Disallow: /*.csv$

# Global Allow rules
Allow: */?product=
Allow: */?app=
Allow: /bitrix/cache/*
Allow: /bitrix/blocks/*
Allow: /bitrix/components/*
Allow: /bitrix/content_fonts/*
Allow: /bitrix/content_vendors/*
Allow: /bitrix/css/*
Allow: /bitrix/js/*
Allow: /bitrix/images/*
Allow: /bitrix/templates/*
Allow: /bitrix/panel/*
Allow: /local/components/*

# Another search systems
User-agent: Bingbot
User-agent: MSNBot
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: Slurp
Disallow: /*?*
Disallow: /*&*

# Rules for Google
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Video

# Pages with parameters
Allow: /*?
Allow: /*&
Allow: /*PAGEN_*

# Rules for Yandex
User-agent: Yandexbot
# Special parameters for Yandex
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term /
Clean-param: _openstat /
Clean-param: from /
Clean-param: yclid /
Clean-param: gclid /
Clean-param: PAGEN_1&PAGEN_2&PAGEN_3&PAGEN_4 /
Clean-param: sort&filter&view /
Crawl-delay: 1

#AI User-agents
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: Perplexity-bot
User-agent: Bingbot-Copilot
User-agent: OAI-SearchBot
User-agent: Facebookbot
User-agent: YouBot
User-agent: DuckAssistBot



#Disallows for AI Bots
Disallow: /bitrix/*
Disallow: /local/*
Disallow: /bitrix/cache/*
Disallow: /bitrix/blocks/*
Disallow: /bitrix/components/*
Disallow: /bitrix/content_fonts/*
Disallow: /bitrix/content_vendors/*
Disallow: /bitrix/css/*
Disallow: /bitrix/js/*
Disallow: /bitrix/templates/*
Disallow: /bitrix/panel/*
Disallow: /local/components/*
Disallow: /*?*
Disallow: /*&*

#Allows for AI Bots
Allow: */?product=
Allow: */?app=
Allow: /bitrix/images/*


# Sitemap
Sitemap: https://www.bitrix24.es/sitemap.xml