# ---------------------------------------------------------------------
# robots.txt — v5
# ---------------------------------------------------------------------
# Default rules for all crawlers (Googlebot, Bingbot, Claude-SearchBot, ...)
# ---------------------------------------------------------------------
User-agent: *

# Backend, administrative and system paths
Disallow: /admin/
Disallow: /*/admin/
Disallow: /app/
Disallow: /*/app/
Disallow: /bin/
Disallow: /*/bin/
Disallow: /dev/
Disallow: /*/dev/
Disallow: /lib/
Disallow: /*/lib/
Disallow: /phpserver/
Disallow: /*/phpserver/
Disallow: /pub/errors/
Disallow: /*/pub/errors/
Disallow: /setup/
Disallow: /*/setup/
Disallow: /update/
Disallow: /*/update/
Disallow: /var/
Disallow: /*/var/
Disallow: /vendor/
Disallow: /*/vendor/
Disallow: /errors/
Disallow: /*/errors/

# User-specific and internal process pages
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /catalogsearch/
Disallow: /*/catalogsearch/
Disallow: /newsletter/
Disallow: /*/newsletter/
Disallow: /review/
Disallow: /*/review/
Disallow: /wishlist/
Disallow: /*/wishlist/
Disallow: /sendfriend/
Disallow: /*/sendfriend/
Disallow: /downloader/
Disallow: /*/downloader/
Disallow: /includes/
Disallow: /*/includes/
Disallow: /pkginfo/
Disallow: /*/pkginfo/
Disallow: /report/
Disallow: /*/report/
Disallow: /scripts/
Disallow: /*/scripts/
Disallow: /shell/
Disallow: /*/shell/

# Common URL parameters that create duplicate content
Disallow: /*?SID=
Disallow: /*&SID=
Disallow: /*?___store=
Disallow: /*&___store=
Disallow: /*?___from_store=
Disallow: /*&___from_store=
Disallow: /*?gclid=
Disallow: /*&gclid=
Disallow: /*?fbclid=
Disallow: /*&fbclid=
Disallow: /*?utm_
Disallow: /*&utm_

# Magento pagination, sorting, view toolbar (no SEO value, all duplicates)
Disallow: /*?p=
Disallow: /*&p=
Disallow: /*?product_list_order=
Disallow: /*&product_list_order=
Disallow: /*?product_list_dir=
Disallow: /*&product_list_dir=
Disallow: /*?product_list_limit=
Disallow: /*&product_list_limit=
Disallow: /*?product_list_mode=
Disallow: /*&product_list_mode=

# ---------------------------------------------------------------------
# Layered-navigation filters — teamsport.ee (Navision / Business Central
# attribute set). PREFIX match: catches every current AND future bc_* and
# navision_* filter, as first (?) or subsequent (&) parameter. You never
# have to add individual attribute codes here again — this is the fix for
# the recurring filter-URL indexing.
# ---------------------------------------------------------------------
Disallow: /*?bc_
Disallow: /*&bc_
Disallow: /*?navision_
Disallow: /*&navision_
Disallow: /*?mtz=
Disallow: /*&mtz=
Disallow: /*?q=
Disallow: /*&q=

# System files
Disallow: /cron.php
Disallow: /*/cron.php
Disallow: /cron.sh
Disallow: /*/cron.sh
Disallow: /error_log
Disallow: /*/error_log
Disallow: /install.php
Disallow: /*/install.php
Disallow: /LICENSE.html
Disallow: /*/LICENSE.html
Disallow: /LICENSE.txt
Disallow: /*/LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /*/LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /*/STATUS.txt
Disallow: /mage
Disallow: /*/mage

# ---------------------------------------------------------------------
# Googlebot-Image — explicit allow for product/category imagery only.
# ---------------------------------------------------------------------
User-agent: Googlebot-Image
Allow: /media/catalog/product/
Allow: /*/media/catalog/product/
Allow: /media/catalog/category/
Allow: /*/media/catalog/category/
Allow: /media/
Allow: /*/media/

# ---------------------------------------------------------------------
# AI training / scraping bots — disallow entirely.
# NOTE: Claude-SearchBot is intentionally NOT listed here. It obeys the
# User-agent:* rules above (so it stops crawling filter URLs) while still
# indexing real product/category pages — good for visibility. If you want
# it gone completely, add a "User-agent: Claude-SearchBot / Disallow: /"
# block, but then it ignores the * group entirely.
# ---------------------------------------------------------------------
User-agent: Meta-WebIndexer
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# ---------------------------------------------------------------------
# Aggressive SEO crawlers — block to reduce server load.
# ---------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Linguee
Disallow: /

# ---------------------------------------------------------------------
# Sitemaps  — use absolute URLs, one per store view (edit to match yours)
# ---------------------------------------------------------------------
Sitemap: https://www.teamsport.lv/sitemap_lv.xml
Sitemap: https://www.teamsport.lv/sitemap_lv_en.xml