# Optimized robots.txt for vBulletin forum – minimize deep forum crawling
# Last optimized: March 2026
# Goal: allow homepage + forum index, block most thread/post/search/pagination waste

User-agent: *
Allow: /$
Disallow: /forumdisplay.php           # or /forums/ – adjust to your actual category listing URL
#Allow: /forum.php*                 # many vB installs use this for forum views
#Allow: /index.php$
Allow: /announcement.php           # optional – if you want announcements indexed
Allow: /calendar.php               # optional – if calendar has value

# ──────────────────────────────────────────────────────────────
# Block deep forum content – biggest waste on vBulletin sites
# ──────────────────────────────────────────────────────────────

Disallow: /showthread.php          # individual threads – biggest source of duplicate/low-value pages
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /editpost.php
Disallow: /private.php             # private messages
Disallow: /member.php              # member profiles
Disallow: /search.php              # internal search – huge crawl waste
Disallow: /login.php
Disallow: /register.php
Disallow: /faq.php
Disallow: /misc.php
Disallow: /printthread.php
Disallow: /sendmessage.php

# Pagination – very important to block deep pages
Disallow: /*page=*
Disallow: /*/page/
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?sort=*
Disallow: /*?order=*
Disallow: /*?pp=*
Disallow: /*?daysprune=*

# Other common vBulletin waste patterns
Disallow: /*?s=                     # session IDs
Disallow: /*sessionhash*
Disallow: /*styleid=*
Disallow: /*do=                     # many action=do= URLs
Disallow: /archive/                 # if using archive mode
Disallow: /tags/                    # tag cloud / tag pages
Disallow: /attachment.php

# Optional – aggressive crawl slowdown (supported by Google/Yandex/etc.)
Crawl-delay: 10                     # 10 seconds delay – reduces bot aggression

# Optional – block known heavy/bad bots separately (2025–2026 reality)
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Optional – point to your sitemap (helps good bots focus)
#Sitemap: https://example.com/sitemap_index.xml