User-agent: Googlebot
Allow: /

User-agent: GPTBot
Crawl-delay: 10

# Disallow anti-social indexers
User-agent: DOC
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: Ense.it
Disallow: /

User-agent: fast
Disallow: /

User-agent: Fasterfox
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: linko
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: wget
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: Zao
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: ZyBORG
Disallow: /

# No file+directory
User-agent: *
Allow: /applications/core/interface/font/
Disallow: /_script/
Disallow: /api/
Disallow: /applications/
Disallow: /cgi-bin/
Disallow: /datastore/
Disallow: /plugins/
Disallow: /system/
Disallow: /403error.php
Disallow: /404error.php
Disallow: /500error.php
Disallow: /Credits.txt
Disallow: /error.php
Disallow: /upgrading.html

# Also these...
Disallow: /activity/
Disallow: /contact/
Disallow: /cookie/
Disallow: /discover/
Disallow: /discover/unread/
Disallow: /leaderboard/
Disallow: /login/
Disallow: /lostpassword/
Disallow: /markallread/
Disallow: /online/
Disallow: /register/
Disallow: /search/
Disallow: /staff/
Disallow: /startTopic/
Disallow: /tags/

# ... And this!
Disallow: /*/?do=download
Disallow: /*?&controller=embed
Disallow: /*?advancedSearchForm=
Disallow: /*?app=*
Disallow: /*?do=
Disallow: /*?filter=
Disallow: /*?forumId*
Disallow: /*?sortby=
Disallow: /*?tab=
Disallow: /*ref=
Disallow: /*sortby=*
Disallow: /?tab=*
Disallow: /index.php?*
Disallow: /profile/*/?do=*
Disallow: /profile/*/content/

# Sitemap URL
Sitemap: https://forum.buildhub.org.uk/sitemap.php