# Allow all good bots, then selectively disallow specific crawlers
User-agent: *

# Don't crawl WP core
Disallow: /wp/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /cms-login

# Don't crawl Bedrock app folders
Disallow: /app/plugins/
Disallow: /app/mu-plugins/
Disallow: /app/themes/

# Don't crawl build/config/vendor files
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /vendor/
Disallow: /config/
Disallow: /.env

# Block misc. 'support' files that don't need to be indexed
Disallow: /readme.html
Disallow: /license.txt
Disallow: /cgi-bin/

# Block query-string URLs
Disallow: /*?*

# Optional: crawl-delay for polite bots
Crawl-delay: 10

# Block known heavy SEO/spam crawlers

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Yandex
Disallow: /

# Declare sitemap at the end
Sitemap: https://www.select-technology.co.uk/sitemap.xml