# =====================
# General User-Agent Rules
# =====================
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /wp-json/
Disallow: /trackback/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /*?outlook-ical=
Disallow: /*tribe-bar-date=
#Disallow: /resources/tag/
Disallow: /events/list/page/

# 🔒 Avoid low-value or duplicate URLs
Disallow: /?attachment_id=*
Disallow: /*?utm_         # block campaign-tagged duplicates
Disallow: /*?fbclid=      # block Facebook click ID duplicates

# 🔍 Optional: Block internal search results (if not useful for indexing)
#Disallow: /?s=
#Disallow: /*?s=

# 📰 Optional: Block RSS feeds if you don’t want them indexed
#Disallow: /feed/
#Disallow: /*/feed/
Disallow: /llms.txt

# 🗂 Allow useful assets and directories
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# 🔎 Allow blog category/tags/search if you want them indexed
Allow: /search/
Allow: /tag/
Allow: /category/
Allow: /blog/

# 🗺 Sitemap
Sitemap: https://flexisourceit.com.au/sitemap_index.xml

# =====================
# Specific Bot Overrides (e.g. SEO tools)
# =====================
User-agent: AhrefsSiteAudit
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

# =====================
# Optional: Block crawl delay on aggressive bots (use with caution)
# =====================
# User-agent: MJ12bot
# Crawl-delay: 10

# =====================
# Optional: Block staging or test environments (if public)
# =====================
# Disallow: /staging/
# Disallow: /beta/