# Robots.txt Optimized for SEO and Security

User-agent: *
# WordPress Core Protection
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /xmlrpc.php

# Security and Performance Optimizations
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /trackback/
Disallow: /comments/

# Limit Indexing of Low-Value Pages
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=

# Additional Security Blocks
Disallow: /author/
Disallow: /feed/
Disallow: /*.php$
Disallow: /readme.html
Disallow: /license.txt

# Block Potentially Harmful Bots
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

# Keep AI Crawlers Open
User-agent: CCBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Explicitly Allow Beneficial Bots
User-agent: GoogleBot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: AhrefsBot
Allow: /

# Crawl Management
Crawl-delay: 10

# Asset Allowances
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js

# Sitemap
Sitemap: https://www.telesign.com/sitemap_index.xml