# Best Practice robots.txt for WordPress
User-agent: *

# Block sensitive areas and admin pages
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/private/

# Block 404 pages from crawling
Disallow: /404-custom-page
Disallow: /404/

# Prevent indexing of search results
Disallow: /?s=
Disallow: /search/

# Sitemap for better indexing
Sitemap: https://rainforestigatpuri.com/sitemap_index.xml

# Optional: Crawl-delay for search engines
Crawl-delay: 10