# Prevent Crawling Unnecessary Endpoints - Dynamically added by BigScoots
User-agent: *
Disallow: /cdn-cgi/
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*?attachment_id=
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?add_to_wishlist=
Disallow: /*&add_to_wishlist=

User-agent: *

Disallow: /cgi-bin       
Disallow: /?             

Disallow: /wp-admin/
Disallow: /wp-includes/*

Disallow: *?s=              
Disallow: *&s=      
Disallow: *&p=
Disallow: /search/          
Disallow: /author/         
Disallow: /users/           
Disallow: */trackback    

Disallow: */feed           
Disallow: */rss            
Disallow: */embed           

Disallow: /xmlrpc.php       
Disallow: *utm*=          
Disallow: *openstat=      

Allow: /wp-includes/css/
Allow: /wp-includes/js/

Allow: /wp-content/plugins/*.css
Allow: /wp-content/uploads/*.css

Allow: /wp-content/plugins/*.js
Allow: /wp-content/uploads/*.js

Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js

Allow: /*/*.js$
Allow: /*/*.css$
Allow: /*/*.jpg$
Allow: /*/*.png$

Allow: /*/*.js            
Allow: /*/*.css      
Allow: /wp-admin/admin-ajax.php


Crawl-delay: 10
Disallow: /dogs/
Disallow: /cats/

Sitemap: https://pets24.co.za/sitemap.xml
Sitemap: https://pets24.co.za/services/sitemap.xml