# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

# Allow crawling of first pages but block deep pagination requests
User-agent: *
Allow: /
Disallow: /*?page=*
Allow: /*?page=1
Allow: /*?page=2

# Block high page numbers to prevent excessive crawling
Disallow: /*?page=10$
Disallow: /*?page=1?
Disallow: /*?page=2?
Disallow: /*?page=3?
Disallow: /*?page=4?
Disallow: /*?page=5?
Disallow: /*?page=6?
Disallow: /*?page=7?
Disallow: /*?page=8?
Disallow: /*?page=9?

# Crawl delay to prevent overloading the server
Crawl-delay: 10