User-agent: * Disallow: /multicrit.html Disallow: /acteurs.html Disallow: /realisateurs.html Disallow: /distributeurs.html Disallow: /mots.html Disallow: /getfilm.html Disallow: /restricted/ Disallow: /arsenique/ Disallow: /stats/ Disallow: /DATA/zipcache/ Disallow: /DATA/halloffame/ Disallow: /forum/message.html Disallow: /forum/search.html Disallow: /forum/search-db.html Disallow: /forum/restricted/ Disallow: /mailto.html # http://www.findsame.com/robot.html User-agent: DIIbot Disallow: / # www.nameprotect.com User-agent: NPBot Disallow: / # Forum internal indexing (password protected, don't bother) User-agent: htdig Allow: /forum/restricted/ # # For your own convenience, please enable the "robots" page exclusion # protocol in your grabber(s)/spider(s)/... # # First line eases the load on our database, multicriterion searches cost! # Those pages aren't needed to reach any other page of the site. # # Third line should help avoiding lots of 401 errors, only authorized # admins have access to those pages. # ## Fifth line filters out forums; They're made of users' contributions, ## and usually aren't ready for robots' consumption. # # The mailto page is of very little interest for bots ;) # # Enjoy MovieCovers.