# To specify the User-agent for Google: User-agent: googlebot Disallow: /favicon-mp.ico Disallow: /favicon.ico # To specify the User-agent for all bots, use the "wild card" character: User-agent: * Disallow: /favicon-mp.ico Disallow: /favicon.ico # To disallow access to a specific file in the root directory: Disallow: /private-stuff.html Disallow: /favicon-mp.ico Disallow: /favicon.ico # To disallow access to an entire directory: #Disallow: /images/ isallow: /favicon-mp.ico Disallow: /favicon.ico # To disallow access to the entire site: #Disallow: / Disallow: /favicon-mp.ico Disallow: /favicon.ico # This disallows all bots everywhere: #User-agent: * #Disallow: / disallow: /favicon-mp.ico Disallow: /favicon.ico # This disallows all bots to several directories and files: Disallow: /private-file.html #Disallow: /images/ Disallow: /cgi-bin/ Disallow: /search.php Disallow: /favicon-mp.ico Disallow: /favicon.ico # This disallows access to engine specific pages: User-agent: MJ12bot Disallow: / Crawl-Delay: 300 #Disallow: /index-msn.html #User-agent: msnbot #Disallow: /index-google.html Disallow: /favicon-mp.ico Disallow: /favicon.ico # PARTIAL access (All Spiders) User-agent: * Disallow: /404.shtml Disallow: /IE7-override.css Disallow: /IMB/ Disallow: /_private/ Disallow: /_vti_bin/ Disallow: /_vti_cnf/ Disallow: /_vti_log/ Disallow: /_vti_pvt/ Disallow: User-agent: MJ12bot Disallow: /_vti_txt/ Disallow: /cgi-bin/ Disallow: /contact-form-error.htm Disallow: /contact-form-ok.htm Disallow: /css-tips/switcher-test.htm Disallow: /dev/ Disallow: /favicon-mp.ico Disallow: /favicon.ico Disallow: /images/ Disallow: /include-test.htm Disallow: /index-ie7.htm Disallow: /info.php Disallow: /logs/ Disallow: /misc_files/ Disallow: /php-test/ Disallow: /phpSitemapNG/ Disallow: /phplist/ Disallow: /config.inc.php Disallow: /wwwroot/awstats.www.freeus.co.uk.html Disallow: /awstats.www.freeus.co.uk.html Disallow: /postinfo.html Disallow: /redhat2/ Disallow: /resources/_vti_cnf/ Disallow: /resources/free-directory-listings-ie7.htm Disallow: /resources/htaccess-mod_write.htm Disallow: /resources/include-fdl-credits-notes.htm Disallow: /resources/include-pdl-credits-notes.htm Disallow: /resources/index.bak.htm Disallow: /resources/robots-txt-files-2.htm Disallow: /resources/seo-made-easy/ Disallow: /robots(20051117).txt Disallow: /search/ Disallow: /style-container-test.css Disallow: /style-under-dev.css Disallow: /style.css Disallow: /style.css.bak.css Disallow: /template-resource-page-table.htm Disallow: /template-resource-page.htm Disallow: /template.htm Disallow: /test/ Disallow: /time.php Disallow: /white-mud-media/ RewriteEngine On RewriteCond %{HTTP_USER_AGENT} ^BlackWidow [OR] RewriteCond %{HTTP_USER_AGENT} ^Bot\ mailto:craftbot@yahoo.com [OR] RewriteCond %{HTTP_USER_AGENT} ^ChinaClaw [OR] RewriteCond %{HTTP_USER_AGENT} ^Custo [OR] RewriteCond %{HTTP_USER_AGENT} ^DISCo [OR] RewriteCond %{HTTP_USER_AGENT} ^Download\ Demon [OR] RewriteCond %{HTTP_USER_AGENT} ^eCatch [OR] RewriteCond %{HTTP_USER_AGENT} ^EirGrabber [OR] RewriteCond %{HTTP_USER_AGENT} ^EmailSiphon [OR] RewriteCond %{HTTP_USER_AGENT} ^EmailWolf [OR] RewriteCond %{HTTP_USER_AGENT} ^Express\ WebPictures [OR] RewriteCond %{HTTP_USER_AGENT} ^ExtractorPro [OR] RewriteCond %{HTTP_USER_AGENT} ^EyeNetIE [OR] RewriteCond %{HTTP_USER_AGENT} ^FlashGet [OR] RewriteCond %{HTTP_USER_AGENT} ^GetRight [OR] RewriteCond %{HTTP_USER_AGENT} ^GetWeb! [OR] RewriteCond %{HTTP_USER_AGENT} ^Go!Zilla [OR] RewriteCond %{HTTP_USER_AGENT} ^Go-Ahead-Got-It [OR] RewriteCond %{HTTP_USER_AGENT} ^GrabNet [OR] RewriteCond %{HTTP_USER_AGENT} ^Grafula [OR] RewriteCond %{HTTP_USER_AGENT} ^HMView [OR] RewriteCond %{HTTP_USER_AGENT} HTTrack [NC,OR] RewriteCond %{HTTP_USER_AGENT} ^Image\ Stripper [OR] RewriteCond %{HTTP_USER_AGENT} ^Image\ Sucker [OR] RewriteCond %{HTTP_USER_AGENT} Indy\ Library [NC,OR] RewriteCond %{HTTP_USER_AGENT} ^InterGET [OR] RewriteCond %{HTTP_USER_AGENT} ^Internet\ Ninja [OR] RewriteCond %{HTTP_USER_AGENT} ^JetCar [OR] RewriteCond %{HTTP_USER_AGENT} ^JOC\ Web\ Spider [OR] RewriteCond %{HTTP_USER_AGENT} ^larbin [OR] RewriteCond %{HTTP_USER_AGENT} ^LeechFTP [OR] RewriteCond %{HTTP_USER_AGENT} ^Mass\ Downloader [OR] RewriteCond %{HTTP_USER_AGENT} ^MIDown\ tool [OR] RewriteCond %{HTTP_USER_AGENT} ^Mister\ PiX [OR] RewriteCond %{HTTP_USER_AGENT} ^Navroad [OR] RewriteCond %{HTTP_USER_AGENT} ^NearSite [OR] RewriteCond %{HTTP_USER_AGENT} ^NetAnts [OR] RewriteCond %{HTTP_USER_AGENT} ^NetSpider [OR] RewriteCond %{HTTP_USER_AGENT} ^Net\ Vampire [OR] RewriteCond %{HTTP_USER_AGENT} ^NetZIP [OR] RewriteCond %{HTTP_USER_AGENT} ^Octopus [OR] RewriteCond %{HTTP_USER_AGENT} ^Offline\ Explorer [OR] RewriteCond %{HTTP_USER_AGENT} ^Offline\ Navigator [OR] RewriteCond %{HTTP_USER_AGENT} ^PageGrabber [OR] RewriteCond %{HTTP_USER_AGENT} ^Papa\ Foto [OR] RewriteCond %{HTTP_USER_AGENT} ^pavuk [OR] RewriteCond %{HTTP_USER_AGENT} ^pcBrowser [OR] RewriteCond %{HTTP_USER_AGENT} ^RealDownload [OR] RewriteCond %{HTTP_USER_AGENT} ^ReGet [OR] RewriteCond %{HTTP_USER_AGENT} ^SiteSnagger [OR] RewriteCond %{HTTP_USER_AGENT} ^SmartDownload [OR] RewriteCond %{HTTP_USER_AGENT} ^SuperBot [OR] RewriteCond %{HTTP_USER_AGENT} ^SuperHTTP [OR] RewriteCond %{HTTP_USER_AGENT} ^Surfbot [OR] RewriteCond %{HTTP_USER_AGENT} ^tAkeOut [OR] RewriteCond %{HTTP_USER_AGENT} ^Teleport\ Pro [OR] RewriteCond %{HTTP_USER_AGENT} ^VoidEYE [OR] RewriteCond %{HTTP_USER_AGENT} ^Web\ Image\ Collector [OR] RewriteCond %{HTTP_USER_AGENT} ^Web\ Sucker [OR] RewriteCond %{HTTP_USER_AGENT} ^WebAuto [OR] RewriteCond %{HTTP_USER_AGENT} ^WebCopier [OR] RewriteCond %{HTTP_USER_AGENT} ^WebFetch [OR] RewriteCond %{HTTP_USER_AGENT} ^WebGo\ IS [OR] RewriteCond %{HTTP_USER_AGENT} ^WebLeacher [OR] RewriteCond %{HTTP_USER_AGENT} ^WebReaper [OR] RewriteCond %{HTTP_USER_AGENT} ^WebSauger [OR] RewriteCond %{HTTP_USER_AGENT} ^Website\ eXtractor [OR] RewriteCond %{HTTP_USER_AGENT} ^Website\ Quester [OR] RewriteCond %{HTTP_USER_AGENT} ^WebStripper [OR] RewriteCond %{HTTP_USER_AGENT} ^WebWhacker [OR] RewriteCond %{HTTP_USER_AGENT} ^WebZIP [OR] RewriteCond %{HTTP_USER_AGENT} ^Wget [OR] RewriteCond %{HTTP_USER_AGENT} ^Widow [OR] RewriteCond %{HTTP_USER_AGENT} ^WWWOFFLE [OR] RewriteCond %{HTTP_USER_AGENT} ^Xaldon\ WebSpider [OR] RewriteCond %{HTTP_USER_AGENT} ^Zeus RewriteCond %{HTTP_USER_AGENT} ^MJ12bot/v1\.0\.*.*$ RewriteCond %{HTTP_USER_AGENT} ^MJ12bot/v1\.1\.*.*$ RewriteCond %{HTTP_USER_AGENT} ^MJ12bot/v1\.2\.*.*$ RewriteCond %{HTTP_USER_AGENT} ^MJ12bot/v1\.3\.1.*$ RewriteCond %{HTTP_USER_AGENT} ^MJ12bot/v1\.3\.*.*$ RewriteRule ^.* - [F,L] RewriteRule .* - [F,L]