# /robots.txt file for http://www.websailor.org/

User-agent: exabot
User-agent: exabot-Images
User-agent: twiceler
User-agent: Yandex
User-agent: Yandex Bot
User-agent: YandexBot
User-agent: YandexImages
disallow: /
User-agent: *
Crawl-delay: 20
Request-rate: 1/20         # maximum rate is one page every 5 seconds
Visit-time: 0600-0845     # only visit between 06:00 and 08:45 UTC (GMT)
Disallow: /tmp/
Disallow: /logs/
Disallow: /www/html/i/
Disallow: /www/cgi-bin/
Disallow: /www/error/
Disallow: /www/usage/
Disallow: /www/html/i/zimg/small/
Disallow: /www/html/i/zimg/anigifs/
Disallow: /www/html/i/zimg/emoticons/
Disallow: /www/html/i/zimg/thumbs/
Disallow: /i/
Disallow: /i/zimg/small/
Disallow: /i/zimg/anigifs/
Disallow: /i/zimg/emoticons/
Disallow: /i/zimg/thumbs/
Disallow: /gallery/
Sitemap: <a target="_new" rel="nofollow" href="http://www.websailor.org/web_sitemap_fec59b09.xml.gz">http://www.websailor.org/web_sitemap_fec59b09.xml.gz</a> 
http://www.websailor.org/web_sitemap_fec59b09.xml.gz # Added by Google Sitemap Generator