User-agent: * Disallow: /bookmarks/ Disallow: /cgi-bin/ Disallow: /icons/ Disallow: /logs/ Disallow: /manual/ Disallow: /private/ Disallow: /security/ Disallow: /~deadpuppy/ Disallow: /~robjen/family-tree/GENDEX Disallow: /~robjen/sa/ Disallow: /webalizer/ # Eliminate FAST Crawler engines User-agent: fastcrawler # Eliminate grub.org searches User-agent: grub-client # Eliminate picsearch.com crawlers User-agent: psbot # Eliminate Inktomi and Yahoo and that new thing User-agent: Slurp # Eliminate Zao User-agent: Zao # Eliminate szukacz.pl User-agent: szukacz # Eliminate MSNbot User-agent: msnbot # Eliminate BecomeBOT User-agent: BecomeBot Disallow: / # Note that robots, crawlers and spiders that read and ignore robots.txt # will denied http access to the server.