# bots that ignore robots.txt

User-agent: *

Disallow: /here-be-gremlins

# advertising-related bots:

User-agent: Mediapartners-Google*

Disallow: /


# Wikipedia work bots:

User-agent: IsraBot

Disallow: /


User-agent: Orthogaffe

Disallow: /


# Crawlers that are kind enough to obey, but which we'd rather not have

# unless they're feeding search engines.

User-agent: UbiCrawler

Disallow: /


User-agent: DOC

Disallow: /


User-agent: Zao

Disallow: /


# Some bots are known to be trouble, particularly those designed to copy

# entire sites. Please obey robots.txt.

User-agent: sitecheck.internetseer.com

Disallow: /


User-agent: Zealbot

Disallow: /


User-agent: MSIECrawler

Disallow: /


User-agent: SiteSnagger

Disallow: /


User-agent: WebStripper

Disallow: /


User-agent: WebCopier

Disallow: /


User-agent: Fetch

Disallow: /


User-agent: Offline Explorer

Disallow: /


User-agent: Teleport

Disallow: /


User-agent: TeleportPro

Disallow: /


User-agent: WebZIP

Disallow: /


User-agent: linko

Disallow: /


User-agent: HTTrack

Disallow: /


User-agent: Microsoft.URL.Control

Disallow: /


User-agent: Xenu

Disallow: /


User-agent: larbin

Disallow: /


User-agent: libwww

Disallow: /


User-agent: ZyBORG

Disallow: /


User-agent: Download Ninja

Disallow: /


# Sorry, wget in its recursive mode is a frequent problem.

# Please read the man page and use it properly; there is a

# --wait option you can use to set the delay between hits,

# for instance.

#

User-agent: wget

Disallow: /


#

# The 'grub' distributed client has been *very* poorly behaved.

#

User-agent: grub-client

Disallow: /


#

# Doesn't follow robots.txt anyway, but...

#

User-agent: k2spider

Disallow: /


# Hits many times per second, not acceptable

# http://www.nameprotect.com/botinfo.html

User-agent: NPBot

Disallow: /


# A capture bot, downloads gazillions of pages with no public benefit

# http://www.webreaper.net/

User-agent: WebReaper

Disallow: /


### Allow internet archiver bot
User-agent: ia_archiver

Allow: /backups/	#Backups
Allow: /w/images/	#Images - for better archiving of pages
Allow: /w/		#Allow archiving of source code?


User-agent: *

Disallow: /secure/

Disallow: /backups/

Disallow: /piwik/

Disallow: /anon/

Disallow: /memcachedadmin/

Disallow: /Lux/

Disaalow: /ISGP/

Disallow: /ISGP-old/

Disaalow: /linux-dash/

Disallow: /thanks/

Disallow: /wikileaks/

Disallow: /wiki/Special:Search

Disallow: /wiki/Special:Random

Disallow: /MANUAL/

Disallow: /backup/

Disallow: /indices-1.1/

Disallow: /uloads/

disallow: /wiki/images/deleted/

disallow: /wiki/cache/


User-agent: googlebot

Disallow: /secure/

Disallow: /piwik/

Disallow: /anon/

Disallow: /w/

Disallow: /wiki/Special:Search

Disallow: /wiki/Special:Random

Disallow: /MANUAL/

Disallow: /backup/

Disallow: /indices-1.1/

Disallow: /uloads/

Disallow: /wiki/Special:Ask/

Disallow: /wiki/Special:Browse/

Disallow: /wiki/Special:SearchByProperty/

Disallow: /wiki/Special:ExportRDF/

Disallow: /wiki/Special:PageProperty/

Disallow: /wiki/Special:Properties/

Disallow: /wiki/Special:UnusedProperties/

Disallow: /wiki/Special:WantedProperties/

Disallow: /wiki/Special:SMWAdmin/

Disallow: /wiki/Special:Types/

Disallow: /wiki/Special:URIResolver/

Disallow: /wiki/Special:QueryCreator/