User-agent: *
Disallow: /cgi-bin/
Disallow: /Images/
Disallow: /docs/
^[A-Z]+$
^Alexibot
^almaden
^anarchie
^ASPSeek
^asterias
^Atomz
^atSpider
^attach
^autoemailspider
^BackDoorBot
^BackWeb
^Bandit
^BatchFTP
^Black.Hole
^BlackWidow
^BlowFish
^Bot\ mailto:craftbot@yahoo.com
^BotALot
^BuiltBotTough
^Buddy
^Bullseye
^BunnySlippers
^Cegbfeieh
^CheeseBot
^ChinaClaw
^CherryPicker
^cherry.?picker
^Collector
^combine
"compatible ; MSIE 6.0"
^Copier
^CopyRightCheck
^cosmos
^Crescent
^Custo
^Deweb
^DIIbot
^Digger
^DISCo
^DittoSpyder
^Download\ Demon
^EasyDL/\d\.\d+
^eCatch
^e-collector
^EirGrabber
^EliteSys
^EmailSiphon
^e?mail.?(collector|magnet|reaper|siphon|sweeper|harvest|collect|wolf)
^EroCrawler
^Express\ WebPictures
^ExtractorPro
^EyeNetIE
^fastlwspider
^Fetch
"Fetch API Request"
^FileHound
^Foobot
^FlashGet
^FlickBot
^FrontPage
^GetRight
^Getleft
^GetSmart
^GetWeb!
^Go!Zilla
^GoZilla.*
^go.?zilla
^Go-Ahead-Got-It
^gotit
^Grabber
^GrabNet
^Grafula
^Harvest
^hloader
^HMView
^HTTrack
^httplib
^humanlinks
^ia_archiver
^Ideare+
^IE\ \d\.\d\ Compatible.*Browser$
^imagefetch
^Indy\ Library
^Image\ Stripper
^Image\ Sucker
^InfoNaviRobot
^InterGET
^Internet\ Ninja
^InternetSeer
"^Internet Explore"
^Iria
^Irvine
^JennyBot
^JetCar
^JOC\ Web\ Spider
^JustView
^Kenjin.Spider
^Keyword.Density
^KWebGet
^larbin
^LexiBot
^lftp
^Libby
^likse
^LinkWalker
"LINKS ARoMATIZED"
^LeechFTP
^libWeb/clsHTTP
^LinkextractorPro
^LinkScan/8.1a.Unix
^LinkWalker
^lwp-trivial
^Magnet
^Mag-Net
^Mass\ Downloader
^Mail\ Sweeper
^Mata.Hari
^Memo
^Microsoft\ URL\ Control
^Microsoft.URL
^MIIxpc
^MSIECrawler
^MIDown\ tool
^Mirror
^Mister\ PiX
^Mister.PiX
^moget
^Mozilla.*NEWT
"^Mozilla/4.0$"
"^Mozilla/\?\?$"
^Mozilla/2
^Mozilla/3.Mozilla/2.01
^Mozilla.*NEWT
^MSProxy
^munky
^Navroad
^NearSite
^net.?(ants|mechanic|spider|prospector|carta|vampire|zip)
^NetAnts
^NetSpider
^Net\ Vampire
^NetZIP
^NG
^NICErsPRO
^Ninja
^NPBot
^Nutscrape
^Octopus
^offline
^Offline.Explorer
^Offline\ Explorer
^Offline\ Navigator
^Openfind
^PageGrabber
^Papa\ Foto
^pcBrowser
^pavuk
^PersonaPilot
^Pockey
^ProPowerBot/2.14
^ProWebWalker
^ProWebWalker
^psbot
^PSurf
^Pump
^PushSite
^QuePasaCreep
^QueryN.Metasearch
^RealDownload
^ReGet
^Reaper
^Recorder
^RepoMonkey
^RMA
^Roverbot
^Rsync
^SearchExpress
^Shai
^Siphon
^SiteSnagger
^SmartDownload
^sitecheck.internetseer.com
^SlySearch
^Snake
^snagger
^SpaceBison
^SpankBot
^spanner
^SpiderBot
^Sqworm
^Stripper
^Sucker
^SuperBot
^SuperHTTP
^Surfbot
^SurfWalker
^SurveyBot
^suzuran
^Szukacz/1.4
^tAkeOut
^Teleport\ Pro
^tarspider
^tele(port|soft)
^Teleport*28
^Templeton
^The.Intraformant
^TheNomad
^TightTwatBot
^Titan
^toCrawl/UrlDispatcher
^toCrawl/UrlDispatcher
^True_Robot
^turingos
^TurnitinBot/1.5
^TurnitinBot
^URLy.Warning
^VoidEYE
^VCI
^vayala
^Vacuum
^w3mir
^Web\ Image\ Collector
^Web.Image.Collector
^Web\ Sucker
^WebAuto
^WebCopier
^WebFetch
^Webweasel
^WebAuto
^WebBandit
^[Ww]eb[Bb]andit
^WebCapture
^Webcollector
^WebCopy
^Webdevil
^Webdownloader
^Webdupe
^WebEMailExtrac
^WebEMailExtrac.*
^WebEnhancer
^WebGo\ IS
^Webhook
^WebLeacher
^WebmasterWorldForumBot
^WebMiner
^Webmirror
^Webmole
^WebReaper
^WebSauger
^Website\ eXtractor
^WebStripper
^WebWhacker
^WebZIP
^Wget
^Website.Quester
^Website\ Quester
^Webster
^Webster.Pro
^WebSnake
^Websucker
^webvac
^webwalk
^Whacker
^WWWOFFLE
^WWW-Collector-E
^Widow
^Xaldon\ WebSpider
^Xenu's
^Zeus
^ZyBorg