# don't allow spiders to get downloadable scripts
User-agent: *
Disallow: download

# stop some bots

# TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html)
User-agent: TurnitinBot
Disallow: /

# NPBot-1/2.0 (http://www.nameprotect.com/botinfo.html)
User-agent: NPBot
Disallow: /

# sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com)
User-agent: sitecheck
Disallow: /

# Rumours-Agent
User-agent: Rumours-Agent
Disallow: /

# larbin_2.6.2 (larbin2.6.2@unspecified.mail)
User-agent: larbin
Disallow: /

# http://www.almaden.ibm.com/cs/crawler   [wf160]
# http://www.almaden.ibm.com/cs/crawler   [c01]
User-agent: almaden
Disallow: /
User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

# Teleport Pro (http://www.tenmax.com/teleport/pro/home.htm)
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /

# WebCopier
User-agent: WebCopier
Disallow: /

# WebStripper (http://www.webstripper.net/)
User-agent: WebStripper
Disallow: /
User-agent: WebStripper/2
Disallow: /

# MSIECrawler - added 12/02/04
User-agent: MSIECrawler
Disallow: /

# Openbot (http://www.openfind.com.tw/robot.html) - added 12/02/04
User-agent: Openbot
Disallow: /

# WebZIP (http://www.spidersoft.com/webzip/default.asp) - added 08/03/04
User-agent: WebZIP
Disallow: /

# QuepasaCreep - added 28/06/04
User-agent: QuepasaCreep
Disallow: /

# WebReaper (http://www.webreaper.net/) - added 20/07/04
User-agent: WebReaper
Disallow: /

# SuperBot (aka Website Downloader ?) - added 02/08/04
User-agent: SuperBot
Disallow: /

# wget - added 16/08/04
User-agent: Wget
Disallow: /

# Web Downloader - added 29/10/04
User-agent: Web Downloader
Disallow: /

# twiceler - added 18/06/09
User-agent: twiceler
Disallow: /

# prevent spidering of names in team listings - first added 12/06/07
User-agent: *
Disallow: /soccer/2009/teams_micro.cgi
Disallow: /soccer/2009/teams_mini.cgi
Disallow: /soccer/2009/teams_teams_girls.cgi
Disallow: /soccer/2009/teams_juniors.cgi
Disallow: /soccer/2009/teams_ladies.cgi
Disallow: /soccer/2009/teams_seniors.cgi
Disallow: /soccer/2008/teams_girls.cgi
Disallow: /soccer/2008/teams_juniors.cgi
Disallow: /soccer/2008/teams_micro.cgi
Disallow: /soccer/2008/teams_mini.cgi
Disallow: /soccer/2008/teams_ladies.cgi
Disallow: /soccer/2008/teams_seniors.cgi
Disallow: /soccer/2007/teams_girls.cgi
Disallow: /soccer/2007/teams_juniors.cgi
Disallow: /soccer/2007/teams_micro.cgi
Disallow: /soccer/2007/teams_mini.cgi
Disallow: /soccer/2007/teams_seniors.cgi
Disallow: /soccer/2006/teams_girls.cgi
Disallow: /soccer/2006/teams_juniors.cgi
Disallow: /soccer/2006/teams_ladies.cgi
Disallow: /soccer/2006/teams_micro.cgi
Disallow: /soccer/2006/teams_mini.cgi
Disallow: /soccer/2006/teams_seniors.cgi
Disallow: /soccer/2005/teams_juniors.cgi
Disallow: /soccer/2005/teams_micro.cgi
Disallow: /soccer/2005/teams_seniors.cgi
Disallow: /soccer/2004/teams_juniors.cgi
Disallow: /soccer/2004/teams_micro.cgi
Disallow: /soccer/2004/teams_seniors.cgi
Disallow: /soccer/2003/teams_juniors.cgi
Disallow: /soccer/2003/teams_micro.cgi
Disallow: /soccer/2003/teams_seniors.cgi

# prevent spidering of names in score results - first added 19/06/07
User-agent: *
Disallow: /soccer/2009/score_marquee.cgi
Disallow: /soccer/2009/score_girls.cgi
Disallow: /soccer/2009/score_juniors.cgi
Disallow: /soccer/2009/score_ladies.cgi
Disallow: /soccer/2009/score_ladies_cup.cgi
Disallow: /soccer/2009/score_seniors.cgi
Disallow: /soccer/2009/score_seniors_cup.cgi
Disallow: /soccer/2008/score_girls.cgi
Disallow: /soccer/2008/score_juniors.cgi
Disallow: /soccer/2008/score_ladies.cgi
Disallow: /soccer/2008/score_seniors.cgi
Disallow: /soccer/2007/score_girls.cgi
Disallow: /soccer/2007/score_juniors.cgi
Disallow: /soccer/2007/score_seniors.cgi
Disallow: /soccer/2006/score_girls.cgi
Disallow: /soccer/2006/score_juniors.cgi
Disallow: /soccer/2006/score_ladies.cgi
Disallow: /soccer/2006/score_seniors.cgi

Disallow: /soccer/2004/committee.cgi
Disallow: /soccer/2005/committee.cgi
Disallow: /soccer/2006/committee.cgi
Disallow: /soccer/2007/committee.cgi
Disallow: /soccer/2008/committee.cgi
Disallow: /soccer/2008/FRSL2008_All_Divisions_Teams.pdf
Disallow: /soccer/2009/committee.cgi

# prevent spidering of special projects stuff for Southern River - added 10/02/08
User-agent: *
Disallow: /projects/