# Created by J.G.Smirniotopoulos
# 9:17 AM 1/7/01
# Edited by Bob Williams, 01/11/05, CQUarles, 05/15/07

User-Agent: *
Disallow: /secrets/
Disallow: /cgi-bin/
Disallow: /_macros/
Disallow: /software/
Disallow: /backgrounds/
Disallow: /pipermail/

# To exclude all dtic.mil or dodsearch.org 
# 
#User-agent: DoD-wide crawl
#Disallow: /
#User-agent: dodsearch
#Disallow: /

# Disallow other non-important search engines:
User-agent: gazz
Disallow: /
#User-agent: MSIECrawler
#Disallow: /
#User-agent: MSNBot
#Disallow: /
 


# A detailed description of robots.txt syntax is at
# http://www.kollar.com/robots.html


# Some of the most active User-agents on rad:
# User-agent: Googlebot
# Disallow: /
# User-agent: inctomi
# User-agent: MSNBot
# User-agent: AskJeeves
# User-agent: Fast-Webcrawler
# User-agent: Alexa
# User-agent: WebReaper
# User-agent: MSIECrawler
# User-agent: gazz
# User-agent: 
# User-agent: 
# User-agent: 


# To exclude all robots from the entire server
# 
# User-agent: *
# Disallow: /


# To allow all robots complete access
# 
# User-agent: *
# Disallow:
# Or create an empty "/robots.txt" file.


# To exclude all robots from part of the server
# 
# User-agent: *
# Disallow: /cgi-bin/
# Disallow: /tmp/
# Disallow: /private/

# To exclude a single robot
# 
# User-agent: BadBot
# Disallow: /

# To allow a single robot
# 
# User-agent: WebCrawler
# Disallow:
# 
# User-agent: *
# Disallow: /

# To exclude all files except one
# This is currently a bit awkward, as there is no "Allow" field. 
# The easy way is to put all files to be disallowed into a 
# separate directory, say "docs", and leave the one file 
# in the level above this directory:
# 
# User-agent: *
# Disallow: /~joe/docs/