# Robots Configuration File
#
# Example: Tell "WebSpider" indexing robot where it can't go (without the #)
# User-agent: WebSpider
# Disallow: /  (disallows all files from being "roboted")
#

User-agent: *
Disallow: /discussions/
Disallow: /air/ozone/editor/
Disallow: /_borders/
Disallow: /_derived/
Disallow: /_fpclass/
Disallow: /_overlay/
Disallow: /_private/
Disallow: /_themes/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_map/
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /fpdb/
Disallow: /air/asbestos/
Disallow: /air/branch_compliance/
Disallow: /air/branch_planning/
Disallow: /air/branch_program_support/
Disallow: /air/isteps/
Disallow: /air/ozone/
Disallow: /poa/branch_market_dev/
Disallow: /poa/branch_recycling/
Disallow: /poa/brochures/
Disallow: /poa/calc/
Disallow: /poa/environmentally_speaking/
Disallow: /poa/envy_award/
Disallow: /poa/forums/
Disallow: /poa/gored/
Disallow: /poa/renew_the_brew/
Disallow: /poa/sba/
Disallow: /poa/treasure_arkansas/
Disallow: /poa/watershed_outreach_education/
Disallow: /rst/branch_enforcement/
Disallow: /rst/branch_programs/
Disallow: /rst/branch_technical/
Disallow: /rst/newsletters/
Disallow: /rst/tankstats/
Disallow: /solwaste/
Disallow: /mgtsvs/


# These Suck up system resources big time and cause bad log statistics analysis
User-agent: SlySearch
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: ia_archiver
Disallow: /
#
# Note:  we decided to allow these back in as we found the asp page problem
#        causing them to go nuts on our site.
#User-agent: gsa-crawler (Enterprise; GID-01405; root@ark.org)
#Disallow: /
#
#User-agent: FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov
#Disallow: /

User-Agent: LinkChecker
Allow: /