# Allow Spiders User-agent: Googlebot Allow: / # Disallow unwanted Spiders User-agent: Enfish Tracker Disallow: / User-agent: Namecrawler/1.0 Disallow: / User-agent: Squid/1.1.19 Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: CherryPickerElite/1.0 Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: EmailCollector/1.0 Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf 1.00 Disallow: / User-agent: /0.5 libwww-perl/0.40 Disallow: / User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32) Disallow: / User-agent: sexsearcher Disallow: / # Ignore FrontPage files User-agent: * Disallow: /_borders Disallow: /_derived Disallow: /_fpclass Disallow: /_overlay Disallow: /_private Disallow: /_themes Disallow: /_vti_bin Disallow: /_vti_cnf Disallow: /_vti_log Disallow: /_vti_map Disallow: /_vti_pvt Disallow: /_vti_txt Disallow: /fpdb # Ignore private areas User-agent: * Disallow: /scripts Disallow: /users # Ignore other areas User-agent: * Disallow: /images Disallow: /editWeb Disallow: /clickheat Disallow: /campus/bookStore/backup Disallow: /ektron