# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: webcrawler
Disallow:

User-agent: lycra
Disallow: 

User-agent: *
Disallow: /admin
Disallow: /cgi-bin
Disallow: /database
Disallow: /cleanup_folder
Disallow: /tempphotos
Disallow: /images
Disallow: /SEARCH
Disallow: /SFLib