# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism User-agent: * Disallow: /_data Disallow: /_config_save Disallow: /_dbm Disallow: /_old_root Disallow: /_removed Disallow: /admin Disallow: /attach Disallow: /eol Disallow: /featured Disallow: /images Disallow: /includes Disallow: /library Disallow: /logs Disallow: /loginsys Disallow: /pdf Disallow: /QB_Admin Disallow: /styles Disallow: /tmp Disallow: /temp Disallow: /templates Disallow: /updated Disallow: /yui