# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism User-agent: * Disallow: /_flv/ Disallow: /_images/ Disallow: /_js/ Disallow: /_manga/ Disallow: /images/ Disallow: /_image_format.php Disallow: /_image_manga.php Disallow: /_menu.php Disallow: /_xmlhttp.php Disallow: /_auteur.php Disallow: /connexion.php Disallow: /image_save.php Disallow: /images.php Disallow: /images_normal.php Disallow: /images_unic.php Disallow: /_jeuxvideo.php Disallow: /_manga.php Disallow: /_news.php Disallow: /_sitemap.php Disallow: /video.html