# Robots.txt file # # All robots will spider the domain User-agent: * # use Disallow: /xxx/ for each directory Disallow: /admin/ Disallow: /css/ Disallow: /cpanel/ Disallow: /images/ Disallow: /pdf/ Disallow: /Templates/ Disallow: /test/ Disallow: /wusage/