# robots.txt for http://www.mayoclinicmedicaldevices.com/ User-agent: * Disallow: /js/ # don't need to crawl js source code Disallow: /inc/ # just page building stuff Disallow: /includes/ # just page building stuff Disallow: /css2/ # why check this? Disallow: /css/ # why check this?