User-agent: crawl
Disallow: /

User-agent: crawler
Disallow: /

#User-agent: gsa-crawler
#Disallow: /

User-agent: experimental crawler
Disallow: /

User-agent: www.discoveryengine.com
Disallow: /

User-agent: disco/Nutch-1.0-dev
Disallow: /

User-agent: *
Disallow: /css/
Disallow: /data/
Disallow: /images/
Disallow: /include/
Disallow: /js/
Disallow: /admin_process
Disallow: /aecontent.php
Disallow: /api_
Disallow: /fb_
Disallow: /lm_
Disallow: /login.php
Disallow: /print_
Disallow: /process_
Disallow: /search_
Disallow: /utility_

Sitemap: sitemap.php