User-agent: * # match all bots
Disallow: /no_crawl/ # Allow access to all parts except anything in the no_crawl-direcotry