# # User-agent: * Crawl-delay: 5 # Directories Disallow: /bilder/ Disallow: /database/ Disallow: /filter/ Disallow: /includes/ Disallow: /misc/ Disallow: /modules/ Disallow: /sites/ Disallow: /themes/ Disallow: /scripts/ Disallow: /updates/ Disallow: /profiles/ Disallow: /node/ Disallow: /taxanomy/ Disallow: /category/ Disallow: /user/ Disallow: /search/ # Paths (clean URLs) Disallow: /admin/ Disallow: /aggregator/ Disallow: /comment/ Disallow: /contact/ Disallow: /logout/ Disallow: /search/ Disallow: /privatemsg/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ # # Files Disallow: /?q= Disallow: /top-helfer Disallow: /meine-punkte # # block tracker pages Disallow: /tracker/ # block paginated tracker pages Disallow: /tracker? # # # Disallow robots from all but the main feed - requires Pathauto feed aliases turned on. Optionally only block Google from the following paths because it causes the most havoc with Google Disallow: /*/feed # Prevent print-friendly duplicate pages from being crawled Disallow: /book/export/ Disallow: /book/print/ # This is important for the Forward module Disallow: /forward/ # This prevents duplicate content caused by some modules like the Forums and Views modules Disallow: /*?sort* # Prevent duplicate content created by the Image module Disallow: /*size= # If you have the Front module this will keep a duplicate front page from being indexed Disallow: /front_page # If you use the Views module's frontpage this will keep the duplicate from being indexed Disallow: /frontpage Disallow: /node Disallow: /*?*