User-agent: *

Allow: /
Disallow: /*.asp
Disallow: /*.php
Disallow: /*.aspx
Disallow: /*.jsp
Disallow: /*?*
Disallow: /*.shtml
Disallow: /about.html
Disallow: /city/*.html
Disallow: /*/index_*.html
Disallow: /common/

User-agent: AhrefsBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DeuSu
Disallow: /
User-agent: grapeshot
Disallow: /