User-agent: *
Disallow: /search.aspx
Disallow: /advanced-search.aspx
Disallow: /login.aspx
Disallow: /plate.aspx
Disallow: /date.aspx
Disallow: /image.aspx
Disallow: /image-pageScan.aspx
Disallow: /secure_members/
Disallow: /privacy.aspx
Disallow: /analysis.aspx
Allow: /questions/latest.aspx
Allow: /questions/archived.aspx
Crawl-delay: 2 

User-Agent: MSIECrawler
Disallow: /

User-agent: WebStripper
Disallow:/

User-agent: WebCopier
Disallow:/

User-agent: EmailWolf
Disallow:/

User-agent: Googlebot-Image
Disallow: /images/*

User-agent: UbiCrawler
Disallow: /

User-agent: BDFetch
Disallow:/

User-agent: TMCrawler
Disallow:/

User-agent: Wget 
Disallow: /

User-agent: Slurp
Crawl-delay: 5

User-Agent: Charlotte
Disallow: / 

User-agent: swish-e
Disallow: / 

User-agent: MJ12bot
Disallow: / 

User-agent: accelobot
Disallow: / 

User-agent: 008
Disallow: / 

Sitemap: http://www.british-history.ac.uk/sitemapindex.xml