User-agent: bingbot
Crawl-Delay: 20
Disallow: /maintenance
Disallow: /sorry

User-Agent: msnbot
Disallow: /maintenance
Disallow: /sorry

User-agent: Mediapartners-Google
Allow: /
Disallow: /maintenance
Disallow: /sorry

User-agent: Googlebot-Image
Disallow: /maintenance
Disallow: /sorry

User-Agent: Googlebot
Disallow: /maintenance
Disallow: /sorry

User-Agent: Googlebot-Mobile
Disallow: /maintenance
Disallow: /sorry

User-agent: Twitterbot
Disallow: /maintenance
Disallow: /sorry

User-agent: Facebot
Disallow: /maintenance
Disallow: /sorry

User-agent: Slurp
Disallow: /maintenance
Disallow: /sorry

User-agent: Yahoo! Slurp
Disallow: /maintenance
Disallow: /sorry

User-agent: Y!J-DLC/1.0
Disallow: /maintenance
Disallow: /sorry

User-agent: Y!J-BRY/YATSH
Disallow: /maintenance
Disallow: /sorry

User-agent: Y!J-BRJ/YATS
Disallow: /maintenance
Disallow: /sorry

User-agent: popIn_Agent
Disallow: /maintenance
Disallow: /sorry

User-agent: logly
Disallow: /maintenance
Disallow: /sorry

User-agent: YahooCacheSystem
Disallow: /
Allow: /topics/feed
Allow: /topics/candidates
Allow: /topics/reserved

User-agent: Pinterest
Disallow: /matome
Disallow: /search2
Disallow: /twitter/search
Disallow: /twitter/pickup
Disallow: /twitter/weather
Disallow: /twitter/conversation

User-agent: Steeler
Disallow: /matome
Disallow: /search2
Disallow: /twitter/search
Disallow: /twitter/pickup
Disallow: /twitter/weather
Disallow: /twitter/conversation

User-agent: grapeshot
Disallow:

# Set for the general agent at the last.
User-agent: *
Disallow: /