User-agent: 008
Disallow: /

User-agent: *
Disallow: /cgi-bin/
Disallow: /aids/e-newsletter/
Disallow: /pubtest/
Disallow: /spanish/aids/e-newsletter/
Disallow: /french/aids/e-newsletter/
Disallow: /hiv-aids-sessions/
Disallow: /2005may/
Disallow: /int/
Disallow: /earlychildhood/parenting_cd/
Disallow: /search/
Disallow: /french/search/
Disallow: /spanish/search/
Disallow: /arabic/search/
Disallow: /chinese/search/
Disallow: /emailarticle.php
Disallow: /voy/discussions/
Disallow: /*search*
Disallow: /appeals/tajikistan.html
Disallow: /appeals/lesotho.html
Disallow: /appeals/madagascar.html
Disallow: /appeals/zimbabwe.html
Disallow: /appeals/pakistan.html
Disallow: /appeals/burkina_faso.html
Disallow: /french/eu/crtoolkit/
Disallow: /spanish/eu/crtoolkit/
Disallow: /search/devprosearch_education.php
Disallow: /search/devprosearch_aids.php
Disallow: /*q=printme
Disallow: /node/

# Drupal sites
# For Main site

# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips/
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/

# For ROCO sites
# CSS, JS, Images
Allow: /*/core/*.css$
Allow: /*/core/*.css?
Allow: /*/core/*.js$
Allow: /*/core/*.js?
Allow: /*/core/*.gif
Allow: /*/core/*.jpg
Allow: /*/core/*.jpeg
Allow: /*/core/*.png
Allow: /*/core/*.svg
Allow: /*/profiles/*.css$
Allow: /*/profiles/*.css?
Allow: /*/profiles/*.js$
Allow: /*/profiles/*.js?
Allow: /*/profiles/*.gif
Allow: /*/profiles/*.jpg
Allow: /*/profiles/*.jpeg
Allow: /*/profiles/*.png
Allow: /*/profiles/*.svg
# Directories
Disallow: /*/core/
Disallow: /*/profiles/
# Files
Disallow: /*/README.txt
Disallow: /*/web.config
# Paths (clean URLs)
Disallow: /*/admin/
Disallow: /*/comment/reply/
Disallow: /*/filter/tips/
Disallow: /*/node/add/
Disallow: /*/search/
Disallow: /*/user/register/
Disallow: /*/user/password/
Disallow: /*/user/login/
Disallow: /*/user/logout/
# Paths (no clean URLs)
Disallow: /*/index.php/admin/
Disallow: /*/index.php/comment/reply/
Disallow: /*/index.php/filter/tips/
Disallow: /*/index.php/node/add/
Disallow: /*/index.php/search/
Disallow: /*/index.php/user/password/
Disallow: /*/index.php/user/register/
Disallow: /*/index.php/user/login/
Disallow: /*/index.php/user/logout/


Crawl-delay: 10

User-agent: Yandex
Crawl-delay: 30

User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36
Crawl-delay: 3

User-agent: reddot-scraper.unicef.org
Crawl-delay: 1