//robots.txt

# Allow all crawlers
User-agent: *
Disallow: /institutions/
Disallow: /ranking/
Disallow: /sources/
Disallow: /fields/
Disallow: /subfields/
Disallow: /topics/

Crawl-delay: 5