User-Agent: googlebot
User-Agent: bingbot
User-Agent: msnbot
User-Agent: Twitterbot
User-Agent: Slurp
User-Agent: DuckDuckBot
Crawl-delay: 30
sitemap: https://www.cuny.edu/sitemap.xml
sitemap: https://policy.cuny.edu/sitemap.xml
Disallow: /global-components/
Disallow: /d-i/
Disallow: /cgi-bin/
Disallow: /search/
Disallow: /wp-admin
Disallow: /alumni-test/
Disallow: /role-template/
Disallow: /home-preview/
Disallow: /about/administration/offices/sa/
Disallow: /homepage/digital-displays/
Disallow: /employment/search-jobs/
Disallow: /old-version/
Disallow: /policyimport/

User-agent: SemanticScholarBot
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: *
Disallow: /