User-agent: *
Allow: /

Disallow: /resources/spotlight/
Disallow: /resources/redirect/
Disallow: /resources/press/
Disallow: /redirects/
Disallow: /landing
Disallow: /search
Disallow: /hidden/
Disallow: /styleguide
Disallow: /new/

Sitemap: https://pangea.app/sitemap1.xml.gz
Sitemap: https://pangea.app/sitemap
Sitemap: https://pangea.app/freelancers/sitemap

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /