# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

User-Agent: *
Crawl-delay: 30
Disallow: /checkout
Disallow: /checkout/direct-from-partner
Disallow: /home
Disallow: /users/*/edit
Disallow: /users/*/impact
Disallow: /user/*
Disallow: /fund/*
Disallow: /borrower/*
Disallow: /timeline/*

User-Agent: AdsBot-Google
Crawl-delay: 30
Disallow: /checkout
Disallow: /checkout/direct-from-partner
Disallow: /home
Disallow: /users/*/edit
Disallow: /users/*/impact
Disallow: /user/*
Disallow: /fund/*
Disallow: /borrower/*
Disallow: /timeline/*

User-agent: Twitterbot
Allow: /users/*
Allow: /users/*/impact/*

Sitemap: https://milaap.org/sitemap.xml.gz