# SiteBot ignores robots.txt
User-agent: sitebot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: *
Disallow: /websvn/dl.php
Disallow: /bugzilla/show_bug.cgi*ctype*
Disallow: /bugzilla/show_bug.cgi*format*

User-agent: AhrefsBot
Crawl-Delay: 3

User-agent: MJ12bot
Crawl-Delay: 3

#Disallow: /websvn/blame.php
#Disallow: /websvn/
#Disallow: /bugzilla/buglist.cgi

# TODO: integrate http://code.google.com/p/bugzilla-sitemap/
## Disallow: /bugzilla/
## Disallow: /bugzilla/show_bug.cgi*&*
## Allow: /bugzilla/show_bug.cgi?id=*
## Allow: /bugzilla/index.cgi
## Allow: /bugzilla/index.cgi$
## Allow: /bugzilla/*atomfeed.cgi
## Allow: /bugzilla/sitemap*xml.gz
## Allow: /bugzilla/sitemap_index.xml
## Sitemap: http://forge.univention.org/bugzilla/sitemap/