# $Id: robots.txt,v 1.9.2.1 2008/12/10 20:12:19 goba Exp $
#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /sites/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
# MW: Boost
Disallow: /boost_stats.php
# MW: Homepage
Disallow: /homepage
# MW: Language duplicates
Allow: /af/dbp/thing/
Disallow: /af/
Disallow: /af
Allow: /an/dbp/thing/
Disallow: /an/
Disallow: /an
Allow: /ar/dbp/thing/
Disallow: /ar/
Disallow: /ar
Allow: /ast/dbp/thing/
Disallow: /ast/
Disallow: /ast
Allow: /az/dbp/thing/
Disallow: /az/
Disallow: /az
Allow: /bat-smg/dbp/thing/
Disallow: /bat-smg/
Disallow: /bat-smg
Allow: /be/dbp/thing/
Disallow: /be/
Disallow: /be
Allow: /be-tarask/dbp/thing/
Disallow: /be-tarask/
Disallow: /be-tarask
Allow: /bg/dbp/thing/
Disallow: /bg/
Disallow: /bg
Allow: /bn/dbp/thing/
Disallow: /bn/
Disallow: /bn
Allow: /bpy/dbp/thing/
Disallow: /bpy/
Disallow: /bpy
Allow: /br/dbp/thing/
Disallow: /br/
Disallow: /br
Allow: /bs/dbp/thing/
Disallow: /bs/
Disallow: /bs
Allow: /ca/dbp/thing/
Disallow: /ca/
Disallow: /ca
Allow: /ceb/dbp/thing/
Disallow: /ceb/
Disallow: /ceb
Allow: /cs/dbp/thing/
Disallow: /cs/
Disallow: /cs
Allow: /cv/dbp/thing/
Disallow: /cv/
Disallow: /cv
Allow: /cy/dbp/thing/
Disallow: /cy/
Disallow: /cy
Allow: /da/dbp/thing/
Disallow: /da/
Disallow: /da
Allow: /de/dbp/thing/
Disallow: /de/
Disallow: /de
Allow: /el/dbp/thing/
Disallow: /el/
Disallow: /el
Allow: /eo/dbp/thing/
Disallow: /eo/
Disallow: /eo
Allow: /es/dbp/thing/
Disallow: /es/
Disallow: /es
Allow: /et/dbp/thing/
Disallow: /et/
Disallow: /et
Allow: /eu/dbp/thing/
Disallow: /eu/
Disallow: /eu
Allow: /fa/dbp/thing/
Disallow: /fa/
Disallow: /fa
Allow: /fi/dbp/thing/
Disallow: /fi/
Disallow: /fi
Allow: /fr/dbp/thing/
Disallow: /fr/
Disallow: /fr
Allow: /fy/dbp/thing/
Disallow: /fy/
Disallow: /fy
Allow: /gl/dbp/thing/
Disallow: /gl/
Disallow: /gl
Allow: /gu/dbp/thing/
Disallow: /gu/
Disallow: /gu
Allow: /he/dbp/thing/
Disallow: /he/
Disallow: /he
Allow: /hi/dbp/thing/
Disallow: /hi/
Disallow: /hi
Allow: /hr/dbp/thing/
Disallow: /hr/
Disallow: /hr
Allow: /ht/dbp/thing/
Disallow: /ht/
Disallow: /ht
Allow: /hu/dbp/thing/
Disallow: /hu/
Disallow: /hu
Allow: /id/dbp/thing/
Disallow: /id/
Disallow: /id
Allow: /io/dbp/thing/
Disallow: /io/
Disallow: /io
Allow: /is/dbp/thing/
Disallow: /is/
Disallow: /is
Allow: /it/dbp/thing/
Disallow: /it/
Disallow: /it
Allow: /ja/dbp/thing/
Disallow: /ja
Disallow: /ja
Allow: /jv/dbp/thing/
Disallow: /jv/
Disallow: /jv
Allow: /ka/dbp/thing/
Disallow: /ka/
Disallow: /ka
Allow: /ko/dbp/thing/
Disallow: /ko/
Disallow: /ko
Allow: /ksh/dbp/thing/
Disallow: /ksh/
Disallow: /ksh
Allow: /ku/dbp/thing/
Disallow: /ku/
Disallow: /ku
Allow: /la/dbp/thing/
Disallow: /la/
Disallow: /la
Allow: /lb/dbp/thing/
Disallow: /lb/
Disallow: /lb
Allow: /lt/dbp/thing/
Disallow: /lt/
Disallow: /lt
Allow: /lv/dbp/thing/
Disallow: /lv/
Disallow: /lv
Allow: /mk/dbp/thing/
Disallow: /mk/
Disallow: /mk
Allow: /mr/dbp/thing/
Disallow: /mr/
Disallow: /mr
Allow: /ms/dbp/thing/
Disallow: /ms/
Disallow: /ms
Allow: /nap/dbp/thing/
Disallow: /nap/
Disallow: /nap
Allow: /nds/dbp/thing/
Disallow: /nds/
Disallow: /nds
Allow: /new/dbp/thing/
Disallow: /new/
Disallow: /new
Allow: /nl/dbp/thing/
Disallow: /nl
Disallow: /nl
Allow: /nn/dbp/thing/
Disallow: /nn/
Disallow: /nn
Allow: /nb/dbp/thing/
Disallow: /nb/
Disallow: /nb
Allow: /oc/dbp/thing/
Disallow: /oc/
Disallow: /oc
Allow: /pl/dbp/thing/
Disallow: /pl/
Disallow: /pl
Allow: /pms/dbp/thing/
Disallow: /pms/
Disallow: /pms
Allow: /pt/dbp/thing/
Disallow: /pt/
Disallow: /pt
Allow: /qu/dbp/thing/
Disallow: /qu/
Disallow: /qu
Allow: /ro/dbp/thing/
Disallow: /ro/
Disallow: /ro
Allow: /ru/dbp/thing/
Disallow: /ru/
Disallow: /ru
Allow: /scn/dbp/thing/
Disallow: /scn/
Disallow: /scn
Allow: /sh/dbp/thing/
Disallow: /sh/
Disallow: /sh
Allow: /sk/dbp/thing/
Disallow: /sk/
Disallow: /sk
Allow: /sl/dbp/thing/
Disallow: /sl/
Disallow: /sl
Allow: /sq/dbp/thing/
Disallow: /sq/
Disallow: /sq
Allow: /sr/dbp/thing/
Disallow: /sr/
Disallow: /sr
Allow: /su/dbp/thing/
Disallow: /su/
Disallow: /su
Allow: /sv/dbp/thing/
Disallow: /sv/
Disallow: /sv
Allow: /sw/dbp/thing/
Disallow: /sw/
Disallow: /sw
Allow: /ta/dbp/thing/
Disallow: /ta/
Disallow: /ta
Allow: /te/dbp/thing/
Disallow: /te/
Disallow: /te
Allow: /th/dbp/thing/
Disallow: /th/
Disallow: /th
Allow: /tl/dbp/thing/
Disallow: /tl/
Disallow: /tl
Allow: /tr/dbp/thing/
Disallow: /tr/
Disallow: /tr
Allow: /uk/dbp/thing/
Disallow: /uk/
Disallow: /uk
Allow: /ur/dbp/thing/
Disallow: /ur/
Disallow: /ur
Allow: /vi/dbp/thing/
Disallow: /vi/
Disallow: /vi
Allow: /vo/dbp/thing/
Disallow: /vo/
Disallow: /vo
Allow: /wa/dbp/thing/
Disallow: /wa/
Disallow: /wa
Allow: /war/dbp/thing/
Disallow: /war/
Disallow: /war
Allow: /zh/dbp/thing/
Disallow: /zh/
Disallow: /zh
Allow: /zh-yue/dbp/thing/
Disallow: /zh-yue/
Disallow: /zh-yue