# robots.txt for http://www.fao.org/
# This file is not for hiding content from people. It is no substitue for security 
# If you are editing the robots.txt file - please COMMENT and DATE reason for every inclusion/exclusion ---nw-OCC-2013
#                                                 ^^^^^^^     ^^^^

#User-agent: 008  # No longer relevant 25/10/2013 nw
#Disallow: /

#User-Agent: cdlwas_bot # No longer relevant 25/10/2013 nw
#Disallow: # No longer relevant 25/10/2013 nw

User-agent: *
Disallow: /index.php
Disallow: /t3lib/                             # Nothing to see here
Disallow: /typo3/                             # Nothing to see here
#Google needs to read CSS and JS here - nw 29 Jul 2015 # Disallow: /typo3conf/
#Google needs to read CSS and JS here - nw 29 Jul 2015 # Disallow: /typo3temp/
Disallow: /appses/esc/escb/user/ #Obsolete site 4/12/2012
Disallow: /tc/tca/ #Obsolete site 4/12/2012
Disallow: /act-network/ #Obsolete site 4/12/2012
Disallow: /countryProfiles/ #Invalid Alias 4/12/2012	
Disallow: /landandwater/ #Obsolete site 4/12/2012
Disallow: /sd/researchinstitutions/ #Large source of errors 5/12/2012 - doesnt crawl 
Disallow: /services/ #IT Service Page
Disallow: /figis/flod/worms/ #issues with Linked Open Data Pages 6/1/2013
#Permitted pending fix (27/05/2014 - nw) Disallow: /figis/vrmf/finder/!/display/vessel/ #generating a lot of errors (30/04/2014 - nw)
Disallow: /alc/ #duplicating content at www.rlc.fao.org
Disallow: /pwb/2000/ #application currently giving errors (25/04/2014 - nw)
Disallow: /pwb/2002/ #application currently giving errors (28/03/2014 - nw)
Disallow: /pwb/2004/ #application currently giving errors (28/03/2014 - nw)
Disallow: /pwb/2005/ #application currently giving errors (28/03/2014 - nw)
Disallow: /pwb/2006/ #application currently giving errors (28/03/2014 - nw)
Disallow: /Participation/ #old site generating a lot of crawl errors 18/06/2013
Disallow: /geonetwork/ #Requested by KV-CIO 10/Oct/2013
Disallow: /*?id=*    #Disable non-realurl - re-instated 10/Oct/2013
Disallow: /*&type=98 #- specified in Google webmaster tools for the Google exclusion  - re-instated 10/Oct/2013
Disallow: /fileadmin/user_upload/PermRep/ #don't need to be indexed (23/06/2014 - nw)
Disallow: /fileadmin/user_upload/en/   #don't need to be indexed (11/07/2014 - permreps)
Disallow: /ag/agn/fv/ #(08/07/2014 - nw - causing a lot of errors)
Disallow: /icatalog/search/ #old application only /icatalog/inter-e.htm is working
Disallow: /glossary/spec-term-n.asp?* #old application giving errors
Disallow: /organicag/display/   #old application giving errors
Disallow: /figis/flod/   #giving errors nw 03/03/2015
Disallow: /easypol/output/forum.asp*   #giving errors nw 06/03/2015
Disallow: /hortivar/hortivar.htm* #giving errors nw 06/03/2015
Disallow: /nems/rss/ #giving errors nw 20/07/2015
Disallow: /ag/agp/ipnis #giving errors nw 20/07/2015