# robots.txt for http://www.fao.org/
# This file is not for hiding content from people. It is no substitue for security 
# If you are editing the robots.txt file - please COMMENT and DATE reason for every inclusion/exclusion ---nw-OCC-2013
#                                                 ^^^^^^^     ^^^^

#User-agent: 008  # No longer relevant 25/10/2013 nw
#Disallow: /

#User-Agent: cdlwas_bot # No longer relevant 25/10/2013 nw
#Disallow: # No longer relevant 25/10/2013 nw

User-agent: *
Disallow: /index.php
Disallow: /t3lib/                             # Nothing to see here
Disallow: /typo3/                             # Nothing to see here
#Google needs to read CSS and JS here - nw 29 Jul 2015 # Disallow: /typo3conf/
#Google needs to read CSS and JS here - nw 29 Jul 2015 # Disallow: /typo3temp/
Disallow: /appses/esc/escb/user/ #Obsolete site 4/12/2012
Disallow: /tc/tca/ #Obsolete site 4/12/2012
Disallow: /act-network/ #Obsolete site 4/12/2012
Disallow: /countryProfiles/ #Invalid Alias 4/12/2012	
Disallow: /landandwater/ #Obsolete site 4/12/2012
Disallow: /sd/researchinstitutions/ #Large source of errors 5/12/2012 - doesnt crawl 
Disallow: /services/ #IT Service Page
Disallow: /figis/flod/worms/ #issues with Linked Open Data Pages 6/1/2013
#Permitted pending fix (27/05/2014 - nw) Disallow: /figis/vrmf/finder/!/display/vessel/ #generating a lot of errors (30/04/2014 - nw)
Disallow: /alc/ #duplicating content at www.rlc.fao.org
Disallow: /pwb/2000/ #application currently giving errors (25/04/2014 - nw)
Disallow: /pwb/2002/ #application currently giving errors (28/03/2014 - nw)
Disallow: /pwb/2004/ #application currently giving errors (28/03/2014 - nw)
Disallow: /pwb/2005/ #application currently giving errors (28/03/2014 - nw)
Disallow: /pwb/2006/ #application currently giving errors (28/03/2014 - nw)
Disallow: /Participation/ #old site generating a lot of crawl errors 18/06/2013
Disallow: /*?id=*    #Disable non-realurl - re-instated 10/Oct/2013
Disallow: /*&type=98 #- specified in Google webmaster tools for the Google exclusion  - re-instated 10/Oct/2013
Disallow: /fileadmin/user_upload/PermRep/ #don't need to be indexed (23/06/2014 - nw)
Disallow: /fileadmin/user_upload/en/   #don't need to be indexed (11/07/2014 - permreps)
Disallow: /ag/agn/fv/ #(08/07/2014 - nw - causing a lot of errors)
Disallow: /icatalog/search/ #old application only /icatalog/inter-e.htm is working
Disallow: /glossary/spec-term-n.asp?* #old application giving errors
Disallow: /organicag/display/   #old application giving errors
Disallow: /figis/flod/   #giving errors nw 03/03/2015
Disallow: /easypol/output/forum.asp*   #giving errors nw 06/03/2015
Disallow: /hortivar/hortivar.htm* #giving errors nw 06/03/2015
Disallow: /nems/rss/ #giving errors nw 20/07/2015
Disallow: /ag/agp/ipnis #giving errors nw 20/07/2015
Disallow: /mediabase #giving errors nw 09/05/2016
Disallow: /fishery/gisfish/ #giving errors nw 09/05/2016
#START Cleanup the web September 2016.The following are marked as GONE in OCC list of sites - and there are no redirects in place
Disallow: /ag/againfo/programmes/en/pplpi/
Disallow: /multidisciplinary/
Disallow: /world/lobr/
Disallow: /sard/en/sard/754/1458/
Disallow: /tc/tcom/
Disallow: /nr/water/ies/
Disallow: /ethics/
Disallow: /waicent/portal/
Disallow: /WAICENT/whats_new/
Disallow: /waicent/promotion/
Disallow: /WAICENT/survey/
Disallow: /TC/spfs/asia/
Disallow: /prods/
Disallow: /waicent/qa/
Disallow: /waicent/promotion/
Disallow: /waicent/consultation/
Disallow: /waicent/forum/
Disallow: /waicent/survey/
Disallow: /waicent/search/
Disallow: /WAICENT/FAOINFO/AGRICULT/AGP/AGPC/doc/hort/
Disallow: /Desertification/
Disallow: /Country/Cuba/
Disallow: /Qinfo/
Disallow: /ICCD/
Disallow: /PRODS/cams/
Disallow: /Qinfo/cams/
Disallow: /urbanag/
Disallow: /xims/nars/
Disallow: /tc/ngo/
Disallow: /sd/sdrr/portal/
Disallow: /corporatecommunications/
Disallow: /renewal/
Disallow: /ag/againfo/themes/governance-livestock/
Disallow: /food/empres-food-safety/
Disallow: /ag/againfo/programmes/
Disallow: /cfs/cfsroundtable/cfsroundtable-home/
Disallow: /es/esc/foodpriceswing/
Disallow: /forestry/organization/2105/
Disallow: /forestry/organization/2113/
Disallow: /forestry/NfaKnowledgeRef/
Disallow: /forestry/35693/
Disallow: /forestry/14689/
Disallow: /forestry/infonote/
Disallow: /forestry/7864/
Disallow: /forestry/9473/
Disallow: /forestry/tof/
Disallow: /landandwater/
Disallow: /sard/
Disallow: /tc/tci/ClimateChange/
Disallow: /tc/dcp/
Disallow: /spatl/
Disallow: /tc/italiantrustfund-fs/
Disallow: /aFRica/central/
Disallow: /landandwater/iptrid/
Disallow: /africa/sfe/
Disallow: /aud/investigatorsconf/
Disallow: /cfs/cfsroundtable/
Disallow: /economic/ess/fcldatabase/
Disallow: /landandwater/cdwa/
Disallow: /ag/agp/orgfert/
Disallow: /sd/dim_pe4/
Disallow: /infoods/biodiversity/
Disallow: /waicent/faoinfo/agricult/agl/agll/prosoil/
Disallow: /sard/en/sardm/
Disallow: /appses/esc/escb/user/
Disallow: /WAICENT/Consultation/
Disallow: /WAICENT/Forum/
Disallow: /tc/tca/
Disallow: /ag/agn/agns/
Disallow: /ag/AGP/AGPC/doc/industr/
Disallow: /ag/magazine/
Disallow: /COAIM/cams/
Disallow: /FI/figis/cams/
Disallow: /fi/figis/gef/cams/
Disallow: /wssd/SARD/cams/ 
Disallow: /partnerships/great-green-wall/
Disallow: /food/nutrition-assessment/
Disallow: /wssd/SARD/cams/
Disallow: /catalog/
Disallow: /ag/agn/fv/
Disallow: /isfp/
#END Cleanup the web September 2016.The following are marked as GONE in OCC list of sites