# Deny Soso spider in the site <http://help.soso.com/webspider.htm>
#allowed on 23/12/2015
User-agent: Sosospider
#Disallow: /
Disallow: /php3
Allow: /
# Deny EC2LinkFinder in the site
User-agent: EC2LinkFinder
Disallow: /
# deny 80legs.com webcrawler
#allowed on 23/12/2015
User-agent: 008
#Disallow: /
Disallow: /php3
Allow: /
#deny http://www.metadatalabs.com/mlbot
#User-agent: MLBot
#Disallow: /
#deny yandex http://yandex.com/bots
#allowed on 23/12/2015
#User-agent: Yandex
#Crawl-delay: 500 # specifies a 500 seconds timeout
#Disallow: /
#Disallow: /php3
#Allow: /
User-agent: Yandex
Disallow: /
# prohibits downloading anything except for the pages 
# starting with '/cgi-bin'
#deny Sogou spider http://www.sogou.com/docs/help/webmasters.htm#07
#allowed on 23/12/2015
User-agent: Sogou
#Disallow: /
Disallow: /php3
Allow: /
# <http://www.youdao.com/help/webmaster/spider/>
#allowed on 23/12/2015
User-agent: YoudaoBot
#Disallow: /
Disallow: /php3
Allow: /
# <http://help.naver.com/robots/>
#allowed on 23/12/2015
User-agent: Naverbot
#Disallow: /
Disallow: /php3
Allow: /
#user agent Yeti
#allowed on 23/12/2015
User-agent: Yeti
#Disallow: /
Disallow: /php3
Allow: /
# <http://help.goo.ne.jp/door/crawler.html>
#allowed on 23/12/2015
User-agent: ichiro
#Disallow: /
Disallow: /php3
Allow: /
# Ils font payer les donnees d'autrui.
# <http://spinn3r.com/robot>
User-agent: Spinn3r
Disallow: /
#google case
#User-agent: Googlebot
#Disallow: /php3
#Allow: /
User-agent: Googlebot
Disallow: 
Allow: /
User-agent: Mediapartners-Google
Disallow: 
Allow: /
#proximic used by amazon EC2 cloud, we already blacklisted some of their IPs for abuse
User-agent: proximic
Disallow: /
#general case
User-agent: *
Disallow: /php3
Allow: /
Sitemap: https://fxtop.com/sitemap.xml
Crawl-delay: 600 
# specifies a 600 seconds timeout