# # WebmasterWorld.com: robots.txt # GNU Robots.txt Feel free to use with credit given to WebmasterWorld. # Please, we do NOT allow nonauthorized robots any longer. # http://www.searchengineworld.com/robots/ # Yes, feel free to copy and use the following. User-agent: * Disallow: /TeamCompare.asp Disallow: /Forum/ Disallow: /forum/ # User-agent: Googlebot # Disallow: / User-agent: jyxobot Disallow: / User-agent: jyxobot/1 Disallow: / User-agent: ASPSeek User-agent: asterias User-agent: baiduspider User-agent: BravoBrian User-agent: dloader(NaverRobot) User-agent: Dumbot User-agent: EgotoBot User-agent: Gaisbot User-agent: Generic User-agent: grub-client User-agent: http://www.almaden.ibm.com/cs/crawler User-agent: InfoNaviRobot User-agent: Larbin User-agent: MSIECrawler User-agent: NaverRobot User-agent: NexaBot User-agent: NPBot User-agent: obot User-agent: OWR_Crawler User-agent: PhpDig User-agent: psbot User-agent: puf User-agent: QuepasaCreep User-agent: rabaz User-agent: Reaper User-agent: RPT-HTTPClient User-agent: ScoutAbout User-agent: semanticdiscovery User-agent: Steeler User-agent: Teleport Pro User-agent: TurnitinBot User-agent: TutorGig User-agent: twiceler User-agent: vsecrawler User-agent: Wget Disallow: / User-agent: fast Disallow: / User-agent: startsidenbot Disallow: / User-agent: Startsidenbot Disallow: / User-agent: Findexa Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: ia_archiver Disallow: /TeamCompare.asp Disallow: /Hotte.asp User-agent: GTS_Crawler Disallow: / User-agent: GTS_Crawler Disallow: / # Korean based ... User-agent: NaverBot Disallow: / User-agent: TurnitinBot Disallow: /TeamCompare.asp Disallow: /Hotte.asp User-agent: slurp # Crawl-delay: 20 # Seconds Disallow: /TeamCompare.asp Disallow: /Hotte.asp Disallow: /Forum/ User-agent: MSNBot Disallow: /