User-agent: * Disallow: /helsinki/ # This is an infinite virtual URL space User-agent: * Disallow: /2.5d.videotroopers.com/ # This is an infinite virtual URL space User-Agent: sitecheck.internetseer.com # die kan er dus niet meer komen. Disallow: / User-agent: Java Disallow: / User-agent: psycheclone Disallow: / User-agent: MLBot Disallow: / User-agent: Yandex Disallow: / User-agent: TurnitinBot Disallow: / User-agent: Speedy Disallow: / User-agent: Fasterfox Disallow: / User-agent: Snappy Disallow: / User-Agent: WebarooBot Disallow: / User-agent: BoardTracker Disallow: / User-agent: Fatbot Disallow: / User-agent: ActiveTouristBot Disallow: / User-agent: Nutch Disallow: / User-agent: Digimarc CGIReader/*.* Disallow: / User-agent: Digimarc WebReader/*.* Disallow: / User-agent: studybot/1.0 Disallow: / User-agent: Steeler Disallow: / User-agent: SnapPreviewBot Disallow: / User-agent: Teemer Disallow: / User-agent: BioSearch Disallow: / User-agent: HTTrack Disallow: / User-agent: HTTrack 3.0 Disallow: / User-agent: Python-urllib Disallow: / User-agent: libwww-perl Disallow: / User-agent: Jakarta Disallow: / User-agent: larbin Disallow: / User-agent: ia_archiver Disallow: / User-agent: webfetch Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: GetRight Disallow: / User-agent: MJ12bot Disallow: / User-agent: MJ12bot/v.7 Disallow: / User-agent: BrightCrawler Disallow: / User-agent: SurveyBot Disallow: / User-agent: librabot Disallow: / User-agent: BecomeBot Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: BoardReader Disallow: / User-agent: TechrigyBot Disallow: / User-agent: Spinn3r Disallow: / User-agent: Gnomit Disallow: / User-agent: zibber Disallow: / User-agent: Banana Crawler Disallow: / User-Agent: OnTownsBot Disallow: / User-Agent: Jyxobot/1 Disallow: / User-Agent: ZipppBot/Nutch-0.9 Disallow: / User-Agent: FyberSpider Disallow: / User-Agent: Speedy Spider Disallow: / User-Agent: Gaisbot Disallow: / User-Agent: OpenfosBot Disallow: / User-Agent: ccbot Disallow: / User-Agent: Flatlandbot Disallow: / User-Agent: baypup Disallow: / User-Agent: Scrubby/3.0 Disallow: / User-Agent: Snapbot Disallow: / User-agent: CentiverseBot Disallow: / User-agent: Indy Library Disallow: / User-agent: twiceler Crawl-delay: 20 User-agent: PageNest Disallow: / User-agent: GeoHasher Disallow: / User-agent: 80bot Disallow: / User-agent: 008 Disallow: / User-agent: Scrapy Disallow: / User-agent: PureBot Disallow: / User-agent: plukkie Disallow: / User-agent: psbot Disallow: /