#User-agent:* #Disallow: /*.zip$ #Disallow: /*.gif$ #Disallow: /*.jpg$ #Disallow: /*.ico$ #Disallow: /*.php$ #Disallow: /*.swf$ #Disallow: /*.txt$ User-agent:* Disallow: /api/ Disallow: /cache/ Disallow: /includes/ Disallow: /locales/ Disallow: /services/ Disallow: /templates/ Disallow: /admin.php Disallow: /password.php User-agent: EmailCollector Disallow: / User-agent: Teleport* Disallow: / User-agent: Webwhacker* Disallow: / User-agent: Webzip* Disallow: / User-agent: Net Attache* Disallow: / User-agent: SiteSnagger* Disallow: / User-agent: HTTrack* Disallow: / User-agent: WebCapture* Disallow: / User-agent: WebSauger* Disallow: / User-agent: WebReaper* Disallow: / User-agent: Roverbot* Disallow: / #User-agent: Firefly/1.0 #Disallow: /