# All search engine robots will spider the domain and all its files using User-agent: * # If you dont want to allow files being spidered use Disallow: filename.htm User-agent: * Disallow: /members_ip.php Disallow: /members_ips.php Disallow: /members_time.php Disallow: /members.php Disallow: /config.php Disallow: /private_directory/ Disallow: /profile.php Disallow: /administration/