###################################### # robots.txt is used by search engine robots # to know what it is allowed to search and scan # # list all search robots that can access our site # example - User-agent: # example - User-agent: * <-- allow all robots access # User-agent: * # # list any folders or files that you do not want robots to search or scan # example - Disallow: /products <-- don't scan products folder # example - Disallow: /support/default.asp <-- don't scan default.asp # example - Disallow: <-- allow robots to scan all files and folders # Disallow: /admin/ Disallow: /downloads/ Disallow: /newsletters/ # #####################################