[interchange-cvs] [interchange] Add some more user-agents to the robots list

[Jon Jensen]

commit 6d392823612a12495f03beaec5388628f844c78b
Author: Jon Jensen <jon at endpoint.com>
Date:   Wed Dec 9 10:12:43 2009 -0700

    Add some more user-agents to the robots list
    
    A few of these include substring "bot", which is already covered,
    but I'm adding them explicitly so they'll still be there in case
    the very short "bot" ever causes trouble and is removed.

 dist/robots.cfg |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)
---
diff --git a/dist/robots.cfg b/dist/robots.cfg
index eb0b5ad..f4b30fe 100644
--- a/dist/robots.cfg
+++ b/dist/robots.cfg
@@ -1,7 +1,9 @@
 RobotUA <<EOR
     adressendeutschland,
+    AdsBot-Google,
     agent,
     AltaVista,
+    Apache (internal dummy connection),
     appie,
     AppleSyndication,
     Arachnoidea,
@@ -21,9 +23,11 @@ RobotUA <<EOR
     BookmarkSync,
     bot,
     Builder,
+    CCBot,
     ccubee,
     cfetch,
     CFNetwork,
+    check_http,
     CMC,
     collector,
     complex_network_group,
@@ -37,6 +41,7 @@ RobotUA <<EOR
     EZResult,
     FavOrg,
     FeedDemon,
+    FeedFetcher-Google,
     Feedreader,
     FeedValidator,
     Ferret,
@@ -88,6 +93,7 @@ RobotUA <<EOR
     Miva,
     moget,
     MSFrontPage,
+    msnbot,
     Nazilla,
     NetMechanic,
     NetScoop,
@@ -113,6 +119,7 @@ RobotUA <<EOR
     search,
     seek,
     shelob,
+    ShopWiki,
     silk,
     Slurp,
     sna,