Sitemap: http://www.caring.com/sitemap_index.xml
User-Agent: *
Allow: /register/to_makes_candles
Disallow: /candles/new
Disallow: /account/*
Disallow: /admin/
Disallow: /register/
Disallow: /search?query=*
Disallow: /questions/new$
Disallow: /answers/new?question=*
Disallow: /*reviews/new$
Disallow: /community/posts/*/post_ratings
Disallow: /forums/posts/*/post_ratings
Disallow: /content/item_users/rate_item*
Allow: /candles/sitemap/*?page=*
# SAF: temporarily remove $ to prevent system bug from leading to unindexed pages; restore trailing $ after 9/1/09
Disallow: /*/*/comments
Disallow: /local/search?
Disallow: /*/comments/new$
# SAF: temporarily remove $ to prevent system bug from leading to unindexed pages; restore trailing $ after 9/1/09
Disallow: /*/*/comments/new

# not our URLs, but crawlers got directed there 
Disallow: /items/search?query=*
Disallow: /ask/search?search%5B*
Disallow: /ask-*
Disallow: /send-to-friends/*

# internal urls (ajax, etc.) that shouldn't be crawled
Disallow: /event_logs/*
Disallow: /local/resources/phone_number*

# old urls that might get stuck in search indicies
Disallow: /community/monitorships/*
Disallow: /forums/monitorships/*

# sadly, google parses our javascript/ajax urls as links, so prevent traversal
#  there's a code fix that should have the /activity url fixed, so the directive
#  could be removed after that time
Disallow: /community/users/*/activity$
Disallow: /people/*/activity$

# sadly, google parses a plain literal 'undefined' from javascript and constructs a URL.
Disallow: /*/undefined$

# no forum searches
Disallow: /posts/search?forum_id=*

# virtual page-views, picked up by google javascript parser
Disallow: /virtual/*

# supress duplicate print-formatted pages
Disallow: /*?print=true$

# unfortunately, agents only look at one section, so duplicate the above for slurp, plus
#  slurp-specific directives
User-Agent: Slurp
Allow: /register/to_makes_candles
Disallow: /candles/new
Disallow: /account/*
Disallow: /admin/
Disallow: /register/
Disallow: /search?query=*
Disallow: /questions/new$
Disallow: /answers/new?question=*
Disallow: /*reviews/new$
Disallow: /community/posts/*/post_ratings
Disallow: /forums/posts/*/post_ratings
Disallow: /content/item_users/rate_item*
Allow: /candles/sitemap/*?page=*
Disallow: /*/*/comments
Disallow: /local/search?
Disallow: /*/comments/new$
Disallow: /*/*/comments/new

# not our URLs, but crawlers got directed there 
Disallow: /items/search?query=*
Disallow: /ask/search?search%5B*
Disallow: /ask-*
Disallow: /send-to-friends/*

# internal urls (ajax, etc.) that shouldn't be crawled
Disallow: /event_logs/*
Disallow: /local/resources/phone_number*

# old urls that might get stuck in search indicies
Disallow: /community/monitorships/*
Disallow: /forums/monitorships/*

# sadly, google parses our javascript/ajax urls as links, so prevent traversal
#  there's a code fix that should have the /activity url fixed, so the directive
#  could be removed after that time
Disallow: /community/users/*/activity$
Disallow: /people/*/activity$

# virtual page-views, picked up by google javascript parser
Disallow: /virtual/*

# no forum searches
Disallow: /posts/search?forum_id=*

# supress duplicate print-formatted pages
Disallow: /*/*?print=true$

# Slurp-specific: we found slurp spending half its time repeatedly downloading css, so exclude...
Disallow: /stylesheets/*