- #347 when running multiple RhodeCode instances, properly invalidates cache

  for all registered servers
  - server start also clears all cache keys now

--HG--
branch : beta

4 files changed, 47 insertions, 21 deletions

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 52a46465..b52ecbb1 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -28,6 +28,8 @@ news
 - #402 removed group prefix from repository name when listing repositories 
   inside a group
 - added gravatars into permission view and permissions autocomplete
+- #347 when running multiple RhodeCode instances, properly invalidates cache 
+  for all registered servers
 
 fixes
 +++++
diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index 079dee57..edc5d4c3 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -51,7 +51,8 @@ from rhodecode.lib.caching_query import FromCache
 
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User, RhodeCodeUi, \
-    UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm
+    UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm,\
+    CacheInvalidation
 from rhodecode.model.meta import Session
 from rhodecode.model.repos_group import ReposGroupModel
 from rhodecode.lib.utils2 import safe_str, safe_unicode
@@ -452,13 +453,19 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     sa.commit()
     removed = []
     if remove_obsolete:
-        #remove from database those repositories that are not in the filesystem
+        # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
+                log.debug("Removing non existing repository found in db %s" %
+                          repo.repo_name)
                 removed.append(repo.repo_name)
                 sa.delete(repo)
                 sa.commit()
 
+    # clear cache keys
+    log.debug("Clearing cache keys now...")
+    CacheInvalidation.clear_cache()
+    sa.commit()
     return added, removed
 
 
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 1e3b8756..2912457f 100755
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -1042,11 +1042,14 @@ class CacheInvalidation(Base, BaseModel):
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
+    @classmethod
+    def clear_cache(cls):
+        cls.query().delete()
 
     @classmethod
     def _get_key(cls, key):
         """
-        Wrapper for generating a key
+        Wrapper for generating a key, together with a prefix
 
         :param key:
         """
@@ -1055,12 +1058,25 @@ class CacheInvalidation(Base, BaseModel):
         iid = rhodecode.CONFIG.get('instance_id')
         if iid:
             prefix = iid
-        return "%s%s" % (prefix, key)
+        return "%s%s" % (prefix, key), prefix, key.rstrip('_README')
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.cache_key == key).scalar()
-
+    
+    @classmethod
+    def _get_or_create_key(cls, key, prefix, org_key):
+        inv_obj = Session.query(cls).filter(cls.cache_key == key).scalar()
+        if not inv_obj:
+            try:
+                inv_obj = CacheInvalidation(key, org_key)
+                Session.add(inv_obj)
+                Session.commit()
+            except Exception:
+                log.error(traceback.format_exc())
+                Session.rollback()                
+        return inv_obj
+            
     @classmethod
     def invalidate(cls, key):
         """
@@ -1070,10 +1086,12 @@ class CacheInvalidation(Base, BaseModel):
 
         :param key:
         """
-        return cls.query()\
-                .filter(CacheInvalidation.cache_key == key)\
-                .filter(CacheInvalidation.cache_active == False)\
-                .scalar()
+        
+        key, _prefix, _org_key = cls._get_key(key)
+        inv = cls._get_or_create_key(key, _prefix, _org_key)
+
+        if inv and inv.cache_active is False:
+            return inv
 
     @classmethod
     def set_invalidate(cls, key):
@@ -1083,17 +1101,16 @@ class CacheInvalidation(Base, BaseModel):
         :param key:
         """
 
-        log.debug('marking %s for invalidation' % key)
-        inv_obj = Session.query(cls)\
-            .filter(cls.cache_key == key).scalar()
-        if inv_obj:
-            inv_obj.cache_active = False
-        else:
-            log.debug('cache key not found in invalidation db -> creating one')
-            inv_obj = CacheInvalidation(key)
-
+        key, _prefix, _org_key = cls._get_key(key)
+        inv_objs = Session.query(cls).filter(cls.cache_args == _org_key).all()
+        log.debug('marking %s key[s] %s for invalidation' % (len(inv_objs),
+                                                             _org_key))
         try:
-            Session.add(inv_obj)
+            for inv_obj in inv_objs:
+                if inv_obj:
+                    inv_obj.cache_active = False
+
+                Session.add(inv_obj)
             Session.commit()
         except Exception:
             log.error(traceback.format_exc())
diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py
index 9d2aa466..cc9ef87a 100644
--- a/rhodecode/model/scm.py
+++ b/rhodecode/model/scm.py
@@ -235,13 +235,13 @@ class ScmModel(BaseModel):
         return group_iter
 
     def mark_for_invalidation(self, repo_name):
-        """Puts cache invalidation task into db for
+        """
+        Puts cache invalidation task into db for
         further global cache invalidation
 
         :param repo_name: this repo that should invalidation take place
         """
         CacheInvalidation.set_invalidate(repo_name)
-        CacheInvalidation.set_invalidate(repo_name + "_README")
 
     def toggle_following_repo(self, follow_repo_id, user_id):