Reimplemented way of caching repos list, hg model now get's repos objects right from cached dict, this way we skipp creating instances of MercurialRepository and gain performance. Some import cleanup

14 files changed, 126 insertions, 91 deletions

diff --git a/pylons_app/controllers/branches.py b/pylons_app/controllers/branches.py
index 0d19772a..cb65337e 100644
--- a/pylons_app/controllers/branches.py
+++ b/pylons_app/controllers/branches.py
@@ -1,14 +1,10 @@
-import logging
-
-from pylons import tmpl_context as c, app_globals as g, session, request, config, url
-from pylons.controllers.util import abort, redirect
-
+from pylons import tmpl_context as c, app_globals as g
+from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model.hg_model import HgModel
-from pylons_app.lib.auth import LoginRequired
-log = logging.getLogger(__name__)
+import logging
 
+log = logging.getLogger(__name__)
 
 class BranchesController(BaseController):
     
diff --git a/pylons_app/controllers/changelog.py b/pylons_app/controllers/changelog.py
index 44658c8a..19fede32 100644
--- a/pylons_app/controllers/changelog.py
+++ b/pylons_app/controllers/changelog.py
@@ -1,11 +1,10 @@
 from mercurial.graphmod import revisions as graph_rev, colored, CHANGESET
 from mercurial.node import short
-from pylons import request, response, session, tmpl_context as c, url, config, \
-    app_globals as g
-from pylons.controllers.util import abort, redirect
+from pylons import request, session, tmpl_context as c
 from pylons_app.lib.auth import LoginRequired
-from pylons_app.lib.base import BaseController, render, _full_changelog_cached
+from pylons_app.lib.base import BaseController, render
 from pylons_app.lib.filters import age as _age, person
+from pylons_app.model.hg_model import _full_changelog_cached
 from simplejson import dumps
 from webhelpers.paginate import Page
 import logging
diff --git a/pylons_app/controllers/changeset.py b/pylons_app/controllers/changeset.py
index f25fee9c..ef1dde55 100644
--- a/pylons_app/controllers/changeset.py
+++ b/pylons_app/controllers/changeset.py
@@ -1,6 +1,4 @@
-from pylons import request, response, session, tmpl_context as c, url, config, \
-    app_globals as g
-from pylons.controllers.util import abort, redirect
+from pylons import tmpl_context as c
 from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
 from pylons_app.model.hg_model import HgModel
diff --git a/pylons_app/controllers/feed.py b/pylons_app/controllers/feed.py
index 44ca6c7d..251739a7 100644
--- a/pylons_app/controllers/feed.py
+++ b/pylons_app/controllers/feed.py
@@ -1,13 +1,10 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-import logging
-from operator import itemgetter
-from pylons import tmpl_context as c, request, config, url, response
-from pylons_app.lib.base import BaseController, render, _full_changelog_cached
-from pylons_app.lib.utils import get_repo_slug
-from pylons_app.model.hg_model import HgModel
-from pylons_app.lib.auth import LoginRequired
+from pylons import tmpl_context as c, url, response
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import _full_changelog_cached
 from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed
+import logging
 log = logging.getLogger(__name__)
 
 class FeedController(BaseController):
@@ -35,8 +32,9 @@ class FeedController(BaseController):
             if cnt > self.feed_nr:
                 break
             feed.add_item(title=cs.message,
-                          link=url('changeset_home', repo_name=repo_name, revision=cs.raw_id, qualified=True),
-                          description=str(cs.date))
+                          link=url('changeset_home', repo_name=repo_name,
+                                   revision=cs.raw_id, qualified=True),
+                                   description=str(cs.date))
         
         response.content_type = feed.mime_type
         return feed.writeString('utf-8')
diff --git a/pylons_app/controllers/files.py b/pylons_app/controllers/files.py
index cf417560..af755c33 100644
--- a/pylons_app/controllers/files.py
+++ b/pylons_app/controllers/files.py
@@ -1,15 +1,12 @@
-import tempfile
-from pylons import request, response, session, tmpl_context as c, url, config, \
-    app_globals as g
-from pylons.controllers.util import abort, redirect
+from mercurial import archival
+from pylons import request, response, session, tmpl_context as c, url
 from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model.hg_model import HgModel
 from vcs.exceptions import RepositoryError, ChangesetError
 from vcs.utils import diffs as differ
 import logging
-from mercurial import archival
+import tempfile
 
         
 log = logging.getLogger(__name__)
diff --git a/pylons_app/controllers/hg.py b/pylons_app/controllers/hg.py
index 1b5ca5b6..ba9b095e 100644
--- a/pylons_app/controllers/hg.py
+++ b/pylons_app/controllers/hg.py
@@ -1,10 +1,11 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-import logging
 from operator import itemgetter
 from pylons import tmpl_context as c, request, config
-from pylons_app.lib.base import BaseController, render
 from pylons_app.lib.auth import LoginRequired
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import HgModel
+import logging
 log = logging.getLogger(__name__)
 
 class HgController(BaseController):
@@ -18,12 +19,12 @@ class HgController(BaseController):
         cs = c.current_sort
         c.cs_slug = cs.replace('-', '')
         sortables = ['name', 'description', 'last_change', 'tip', 'contact']
-        
+        cached_repo_list = HgModel().get_repos()
         if cs and c.cs_slug in sortables:
             sort_key = c.cs_slug + '_sort'
             if cs.startswith('-'):
-                c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), reverse=True)
+                c.repos_list = sorted(cached_repo_list, key=itemgetter(sort_key), reverse=True)
             else:
-                c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), reverse=False)
+                c.repos_list = sorted(cached_repo_list, key=itemgetter(sort_key), reverse=False)
             
         return render('/index.html')
diff --git a/pylons_app/controllers/shortlog.py b/pylons_app/controllers/shortlog.py
index 7743909d..fe7730f3 100644
--- a/pylons_app/controllers/shortlog.py
+++ b/pylons_app/controllers/shortlog.py
@@ -1,13 +1,9 @@
-import logging
-
-from pylons import tmpl_context as c, app_globals as g, session, request, config, url
-from pylons.controllers.util import abort, redirect
-
+from pylons import tmpl_context as c, request
+from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model.hg_model import HgModel
 from webhelpers.paginate import Page
-from pylons_app.lib.auth import LoginRequired
+import logging
 
 log = logging.getLogger(__name__)
 
diff --git a/pylons_app/controllers/summary.py b/pylons_app/controllers/summary.py
index 85be97aa..691093a0 100644
--- a/pylons_app/controllers/summary.py
+++ b/pylons_app/controllers/summary.py
@@ -1,7 +1,7 @@
 from pylons import tmpl_context as c, request
 from pylons_app.lib.auth import LoginRequired
-from pylons_app.lib.base import BaseController, render, _full_changelog_cached
-from pylons_app.model.hg_model import HgModel
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import HgModel, _full_changelog_cached
 import logging
 
 log = logging.getLogger(__name__)
@@ -16,7 +16,6 @@ class SummaryController(BaseController):
         hg_model = HgModel()
         c.repo_info = hg_model.get_repo(c.repo_name)
         c.repo_changesets = _full_changelog_cached(c.repo_name)[:10]
-        
         e = request.environ
         uri = u'%(protocol)s://%(user)s@%(host)s/%(repo_name)s' % {
                                         'protocol': e.get('wsgi.url_scheme'),
diff --git a/pylons_app/controllers/tags.py b/pylons_app/controllers/tags.py
index 30f39ee2..79d3f66c 100644
--- a/pylons_app/controllers/tags.py
+++ b/pylons_app/controllers/tags.py
@@ -1,14 +1,10 @@
-import logging
-
-from pylons import tmpl_context as c, app_globals as g, session, request, config, url
-from pylons.controllers.util import abort, redirect
-
+from pylons import tmpl_context as c
+from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model.hg_model import HgModel
-from pylons_app.lib.auth import LoginRequired
-log = logging.getLogger(__name__)
+import logging
 
+log = logging.getLogger(__name__)
 
 class TagsController(BaseController):
     
diff --git a/pylons_app/controllers/users.py b/pylons_app/controllers/users.py
index 91d31d9c..06f166b7 100644
--- a/pylons_app/controllers/users.py
+++ b/pylons_app/controllers/users.py
@@ -1,9 +1,8 @@
 from formencode import htmlfill
-from pylons import request, response, session, tmpl_context as c, url, \
-    app_globals as g
-from pylons.i18n.translation import _
-from pylons_app.lib import helpers as h    
+from pylons import request, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect
+from pylons.i18n.translation import _
+from pylons_app.lib import helpers as h
 from pylons_app.lib.auth import LoginRequired, CheckPermissionAll
 from pylons_app.lib.base import BaseController, render
 from pylons_app.model.db import User, UserLog
@@ -12,8 +11,6 @@ from pylons_app.model.user_model import UserModel
 import formencode
 import logging
 
-
-
 log = logging.getLogger(__name__)
 
 class UsersController(BaseController):
diff --git a/pylons_app/lib/base.py b/pylons_app/lib/base.py
index e3060278..a490f0c4 100644
--- a/pylons_app/lib/base.py
+++ b/pylons_app/lib/base.py
@@ -2,24 +2,15 @@
 
 Provides the BaseController class for subclassing.
 """
-from beaker.cache import cache_region
 from pylons import config, tmpl_context as c, request, session
 from pylons.controllers import WSGIController
 from pylons.templating import render_mako as render
 from pylons_app.lib.auth import LoginRequired, AuthUser
 from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model import meta
-from pylons_app.model.hg_model import HgModel
+from pylons_app.model.hg_model import _get_repos_cached
 from pylons_app import __version__
 
-@cache_region('long_term', 'cached_repo_list')
-def _get_repos_cached():
-    return [rep for rep in HgModel().get_repos()]
-
-@cache_region('long_term', 'full_changelog')
-def _full_changelog_cached(repo_name):
-    return list(reversed(list(HgModel().get_repo(repo_name))))  
-
 class BaseController(WSGIController):
     
     def __before__(self):
diff --git a/pylons_app/lib/utils.py b/pylons_app/lib/utils.py
index 0bd5fce9..51aa1e16 100644
--- a/pylons_app/lib/utils.py
+++ b/pylons_app/lib/utils.py
@@ -107,11 +107,11 @@ def invalidate_cache(name, *args):
     args = tuple(tmp)
     
     if name == 'cached_repo_list':
-        from pylons_app.lib.base import _get_repos_cached
+        from pylons_app.model.hg_model import _get_repos_cached
         region_invalidate(_get_repos_cached, None, *args)
         
     if name == 'full_changelog':
-        from pylons_app.lib.base import _full_changelog_cached
+        from pylons_app.model.hg_model import _full_changelog_cached
         region_invalidate(_full_changelog_cached, None, *args)
         
 from vcs.backends.base import BaseChangeset
@@ -128,3 +128,17 @@ class EmptyChangeset(BaseChangeset):
         """
         return '0' * 12
 
+
+def repo2db_mapper():
+    """
+    put !
+    """
+    pass
+    #scann all dirs for .hgdbid
+    #if some dir doesn't have one generate one.
+    #
+    
+    
+    
+    
+    
diff --git a/pylons_app/model/hg_model.py b/pylons_app/model/hg_model.py
index af8be547..5321543c 100644
--- a/pylons_app/model/hg_model.py
+++ b/pylons_app/model/hg_model.py
@@ -3,65 +3,118 @@
 #
 # Copyright (c) 2010 marcink.  All rights reserved.
 #
-from vcs.exceptions import RepositoryError
 '''
 Created on Apr 9, 2010
 
 @author: marcink
 '''
+
+from beaker.cache import cache_region
+from mercurial import ui
+from mercurial.hgweb.hgwebdir_mod import findrepos
+from pylons import app_globals as g
+from vcs.exceptions import RepositoryError, VCSError
+import logging
 import os
-from pylons import tmpl_context as c, app_globals as g, session, request, config
-from pylons.controllers.util import abort
 import sys
+log = logging.getLogger(__name__)
+
 try:
-    from vcs.backends.hg import get_repositories, MercurialRepository
+    from vcs.backends.hg import MercurialRepository
 except ImportError:
     sys.stderr.write('You have to import vcs module')
     raise Exception('Unable to import vcs')
 
+
+@cache_region('long_term', 'cached_repo_list')
+def _get_repos_cached():
+    """
+    return cached dict with repos
+    """
+    return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui)
+
+@cache_region('long_term', 'full_changelog')
+def _full_changelog_cached(repo_name):
+    log.info('getting full changelog for %s', repo_name)
+    return list(reversed(list(HgModel().get_repo(repo_name))))
+
 class HgModel(object):
     """
     Mercurial Model
     """
 
-
     def __init__(self):
         """
         Constructor
         """
         pass
-
-    def get_repos(self):
-        for mercurial_repo in get_repositories(g.paths[0][0], g.paths[0][1], g.baseui):
+    
+    @staticmethod
+    def repo_scan(repos_prefix, repos_path, baseui):
+        """
+        Listing of repositories in given path. This path should not be a 
+        repository itself. Return a dictionary of repository objects
+        :param repos_path: path to directory it could take syntax with 
+        * or ** for deep recursive displaying repositories
+        """
+        def check_repo_dir(path):
+            """
+            Checks the repository
+            :param path:
+            """
+            repos_path = path.split('/')
+            if repos_path[-1] in ['*', '**']:
+                repos_path = repos_path[:-1]
+            if repos_path[0] != '/':
+                repos_path[0] = '/'
+            if not os.path.isdir(os.path.join(*repos_path)):
+                raise RepositoryError('Not a valid repository in %s' % path[0][1])        
+        if not repos_path.endswith('*'):
+            raise VCSError('You need to specify * or ** at the end of path '
+                            'for recursive scanning')
             
-            if mercurial_repo._get_hidden():
+        check_repo_dir(repos_path)
+        log.info('scanning for repositories in %s', repos_path)
+        repos = findrepos([(repos_prefix, repos_path)])
+        if not isinstance(baseui, ui.ui):
+            baseui = ui.ui()
+    
+        repos_list = {}
+        for name, path in repos:
+            try:
+                repos_list[name] = MercurialRepository(path, baseui=baseui)
+            except OSError:
+                continue
+        return repos_list
+        
+    def get_repos(self):
+        for name, repo in _get_repos_cached().items():
+            if repo._get_hidden():
                 #skip hidden web repository
                 continue
             
-            last_change = mercurial_repo.last_change
+            last_change = repo.last_change
             try:
-                tip = mercurial_repo.get_changeset('tip')
+                tip = repo.get_changeset('tip')
             except RepositoryError:
                 from pylons_app.lib.utils import EmptyChangeset
                 tip = EmptyChangeset()
                 
             tmp_d = {}
-            tmp_d['name'] = mercurial_repo.name
+            tmp_d['name'] = repo.name
             tmp_d['name_sort'] = tmp_d['name'].lower()
-            tmp_d['description'] = mercurial_repo.description
+            tmp_d['description'] = repo.description
             tmp_d['description_sort'] = tmp_d['description']
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = last_change[1] - last_change[0]
             tmp_d['tip'] = tip.raw_id
             tmp_d['tip_sort'] = tip.revision 
             tmp_d['rev'] = tip.revision
-            tmp_d['contact'] = mercurial_repo.contact
+            tmp_d['contact'] = repo.contact
             tmp_d['contact_sort'] = tmp_d['contact']
-            tmp_d['repo_archives'] = list(mercurial_repo._get_archives())
+            tmp_d['repo_archives'] = list(repo._get_archives())
             
             yield tmp_d
 
     def get_repo(self, repo_name):
-        path = g.paths[0][1].replace('*', '')
-        repo = MercurialRepository(os.path.join(path, repo_name), baseui=g.baseui)
-        return repo
+        return _get_repos_cached()[repo_name]
diff --git a/pylons_app/templates/base/base.html b/pylons_app/templates/base/base.html
index 43c02dcd..84a7acb6 100644
--- a/pylons_app/templates/base/base.html
+++ b/pylons_app/templates/base/base.html
@@ -92,8 +92,8 @@ def is_current(selected):
 					<a id="repo_switcher" title="${_('Switch repository')}" href="#">&darr;</a>
 					<div id="switch_repos" style="display:none;position: absolute;width: 150px;height: 25px">
 						<select id="repos_list" size="=10">
-						%for repo in c.cached_repo_list:
-							<option value="${repo['name']}">${repo['name']}</option>
+						%for repo in c.cached_repo_list.values():
+							<option value="${repo.name}">${repo.name}</option>
 						%endfor
 						</select>
 					</div>