diff options
| -rw-r--r-- | MANIFEST.in | 2 | ||||
| -rw-r--r-- | development.ini | 1 | ||||
| -rw-r--r-- | docs/changelog.rst | 1 | ||||
| -rw-r--r-- | docs/setup.rst | 37 | ||||
| -rw-r--r-- | production.ini | 1 | ||||
| -rw-r--r-- | rhodecode/config/deployment.ini_tmpl | 1 | ||||
| -rw-r--r-- | rhodecode/lib/indexers/__init__.py | 32 | ||||
| -rw-r--r-- | rhodecode/lib/indexers/daemon.py | 10 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | setup.py | 5 |
10 files changed, 57 insertions, 35 deletions
diff --git a/MANIFEST.in b/MANIFEST.in index 2b755842..692f207d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,7 +7,7 @@ recursive-include rhodecode/i18n/ * recursive-include rhodecode/public/css * recursive-include rhodecode/public/images * #js -include rhodecode/public/js/yui2.js +include rhodecode/public/js/yui2a.js include rhodecode/public/js/excanvas.min.js include rhodecode/public/js/yui.flot.js include rhodecode/public/js/graph.js diff --git a/development.ini b/development.ini index ec294cd2..77e3c4f2 100644 --- a/development.ini +++ b/development.ini @@ -43,6 +43,7 @@ full_stack = true static_files = true lang=en cache_dir = %(here)s/data +index_dir = %(here)s/data/index #################################### ### BEAKER CACHE #### diff --git a/docs/changelog.rst b/docs/changelog.rst index 86b544ee..29022fa4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,7 @@ Changelog - introduced new enhanced changelog for merges that shows more accurate results - gui optimizations, fixed application width to 1024px - numerous small bugfixes +- whoosh index moved to paster command 1.0.2 (**2010-11-12**) ---------------------- diff --git a/docs/setup.rst b/docs/setup.rst index 8930b35b..05f9b35d 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -41,21 +41,40 @@ Setting up the application remember to update these if needed. -Setting up Whoosh ------------------ +Setting up Whoosh full text search +---------------------------------- + +Index for whoosh can be build starting from version 1.1 using paster command +passing repo locations to index, as well as Your config file that stores +whoosh index files locations. There is possible to pass `-f` to the options +to enable full index rebuild. Without that indexing will run always in in +incremental mode. + +:: + paster make-index --repo-location=<location for repos> production.ini + +for full index rebuild You can use + +:: + paster make-index -f --repo-location=<location for repos> production.ini - For full text search You can either put crontab entry for +This command can be run even from crontab in order to do periodical +index builds and keep Your index always up to date. An example entry might +look like this + :: - python /var/www/rhodecode/<rhodecode_installation_path>/lib/indexers/daemon.py incremental <put_here_path_to_repos> + /path/to/python/bin/paster --repo-location=<location for repos> /path/to/rhodecode/production.ini -When using incremental mode whoosh will check last modification date of each file -and add it to reindex if newer file is available. Also indexing daemon checks -for removed files and removes them from index. Sometime You might want to rebuild -index from scrach, in admin pannel You can check `build from scratch` flag -and in standalone daemon You can pass `full` instead on incremental to build -remove previos index and build new one. +When using incremental(default) mode whoosh will check last modification date +of each file and add it to reindex if newer file is available. Also indexing +daemon checks for removed files and removes them from index. + +Sometime You might want to rebuild index from scratch. You can do that using +the `-f` flag passed to paster command or, in admin panel You can check +`build from scratch` flag. Nginx virtual host example -------------------------- diff --git a/production.ini b/production.ini index 07bf4059..9b4d1d93 100644 --- a/production.ini +++ b/production.ini @@ -43,6 +43,7 @@ full_stack = true static_files = false lang=en cache_dir = %(here)s/data +index_dir = %(here)s/data/index #################################### ### BEAKER CACHE #### diff --git a/rhodecode/config/deployment.ini_tmpl b/rhodecode/config/deployment.ini_tmpl index eca3a1e3..b900cc47 100644 --- a/rhodecode/config/deployment.ini_tmpl +++ b/rhodecode/config/deployment.ini_tmpl @@ -43,6 +43,7 @@ full_stack = true static_files = true lang=en cache_dir = %(here)s/data +index_dir = %(here)s/data/index app_instance_uuid = ${app_instance_uuid} #################################### diff --git a/rhodecode/lib/indexers/__init__.py b/rhodecode/lib/indexers/__init__.py index 5f960f2b..3822fec5 100644 --- a/rhodecode/lib/indexers/__init__.py +++ b/rhodecode/lib/indexers/__init__.py @@ -59,14 +59,14 @@ class MakeIndex(command.Command): usage = "CONFIG_FILE" summary = "Creates index for full text search given configuration file" - group_name = "Whoosh indexing" - + group_name = "RhodeCode" + takes_config_file = -1 parser = command.Command.standard_parser(verbose=True) -# parser.add_option('--repo-location', -# action='store', -# dest='repo_location', -# help="Specifies repositories location to index", -# ) + parser.add_option('--repo-location', + action='store', + dest='repo_location', + help="Specifies repositories location to index REQUIRED", + ) parser.add_option('-f', action='store_true', dest='full_index', @@ -75,27 +75,23 @@ class MakeIndex(command.Command): default=False) def command(self): config_name = self.args[0] - p = config_name.split('/') - if len(p) == 1: - root = '.' - else: - root = '/'.join(p[:-1]) - print root + root = '.' if len(p) == 1 else '/'.join(p[:-1]) config = ConfigParser.ConfigParser({'here':root}) config.read(config_name) - print dict(config.items('app:main'))['index_dir'] + index_location = dict(config.items('app:main'))['index_dir'] - #return + repo_location = self.options.repo_location - #======================================================================= + #====================================================================== # WHOOSH DAEMON - #======================================================================= + #====================================================================== from rhodecode.lib.pidlock import LockHeld, DaemonLock from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon try: l = DaemonLock() - WhooshIndexingDaemon(index_location=index_location)\ + WhooshIndexingDaemon(index_location=index_location, + repo_location=repo_location)\ .run(full_index=self.options.full_index) l.release() except LockHeld: diff --git a/rhodecode/lib/indexers/daemon.py b/rhodecode/lib/indexers/daemon.py index e0386caa..f0c4b6f8 100644 --- a/rhodecode/lib/indexers/daemon.py +++ b/rhodecode/lib/indexers/daemon.py @@ -78,9 +78,7 @@ class WhooshIndexingDaemon(object): if not repo_location: raise Exception('You have to provide repositories location') - - - self.repo_paths = HgModel.repo_scan('/', self.repo_location, None, True) + self.repo_paths = HgModel().repo_scan(self.repo_location, None, True) self.initial = False if not os.path.isdir(self.index_location): os.mkdir(self.index_location) @@ -89,8 +87,7 @@ class WhooshIndexingDaemon(object): self.initial = True def get_paths(self, repo): - """ - recursive walk in root dir and return a set of all path in that dir + """recursive walk in root dir and return a set of all path in that dir based on repository walk function """ index_paths_ = set() @@ -115,7 +112,8 @@ class WhooshIndexingDaemon(object): return mktime(node.last_changeset.date.timetuple()) def add_doc(self, writer, path, repo): - """Adding doc to writer""" + """Adding doc to writer this function itself fetches data from + the instance of vcs backend""" node = self.get_node(repo, path) #we just index the content of chosen files @@ -1,5 +1,5 @@ [egg_info] -tag_build = rc4 +tag_build = beta tag_svn_revision = true [easy_install] @@ -89,5 +89,10 @@ setup( [paste.app_install] main = pylons.util:PylonsInstaller + + [paste.global_paster_command] + make-index = rhodecode.lib.indexers:MakeIndex + upgrade-db = rhodecode.lib.utils:UpgradeDb + """, ) |