diff options
author | Marcin Kuzminski <marcin@python-works.com> | 2010-09-27 02:17:03 +0200 |
---|---|---|
committer | Marcin Kuzminski <marcin@python-works.com> | 2010-09-27 02:17:03 +0200 |
commit | d0b2ac989ea76714d1a76fdcb08d478b597d251d (patch) | |
tree | 6b46812fcebd12204598166cfcf26446b425cc2c | |
parent | c6bd2e0e6a4d332197a820cdd8a4461f7729b3be (diff) |
fixed lock decorator bug which didn't release the lock after func execution and rewrote the pidlock a little with Ask Solem suggestions!
added tredning languages stats
-rw-r--r-- | celeryconfig.py | 2 | ||||
-rw-r--r-- | pylons_app/controllers/summary.py | 17 | ||||
-rw-r--r-- | pylons_app/lib/celerylib/__init__.py | 2 | ||||
-rw-r--r-- | pylons_app/lib/celerylib/tasks.py | 54 | ||||
-rw-r--r-- | pylons_app/lib/pidlock.py | 50 | ||||
-rw-r--r-- | pylons_app/public/css/style.css | 22 | ||||
-rw-r--r-- | pylons_app/templates/summary/summary.html | 57 |
7 files changed, 169 insertions, 35 deletions
diff --git a/celeryconfig.py b/celeryconfig.py index d54a2e80..c417050c 100644 --- a/celeryconfig.py +++ b/celeryconfig.py @@ -36,7 +36,7 @@ BROKER_PASSWORD = "qweqwe" CELERYD_CONCURRENCY = 2 # CELERYD_LOG_FILE = "celeryd.log" CELERYD_LOG_LEVEL = "DEBUG" -CELERYD_MAX_TASKS_PER_CHILD = 1 +CELERYD_MAX_TASKS_PER_CHILD = 3 #Tasks will never be sent to the queue, but executed locally instead. CELERY_ALWAYS_EAGER = False diff --git a/pylons_app/controllers/summary.py b/pylons_app/controllers/summary.py index 2000e173..ebe97c27 100644 --- a/pylons_app/controllers/summary.py +++ b/pylons_app/controllers/summary.py @@ -35,7 +35,7 @@ from datetime import datetime, timedelta from time import mktime import calendar import logging - +import json log = logging.getLogger(__name__) class SummaryController(BaseController): @@ -79,18 +79,25 @@ class SummaryController(BaseController): c.ts_min = ts_min_m c.ts_max = ts_max_y - stats = self.sa.query(Statistics)\ .filter(Statistics.repository == c.repo_info.dbrepo)\ .scalar() - - if stats: + + + if stats and stats.languages: + lang_stats = json.loads(stats.languages) c.commit_data = stats.commit_activity c.overview_data = stats.commit_activity_combined + c.trending_languages = json.dumps(OrderedDict( + sorted(lang_stats.items(), reverse=True, + key=lambda k: k[1])[:2] + ) + ) + print c.trending_languages else: - import json c.commit_data = json.dumps({}) c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ]) + c.trending_languages = json.dumps({}) return render('summary/summary.html') diff --git a/pylons_app/lib/celerylib/__init__.py b/pylons_app/lib/celerylib/__init__.py index 051d3477..68cb8ef8 100644 --- a/pylons_app/lib/celerylib/__init__.py +++ b/pylons_app/lib/celerylib/__init__.py @@ -42,7 +42,7 @@ def locked_task(func): log.info('running task with lockkey %s', lockkey) try: l = DaemonLock(lockkey) - return func(*fargs, **fkwargs) + func(*fargs, **fkwargs) l.release() except LockHeld: log.info('LockHeld') diff --git a/pylons_app/lib/celerylib/tasks.py b/pylons_app/lib/celerylib/tasks.py index 4a4a91fa..fab82b7c 100644 --- a/pylons_app/lib/celerylib/tasks.py +++ b/pylons_app/lib/celerylib/tasks.py @@ -1,16 +1,16 @@ from celery.decorators import task from celery.task.sets import subtask from celeryconfig import PYLONS_CONFIG as config +from operator import itemgetter from pylons.i18n.translation import _ from pylons_app.lib.celerylib import run_task, locked_task from pylons_app.lib.helpers import person from pylons_app.lib.smtp_mailer import SmtpMailer from pylons_app.lib.utils import OrderedDict -from operator import itemgetter -from vcs.backends.hg import MercurialRepository from time import mktime -import traceback +from vcs.backends.hg import MercurialRepository import json +import traceback __all__ = ['whoosh_index', 'get_commits_stats', 'reset_user_password', 'send_email'] @@ -75,10 +75,10 @@ def whoosh_index(repo_location, full_index): @task @locked_task def get_commits_stats(repo_name, ts_min_y, ts_max_y): - author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty - from pylons_app.model.db import Statistics, Repository log = get_commits_stats.get_logger() + author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty + commits_by_day_author_aggregate = {} commits_by_day_aggregate = {} repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') @@ -99,7 +99,7 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y): if cur_stats: last_rev = cur_stats.stat_on_revision - if last_rev == repo.revisions[-1]: + if last_rev == repo.revisions[-1] and len(repo.revisions) > 1: #pass silently without any work return True @@ -109,6 +109,7 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y): cur_stats.commit_activity_combined)) commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity) + log.debug('starting parsing %s', parse_limit) for cnt, rev in enumerate(repo.revisions[last_rev:]): last_cs = cs = repo.get_changeset(rev) k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1], @@ -187,9 +188,16 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y): stats = cur_stats if cur_stats else Statistics() stats.commit_activity = json.dumps(commits_by_day_author_aggregate) stats.commit_activity_combined = json.dumps(overview_data) + + log.debug('last revison %s', last_rev) + leftovers = len(repo.revisions[last_rev:]) + log.debug('revisions to parse %s', leftovers) + + if last_rev == 0 or leftovers < parse_limit: + stats.languages = json.dumps(__get_codes_stats(repo_name)) + stats.repository = dbrepo stats.stat_on_revision = last_cs.revision - stats.languages = json.dumps({'_TOTAL_':0, '':0}) try: sa.add(stats) @@ -198,8 +206,8 @@ def get_commits_stats(repo_name, ts_min_y, ts_max_y): log.error(traceback.format_exc()) sa.rollback() return False - - run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) + if len(repo.revisions) > 1: + run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) return True @@ -259,3 +267,31 @@ def send_email(recipients, subject, body): log.error(traceback.format_exc()) return False return True + +def __get_codes_stats(repo_name): + LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', + 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl', + 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', + 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', + 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', + 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', + 'yaws'] + repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') + repo = MercurialRepository(repos_path + repo_name) + + code_stats = {} + for topnode, dirs, files in repo.walk('/', 'tip'): + for f in files: + k = f.mimetype + if f.extension in LANGUAGES_EXTENSIONS: + if code_stats.has_key(k): + code_stats[k] += 1 + else: + code_stats[k] = 1 + + return code_stats or {} + + + + + diff --git a/pylons_app/lib/pidlock.py b/pylons_app/lib/pidlock.py index 32058e06..e9b429c5 100644 --- a/pylons_app/lib/pidlock.py +++ b/pylons_app/lib/pidlock.py @@ -1,6 +1,8 @@ import os, time import sys from warnings import warn +from multiprocessing.util import Finalize +import errno class LockHeld(Exception):pass @@ -27,55 +29,67 @@ class DaemonLock(object): self.held = False #run the lock automatically ! self.lock() + self._finalize = Finalize(self, DaemonLock._on_finalize, + args=(self, debug), exitpriority=10) - def __del__(self): - if self.held: - -# warn("use lock.release instead of del lock", -# category = DeprecationWarning, -# stacklevel = 2) - - # ensure the lock will be removed - self.release() + @staticmethod + def _on_finalize(lock, debug): + if lock.held: + if debug: + print 'leck held finilazing and running lock.release()' + lock.release() def lock(self): """locking function, if lock is present it will raise LockHeld exception """ lockname = '%s' % (os.getpid()) - + if self.debug: + print 'running lock' self.trylock() self.makelock(lockname, self.pidfile) return True def trylock(self): running_pid = False + if self.debug: + print 'checking for already running process' try: pidfile = open(self.pidfile, "r") pidfile.seek(0) - running_pid = pidfile.readline() + running_pid = int(pidfile.readline()) + + pidfile.close() + if self.debug: print 'lock file present running_pid: %s, checking for execution'\ % running_pid # Now we check the PID from lock file matches to the current # process PID if running_pid: - if os.path.exists("/proc/%s" % running_pid): - print "You already have an instance of the program running" - print "It is running as process %s" % running_pid - raise LockHeld - else: + try: + os.kill(running_pid, 0) + except OSError, exc: + if exc.errno in (errno.ESRCH, errno.EPERM): print "Lock File is there but the program is not running" - print "Removing lock file for the: %s" % running_pid + print "Removing lock file for the: %s" % running_pid self.release() + raise + else: + print "You already have an instance of the program running" + print "It is running as process %s" % running_pid + raise LockHeld() + except IOError, e: if e.errno != 2: raise - def release(self): """releases the pid by removing the pidfile """ + if self.debug: + print 'trying to release the pidlock' + if self.callbackfn: #execute callback function on release if self.debug: diff --git a/pylons_app/public/css/style.css b/pylons_app/public/css/style.css index 70ced7a9..2e1c9860 100644 --- a/pylons_app/public/css/style.css +++ b/pylons_app/public/css/style.css @@ -2963,6 +2963,28 @@ div.form div.fields div.buttons input /* -----------------------------------------------------------
SUMMARY
----------------------------------------------------------- */
+.trending_language_tbl, .trending_language_tbl td {
+ margin: 0px !important;
+ padding: 0px !important;
+ border: 0 !important;
+
+}
+.trending_language{
+ -moz-border-radius-bottomright:4px;
+ -moz-border-radius-topright:4px;
+ border-bottom-right-radius: 4px 4px;
+ border-top-right-radius: 4px 4px;
+ background-color:#336699;
+ color:#FFFFFF;
+ display:block;
+ min-width:20px;
+ max-width:400px;
+ padding:3px;
+ text-decoration:none;
+ height: 10px;
+ margin-bottom: 4px;
+ margin-left: 5px;
+}
#clone_url{
border: none;
diff --git a/pylons_app/templates/summary/summary.html b/pylons_app/templates/summary/summary.html index 863697db..33e49123 100644 --- a/pylons_app/templates/summary/summary.html +++ b/pylons_app/templates/summary/summary.html @@ -90,7 +90,62 @@ E.onDOMReady(function(e){ <input type="text" id="clone_url" readonly="readonly" value="hg clone ${c.clone_repo_url}" size="70"/> </div> </div> - + + <div class="field"> + <div class="label"> + <label>${_('Trending languages')}:</label> + </div> + <div class="input-short"> + <div id="lang_stats"> + + </div> + <script type="text/javascript"> + var data = ${c.trending_languages|n}; + var total = 0; + var no_data = true; + for (k in data){ + total += data[k]; + no_data = false; + } + var tbl = document.createElement('table'); + tbl.setAttribute('class','trending_language_tbl'); + for (k in data){ + var tr = document.createElement('tr'); + var percentage = Math.round((data[k]/total*100),2); + var value = data[k]; + var td1 = document.createElement('td'); + td1.width=150; + var trending_language_label = document.createElement('div'); + trending_language_label.innerHTML = k; + td1.appendChild(trending_language_label); + + var td2 = document.createElement('td'); + var trending_language = document.createElement('div'); + trending_language.title = k; + trending_language.innerHTML = "<b>"+value+" ${_('files')} - "+percentage+" %</b>"; + trending_language.setAttribute("class", 'trending_language'); + trending_language.style.width=percentage+"%"; + td2.appendChild(trending_language); + + tr.appendChild(td1); + tr.appendChild(td2); + tbl.appendChild(tr); + //YAHOO.util.Dom.get('lang_stats').appendChild(trending_language_label); + + } + if(no_data){ + var tr = document.createElement('tr'); + var td1 = document.createElement('td'); + td1.innerHTML = "${_('No data loaded yet...')}"; + tr.appendChild(td1); + tbl.appendChild(tr); + } + YAHOO.util.Dom.get('lang_stats').appendChild(tbl); + </script> + + </div> + </div> + <div class="field"> <div class="label"> <label>${_('Download')}:</label> |