diff options
44 files changed, 1592 insertions, 661 deletions
diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2ea05ac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/.git +**/.venv @@ -2,3 +2,4 @@ *.pyc .idea .venv +linaro_ldap.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e94da84 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM linaro/jenkins-amd64-ubuntu:bionic + +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ + python-dev \ + python-virtualenv \ + virtualenv \ + python-flake8 \ + flake8 \ + libldap2-dev \ + libsasl2-dev \ + libssl-dev + +COPY . patchwork-tools +RUN cd patchwork-tools \ +&& ./unit-test.sh + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..a2735b1 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,17 @@ +FROM ansible/baseimage:20.04 + +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ + python3-apt \ + python3-dev \ + python3-virtualenv \ + virtualenv \ + python3-flake8 \ + flake8 \ + libldap2-dev \ + libsasl2-dev \ + libssl-dev +RUN mkdir -p /srv/patchwork-tools + + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/alt_email.py b/alt_email.py new file mode 100755 index 0000000..5901b1a --- /dev/null +++ b/alt_email.py @@ -0,0 +1,245 @@ +#!/usr/bin/python3 +""" A script to export an identities.yaml file from Patchworks that + attempts to cross-reference with user information from LDAP. + + This script shold be run with the following exports: + export PYTHONPATH=$PYTHONPATH:../project:/srv/linaro-git-tools + export DJANGO_SETTINGS_MODULE=local_settings +""" + +import sys +import os +import time +from jinja2 import Template +import linaro_ldap +from patchwork.models import User +from patchwork.models import Person +import django + +django.setup() + +# hack to make python 2.7 use unicode by default +# since some of our usernames have non-ascii chars +sys.setdefaultencoding('utf8') + +SKIP_PROFILES = [ + 'bugzilla-daemon' +] + +OUTFILE = "/tmp/identities.yaml" + +if os.path.isfile(OUTFILE): + os.unlink(OUTFILE) + +MEMBER_DOMAINS_TABLE = {} + +# TODO: add more domains +KNOWN_DOMAINS_TABLE = { + "debian.org": "Debian", + "gcc.gnu.org": "GNU", + "gnu.org": "GNU", + "lge.com": "LG", + "ibm.com": "IBM", + "il.ibm.com": "IBM", + "in.ibm.com": "IBM", + "linux.vnet.ibm.com": "IBM", + "suse.com": "SuSE", + "linuxfoundation.org": "Linux Foundation", + "arm.com": "ARM", + "freescale.com": "Freescale", + "hp.com": "HP", + "hpe.com": "HP", + "caviumnetworks.com": "Cavium Networks", + "ubuntu.com": "Ubuntu", + "canonical.com": "Canonical", + "stericsson.com": "ST", + "amd.com": "AMD", + "broadcom.com": "Broadcom", + "qti.qualcomm.com": "Qualcomm", + "quicinc.com": "Qualcomm", + "collabora.com": "Collabora", + "collabora.co.uk": "Collabora" +} + + +MERGE_TABLE = {} + + +class NoLdapUserException(Exception): + pass + + +def build_member_table(): + """ Builds a lookup table using email domains as the key to match + email addresses with member organizations. """ + member_ous = linaro_ldap.do_complex_query( + search_filter='(organizationalStatus=*)', + attrlist=['*']) + + for mou in member_ous: + member_name = mou[1]["description"][0] + if "mail" in mou[1]: + for domain in mou[1]["mail"]: + MEMBER_DOMAINS_TABLE[domain] = member_name + + +def get_name(target_email, target_attr='displayName'): + """ Attempts to get the user's real name from LDAP based on email address. + Raises an NoLdapUserException if the user is not found, otherwise + returns the "target_attr" setting from LDAP (defaults to + "displayName").""" + try: + result = linaro_ldap.do_query( + search_attr='mail', + search_pat=target_email, + attrlist=[target_attr]) + if result and target_attr in result[0][1]: + return result[0][1][target_attr][0] + except linaro_ldap.ldap.FILTER_ERROR: + # user entered in a bogus email and used illegel chars that + # caused an LDAP error + pass + + raise NoLdapUserException( + "no user found in LDAP for %s" % target_email) + + +def get_org(target_email): + """ Attempts to determine the organization a user belongs to based + on their email addresses. Look first for a linaro.org address, + then searches through member domains. If nothing is found, + returns 'Unknown' as default. """ + # if they have a l.o address, claim them + if target_email.endswith('@linaro.org'): + return 'Linaro' + + domain = target_email.split('@')[-1] + # if still here, see if we can match a domain to a member + if domain in MEMBER_DOMAINS_TABLE: + return MEMBER_DOMAINS_TABLE[domain] + + # last ditch effort, try to see if it's a domain we recognize + if domain in KNOWN_DOMAINS_TABLE: + return KNOWN_DOMAINS_TABLE[domain] + + return 'Unknown' + + +def merge_entry(uid, target_emails, target_org, target_end_date=None): + """ Add a profile entry for the user or merge with existing entry + to prevent duplicates """ + if uid in MERGE_TABLE: + for e in target_emails: + if e not in MERGE_TABLE[uid]['emails']: + MERGE_TABLE[uid]['emails'].append(e) + # if the new org is Linaro or previous org unknown, override it + if target_org == 'Linaro' or MERGE_TABLE[uid]['org'] == 'Unknown': + MERGE_TABLE[uid]['org'] = target_org + if target_end_date is not None: + MERGE_TABLE[uid]['end_date'] = target_end_date + else: + MERGE_TABLE[uid] = {} + MERGE_TABLE[uid]['emails'] = target_emails + MERGE_TABLE[uid]['org'] = target_org + MERGE_TABLE[uid]['end_date'] = target_end_date + + +TMPL = Template(u'''\ +- profile: + name: {{ username }} + enrollments: + - organization: {{ org }} +{%- if end_date %} + end_date: {{ end_date }} +{%- endif %} + email: +{%- for email in emails %} + - {{ email }} +{%- endfor %} + +''') + + +def write_entry(uid, target_emails, target_org, target_end_date=None): + """ Write a profile entry for the specified user to output """ + try: + + entry_template = TMPL.render( + username=uid, + org=target_org, + end_date=target_end_date, + emails=target_emails + ) + with open(OUTFILE, 'a') as outfile: + outfile.write(entry_template) + except TypeError: + pass + + +build_member_table() + +for user in User.objects.filter(is_active=True): + # skip unwanted profiles + if user.username in SKIP_PROFILES: + continue + + persons = Person.objects.filter(user=user) + # concat all emails from Person objs, but skip '(address hidden)' + emails = [x.email for x in persons if '@' in x.email] + + # some cases we have a linaro Person linked to non-Linaro User.. + # make sure their email is included before we start looking + # for organization membership + if '@' in user.username and user.username not in emails: + emails.append(user.username) + + # don't bother if user just has an account but no email addresses + if not emails: + continue + + end_date = None + org = None + + stop_asking_ldap = False + for email in emails: + # if we've already found them in ldap, no need to keep searching + if stop_asking_ldap: + continue + + # see if this email is in LDAP. If yes, it's an + # an active account (either @linaro.org, member, + # or community account) + try: + name = get_name(email, "displayName") + stop_asking_ldap = True + org = get_org(email) + # if it's still unknown, let's call it "Linaro Community" + if org == "Unknown": + org = "Linaro Community" + except NoLdapUserException: + # not in ldap. No matter what happens next, + # we have to get username from patchworks. + name = user.username + + # If it's a l.o address, then assume user is no longer + # an employee since no LDAP entry. Exit loop. + if email.endswith('@linaro.org'): + org = "Linaro" + end_date = time.strftime('%Y-%m-%d', time.localtime()) + stop_asking_ldap = True + else: + # only bother doing a look up if the + # org hasn't been set yet. This will + # continue through the loop and let us + # pick up @l.o addresses if they occur + # later. + if org is None or org == 'Unknown': + org = get_org(email) + + merge_entry(name, emails, org, end_date) + +for uid in MERGE_TABLE.keys(): + write_entry(uid, + MERGE_TABLE[uid]['emails'], + MERGE_TABLE[uid]['org'], + MERGE_TABLE[uid]['end_date']) @@ -9,7 +9,7 @@ import django def django_setup(): try: import patchwork - except: + except Exception: # try and guess location based on how ansible deploys patchwork here = os.path.abspath(os.path.dirname(__file__)) patchwork = os.path.join(here, '../project') @@ -30,9 +30,9 @@ def _init_logging(level): '%(asctime)s %(levelname)-5s %(name)s: %(message)s', datefmt='%H:%M:%S') handler.setFormatter(formatter) - l = logging.getLogger('') - l.addHandler(handler) - l.setLevel(getattr(logging, level)) + log = logging.getLogger('') + log.addHandler(handler) + log.setLevel(getattr(logging, level)) class LoggingAction(argparse.Action): diff --git a/ensure_projects.py b/ensure_projects.py index cfce618..061a413 100755 --- a/ensure_projects.py +++ b/ensure_projects.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from bin import django_setup from django.conf import settings @@ -9,6 +9,6 @@ django_setup() # must be called to get sys.path and django settings in place for p in settings.DEFAULT_PROJECTS: try: Project.objects.get(linkname=p['linkname']) - except: + except Exception: print('Creating project: %s' % p['linkname']) Project.objects.create(**p) diff --git a/gen_project_json.py b/gen_project_json.py new file mode 100755 index 0000000..8d098c2 --- /dev/null +++ b/gen_project_json.py @@ -0,0 +1,186 @@ +#!/usr/bin/python3 +""" A script to generate a projects.json file from Patchworks. + + Details of the projects.json file can be found at: + https://chaoss.github.io/grimoirelab-tutorial/sirmordred/projects.html + + This script should be run with the following exports: + export PYTHONPATH=$PYTHONPATH:../project:/srv/linaro-git-tools + export DJANGO_SETTINGS_MODULE=local_settings +""" + +import sys +import os +import json +import re +import django +from patchwork.models import Project +from linaro_metrics.models import Team, TeamCredit + +django.setup() + +# hack to make python 2.7 use unicode by default +# since some of our usernames have non-ascii chars +sys.setdefaultencoding('utf8') + +OUTFILE = "/tmp/projects.json" +# a table of Project objects from patchworks +PROJECTS = {} +EXCLUDE_PROJECTS = [ + 'Unknown', + 'No Project', + 'Not upstream' +] +# a table representing the json that will be used +# to create the projects.json file +PW_PROJECT_TABLE = {} + + +def clean_git(git_string): + """ Some of the git URLs in the pw db have trailing junk + that needs to be removed. """ + url = re.sub(';.*$', '', git_string) + # remove cruft from http string + url = re.sub('/commit(.*)?$', '', url) + # make sure doesn't end with a / + url = re.sub('/$', '', url) + + return url + + +def compare_repo(a_url, b_url, depth=2): + """ Attempts to compare 2 URLs to determine if they + are both pointing to the same repo up to <depth> + directories. Returns True if there's a match, + False if repos appear to be distinct. """ + a_base = get_base_url(a_url) + b_base = get_base_url(b_url) + a = a_base.split('/') + b = b_base.split('/') + + a.reverse() + b.reverse() + + a_server = a.pop() + b_server = b.pop() + + # server's don't match + if a_server != b_server: + return False + + matches = 0 + if len(a) < len(b): + limit = len(a) + else: + limit = len(b) + + # step through reversed paths until we + # either find a mistmatch, run out of + # path fields for one of the repos, or + # reach our depth limit + while matches < limit and matches < depth: + if a[matches] != b[matches]: + return False + matches += 1 + + return True + + +def get_base_url(url): + """ returns a URL with the protocol and any trailing ".git" + stripped off. This is meant to create an abstract + URL that can be used to compare a git:// and http:// + url to see if they refer to the same repository """ + noproto = re.sub('^.*://', '', url) + noproto = re.sub('.git$', '', noproto) + return noproto + + +def load_projects(): + """ Create a table of active projects and any git or + pipermail URLs associated with it. These will + later be aggregated into a team's "project" entry + in the projects.json file. """ + projs = {} + + for proj in Project.objects.all(): + if proj.name in EXCLUDE_PROJECTS: + continue + + projs[proj.name] = {} + + projs[proj.name]['pipermail'] = [] + projs[proj.name]['git'] = [] + projs[proj.name]['github'] = [] + + # Project can have either a git:// or http[s]:// + # repo url, so check both scm and webscm. Also + # github links need to be separated out to their + # own list. + for url in [proj.scm_url, proj.webscm_url]: + if url not in [None, '', 'n/a']: + clean_url = clean_git(url) + if 'github' in clean_url: + target = 'github' + else: + target = 'git' + + projs[proj.name][target].append(clean_url) + + # hack.. assume if hostname is "lists" that it's using mailman + # and try to guess pipermail URL + if '@lists' in proj.listemail: + (mlist, host) = proj.listemail.split('@') + pipermail = 'https://{0}/pipermail/{1}'.format(host, mlist) + projs[proj.name]['pipermail'].append(pipermail) + + return projs + + +# start of main program +PROJECTS = load_projects() + +if os.path.isfile(OUTFILE): + os.unlink(OUTFILE) + +# iterate through each team in the PW db, and get a list +# of the TeamCredit objects for the team. From there, +# we can use the TeamCredit object to look up which Project +# was contributed to, and then add that PW Project's information +# to the GL team "project" entry. +for t in Team.objects.filter(active=True): + team_name = t.display_name + teamcredits = TeamCredit.objects.filter(team=t) + projects = [] + + for c in teamcredits: + proj_name = c.patch.project.name + + if proj_name not in projects and proj_name in PROJECTS: + projects.append(proj_name) + + PW_PROJECT_TABLE[team_name] = {} + PW_PROJECT_TABLE[team_name]['git'] = [] + PW_PROJECT_TABLE[team_name]['github'] = [] + PW_PROJECT_TABLE[team_name]['pipermail'] = [] + + for p in projects: + for t in ['git', 'github']: + for candidate_url in list(PROJECTS[p][t]): + git_matches = [compare_repo(candidate_url, x) + for x in list(PW_PROJECT_TABLE[team_name][t])] + if True not in git_matches: + PW_PROJECT_TABLE[team_name][t].append(candidate_url) + PW_PROJECT_TABLE[team_name]['pipermail'] += PROJECTS[p]['pipermail'] + +# remove empty entries +for t in PW_PROJECT_TABLE: + if not PW_PROJECT_TABLE[t]['git']: + del PW_PROJECT_TABLE[t]['git'] + if not PW_PROJECT_TABLE[t]['github']: + del PW_PROJECT_TABLE[t]['github'] + if not PW_PROJECT_TABLE[t]['pipermail']: + del PW_PROJECT_TABLE[t]['pipermail'] + +with open(OUTFILE, "w") as outfile: + json.dump(PW_PROJECT_TABLE, outfile, indent=4) diff --git a/generate_mbox.py b/generate_mbox.py new file mode 100755 index 0000000..59b9bbc --- /dev/null +++ b/generate_mbox.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 +""" A script to generate a projects.json file from Patchworks. + + This script should be run with the following exports: + export PYTHONPATH=$PYTHONPATH:../project:/srv/linaro-git-tools + export DJANGO_SETTINGS_MODULE=local_settings +""" + +import django +import subprocess +from patchwork.models import Project + +django.setup() + +for proj in Project.objects.all(): + if ( + "kernel.org" in proj.listemail + or "alsa-devel" in proj.name + or "qemu-devel" in proj.name + ): + subprocess.call( + [ + "/srv/patches.linaro.org/tools/import_mbox.py", + "--mbox_repo", + proj.linkname, + "--log", + "INFO", + "--days", + "2", + ] + ) @@ -9,41 +9,42 @@ import dulwich.repo from django.conf import settings from patchwork.parser import parse_patch -log = logging.getLogger('gitrepo') +log = logging.getLogger("gitrepo") -def croncmd(args, cwd='./', timeout=None, get_fail_logger=None): +def croncmd(args, cwd="./", timeout=None, get_fail_logger=None): if timeout: - args = ['timeout', str(timeout)] + args + args = ["timeout", str(timeout)] + args with tempfile.SpooledTemporaryFile(max_size=4096) as f: try: subprocess.check_call(args, cwd=cwd, stdout=f, stderr=f) if log.level == logging.DEBUG: - log.debug('Results of cmd(%s): %r', cwd, args) + log.debug("Results of cmd(%s): %r", cwd, args) f.seek() - log.debug('COMBINED_OUTPUT\n%s', f.read()) + log.debug("COMBINED_OUTPUT\n%s", f.read()) except subprocess.CalledProcessError as e: logger = log.error if get_fail_logger: logger = get_fail_logger() - logger('Unable to run command(%s): %r', cwd, args) + logger("Unable to run command(%s): %r", cwd, args) if timeout and e.returncode == 124: - logger('Command timed out') + logger("Command timed out") f.seek(0) if e.output: - logger('STDOUT:\n%s', e.output) - logger('STDERR:\n%s', f.read()) + logger("STDOUT:\n%s", e.output) + logger("STDERR:\n%s", f.read()) else: - logger('COMBINED OUTPUT:\n%s', f.read()) + logger("COMBINED OUTPUT:\n%s", f.read()) raise class Repo(object): - '''Our patchwork deployments try and automatically update patches by + """Our patchwork deployments try and automatically update patches by looking at the change history on a repository. This class provides a simple interface to analyze new commits - ''' + """ + def __init__(self, repo_dir, name, scm_url): self.path = os.path.join(repo_dir, name) self.scm_url = scm_url @@ -59,24 +60,27 @@ class Repo(object): return self.repo[key] def _clone(self): - croncmd(['git', 'clone', '--mirror', self.scm_url, self.path]) + croncmd(["git", "clone", "--mirror", self.scm_url, self.path]) def _pull(self): - fail_file = os.path.join(self.path, 'failures') + fail_file = os.path.join(self.path, "failures") def get_fail_logger(): - with open(fail_file, 'a+') as f: - f.write('failed at: %s\n' % str(datetime.datetime.now())) + with open(fail_file, "a+") as f: + f.write("failed at: %s\n" % str(datetime.datetime.now())) f.seek(0) for count, line in enumerate(f, 1): if count > 3: return log.error return log.info - timeout = str(getattr(settings, 'REPO_TIMEOUT', 120)) + timeout = str(getattr(settings, "REPO_TIMEOUT", 120)) croncmd( - ['git', 'remote', '-v', 'update', '--prune'], self.path, timeout, - get_fail_logger) + ["git", "remote", "-v", "update", "--prune"], + self.path, + timeout, + get_fail_logger, + ) if os.path.exists(fail_file): # clean out old failures, now that we've succeeded os.unlink(fail_file) @@ -94,47 +98,49 @@ class Repo(object): pass def process_unchecked_commits(self, save_state=True): - last_commit = os.path.join(self.path, 'patchwork-last-commit') + last_commit = os.path.join(self.path, "patchwork-last-commit") if os.path.exists(last_commit): - with open(last_commit) as f: + with open(last_commit, "r") as f: start_at = f.read().strip() else: - start_at = 'HEAD~100' + start_at = "HEAD~100" - log.debug('looking for commits since: %s', start_at) - args = ['git', 'rev-list', '--reverse', start_at + '..HEAD'] - with open('/dev/null', 'w') as f: + log.debug("looking for commits since: %s", start_at) + args = ["git", "rev-list", "--reverse", start_at + "..HEAD"] + with open("/dev/null", "w") as f: rc = subprocess.call( - ['git', 'show', start_at], cwd=self.path, stdout=f, stderr=f) + ["git", "show", start_at], cwd=self.path, stdout=f, stderr=f + ) if rc != 0: # we may have had a branch who's history was re-written # just try and get changes for past day - args = ['git', 'rev-list', '--reverse', - '--since', '1 day ago', 'HEAD'] + args = ["git", "rev-list", "--reverse", "--since", + "1 day ago", "HEAD"] try: - for x in subprocess.check_output(args, cwd=self.path).split('\n'): + for x in subprocess.check_output( + args, cwd=self.path, text=True).split("\n"): if x: - yield self.repo[x] + yield self.repo[x.encode("utf-8")] start_at = x finally: if save_state: - with open(last_commit, 'w') as f: + with open(last_commit, "w") as f: f.write(start_at) def get_patch(self, commit): - args = ['git', 'show', '--format=format:%e', '-M', str(commit.id)] + args = ["git", "show", "--format=format:%e", "-M", commit.id.decode()] patch = subprocess.check_output(args, cwd=self.path) # the patchwork parser code operates character by character so we must # convert to unicode so it can be handled properly - patch = patch.decode('utf-8', errors='replace') + patch = patch.decode("utf-8", errors="replace") # Don't try and process >5Mb patches, they flood the server if len(patch) > 5000000: - raise MemoryError('patch too large to process: %d' % len(patch)) + raise MemoryError("patch too large to process: %d" % len(patch)) patch = parse_patch(patch)[0] if patch is None: # happens for binary only patches like: # https://git.linaro.org/uefi/OpenPlatformPkg.git/commit/ \ # ?id=7ab4bb34b2464a2491868264bdf2931f2acd6452 - patch = '' + patch = "" return patch diff --git a/import_emails.py b/import_emails.py index 15338d9..abf8b09 100755 --- a/import_emails.py +++ b/import_emails.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from bin import django_setup, add_logging_arguments @@ -53,7 +53,7 @@ def move_message(mail, uid, folder): # put in proper folder and move out of inbox status, _ = mail.uid('copy', uid, folder) assert status == 'OK' - status, _ = mail.uid('store', uid, '+FLAGS', '(\Deleted)') + status, _ = mail.uid('store', uid, '+FLAGS', r'(\Deleted)') assert status == 'OK' @@ -76,7 +76,7 @@ def process_inbox(mail, max_messages=0): added += 1 move_message(mail, uid, 'processed') processed += 1 - except: + except Exception: log.exception('Unknown error for %s, moving to retry folder', uid) move_message(mail, uid, 'retry') break diff --git a/import_mbox.py b/import_mbox.py new file mode 100755 index 0000000..25a7488 --- /dev/null +++ b/import_mbox.py @@ -0,0 +1,187 @@ +#!/usr/bin/python3 + +from bin import django_setup, add_logging_arguments + +import logging +import imaplib +import mailbox +import git +import datetime +import dateutil.parser +import re +import requests + +django_setup() # must be called to get sys.path and django settings in place + +from django.db import IntegrityError +from django.conf import settings +from django.utils.module_loading import import_string + +from patchwork import parser +from patchwork.models import Patch, State + +import patch_matcher + +log = logging.getLogger("import_mbox") + + +def x_days_ago(days): + start_date = datetime.datetime.now() - datetime.timedelta(days=days) + return start_date.strftime("%Y-%m-%d") + + +def get_commits_from_author(repo, start_date, end_date): + start_date, end_date = [ + dateutil.parser.parse(d).date() for d in (start_date, end_date) + ] + return [ + commit + for commit in repo.iter_commits("master") + if start_date <= commit.committed_datetime.date() <= end_date + ] + + +def find_old_revisions(patch): + log.debug("looking for old versions of patch %d", patch.id) + it = patch_matcher.get_patches_matching( + patch.project, [patch.submitter], patch.name, patch.diff + ) + for p in it: + # skip ourself + if p.id != patch.id: + yield p + + +def scan_for_latest_repo(url, targ): + resp = requests.get('/'.join([url, targ])) + last_git = None + if resp.status_code >= 200 and resp.status_code < 300: + for line in resp.text.split('\n'): + if re.match(r'\s+git clone.+%s/%s/.+' % (url, targ), str(line)): + # isolate the url and set it to be last one seen + fields = line.split() + for x in fields: + if x.startswith('http'): + last_git = x + + return last_git + + +def process_mbox_repo(mailing_list, start_date, end_date, days): + log.info("processing mailbox %s", mailing_list) + + url = scan_for_latest_repo("https://lore.kernel.org", mailing_list) + to_path = "/srv/mailinglists/%s" % mailing_list + repo = None + try: + repo = git.Repo(to_path) + log.info("Using exisiting git repo %s", to_path) + except git.exc.GitError as ex: + print("exception", ex) + + if repo is None: + # if none must be empty + log.info("Clone repository from {}".format(url)) + repo = git.Repo.clone_from(url, to_path) + else: + repo.git.reset("--hard") + repo.git.clean("-xdf") + repo.git.checkout("master") + repo.git.pull() + if days: + start_date = x_days_ago(int(days)) + commits = list(get_commits_from_author(repo, start_date, end_date)) + for commit in commits: + try: + repo.git.checkout(commit.hexsha) + log.info( + "processing commit %s, %s, %s", + commit, + commit.message, + commit.committed_datetime, + ) + process_mbox(repo._working_tree_dir + "/m") + except git.exc.GitCommandError: + repo.git.checkout("-f") + + +def line_prepender(filename, line): + with open(filename, "r+") as f: + content = f.read() + f.seek(0, 0) + f.write(line.rstrip("\r\n") + "\n" + content) + + +def process_mbox(mbox): + line_prepender(mbox, "From mboxrd@z Thu Jan 1 00:00:00 1970") + mbox = mailbox.mbox(mbox) + log.info(mbox) + for message in mbox: + if message["From"] is None: + # some reason messages from lore have a none type in from field + break + log.info("processing mailbox %s", message["subject"]) + p = None + try: + p = parser.parse_mail(message) + except IntegrityError as e: + log.info(e) + if p: + log.info("saved mail: %d", p.id) + for patch in Patch.objects.filter(msgid=p.msgid): + for old in find_old_revisions(patch): + log.info( + "marking patch %d as superseded by %d", + old.id, + patch.id, + ) + old.state = State.objects.get(name="Superseded") + old.save() + return p + + +def get_monkey_patcher(): + p = getattr(settings, "PARSEMAIL_MONKEY_PATCHER", None) + if p: + return import_string(p) + + +if __name__ == "__main__": + import argparse + + arg_parser = argparse.ArgumentParser( + description="Check configured inbox for new patches to import" + ) + arg_parser.add_argument( + "--start_date", + default="2020-01-01", + help="""Start date of emails to analyze. + default=%(default)d""", + ) + arg_parser.add_argument( + "--end_date", + default=datetime.datetime.now().strftime("%Y-%m-%d"), + help="""End date of emails to analyze. + default=%(default)d""", + ) + arg_parser.add_argument( + "--days", help="""Number of days instead of start/end date""" + ) + arg_parser.add_argument("--mbox") + arg_parser.add_argument("--mbox_repo") + add_logging_arguments(arg_parser) + args = arg_parser.parse_args() + parser.logger = log + + mail = imaplib.IMAP4_SSL(settings.IMAP_SERVER) + status, _ = mail.login(settings.IMAP_USER, settings.IMAP_PASS) + assert status == "OK" + + if args.mbox_repo: + process_mbox_repo( + args.mbox_repo, args.start_date, args.end_date, args.days + ) + + if args.mbox: + with open(args.mbox) as file: + process_mbox(args.mbox) diff --git a/linaro_metrics/backends.py b/linaro_metrics/backends.py deleted file mode 100644 index 12b168c..0000000 --- a/linaro_metrics/backends.py +++ /dev/null @@ -1,11 +0,0 @@ -from crowdrest.backend import CrowdRestBackend - - -class LowerCaseCrowdBackend(CrowdRestBackend): - def create_or_update_user(self, user_id): - return super(LowerCaseCrowdBackend, self).create_or_update_user( - user_id.lower()) - - def authenticate(self, username=None, password=None): - return super(LowerCaseCrowdBackend, self).authenticate( - username.lower(), password) diff --git a/linaro_metrics/cli.py b/linaro_metrics/cli.py new file mode 100755 index 0000000..fba5d8c --- /dev/null +++ b/linaro_metrics/cli.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 + +# Python fragment that can +# be used to load up all the +# required django infrastructure +# to get a patchwork+linaro_tools +# aware python cli +# +# from cli import * +# import linaro_metrics.teamcredit +# +# tcs = TeamCredit.objects.filter(....) + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from bin import django_setup, add_logging_arguments +django_setup() + +from django.conf import settings diff --git a/linaro_metrics/crowd.py b/linaro_metrics/crowd.py deleted file mode 100644 index 1aab51b..0000000 --- a/linaro_metrics/crowd.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright (C) 2013 Linaro -# -# Author: Milo Casagrande <milo.casagrande@linaro.org> -# This file is part of the Patchmetrics package. -# -# Patchmetrics is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Patchmetrics is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Patchwork; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import base64 -import contextlib -import fcntl -import httplib -import json -import time -import urllib - - -class CrowdException(Exception): - """Base class for Crowd exceptions.""" - - -class CrowdNotFoundException(CrowdException): - """An exception for 404 status.""" - - -class CrowdForbiddenException(CrowdException): - """An exception for 403 status.""" - - -class CrowdUnauthorizedException(CrowdException): - """ An exception for 401 status.""" - - -class Crowd(object): - """A Crowd object used to perform query operations.""" - - def __init__(self, usr, pwd, url): - self._cache = None - self._usr = usr - self._pwd = pwd - assert url.startswith('https://') - _, _, self._host, self._uri = url.split('/', 3) - if ":" in self._host: - self._host, self._port = self._host.split(':') - else: - self._port = 443 - - self._auth = base64.encodestring( - '%s:%s' % (self._usr, self._pwd)).strip() - self._headers = { - 'Authorization': 'Basic ' + self._auth, - 'Accept': 'application/json' - } - - @contextlib.contextmanager - def cached(self, cache_file): - '''provide a cached version of the api to speed things up''' - with open(cache_file, 'a+') as f: - try: - fcntl.lockf(f, fcntl.LOCK_EX) - f.seek(0) - try: - self._cache = json.load(f) - except: - self._cache = {} # in case things are corrupted - - yield - self._cache_clean() - - f.truncate(0) - f.seek(0) - json.dump(self._cache, f) - finally: - fcntl.lockf(f, fcntl.LOCK_UN) - self._cache = None - - def _cached(self, email): - if self._cache is None: - return None - user = self._cache.get(email) - if user and user['expires'] < time.time(): - return None - return user - - def _cache_user(self, email, valid): - if self._cache is None: - return None - self._cache[email] = { - 'email': email, - 'valid': valid, - 'expires': time.time() + 60 * 60 * 24 * 7 # one week - } - - def _cache_clean(self): - # remove stale entries so file doesn't grow out of hand - now = time.time() - self._cache = {k: v for (k, v) in self._cache.iteritems() - if v['expires'] > now} - - def get_user_no_cache(self, email): - params = {'username': email.encode('utf-8')} - resource = '/user?{0}'.format(urllib.urlencode(params)) - try: - resp = json.loads(self._get_rest_usermanagement(resource)) - except CrowdNotFoundException: - resp = None - return resp - - def user_valid(self, email): - user = self._cached(email) - if user: - return user['valid'] - valid = self.get_user_no_cache(email) is not None - self._cache_user(email, valid) - return valid - - def get_group(self, grp): - resource = '/group/user/nested?' + urllib.urlencode({'groupname': grp}) - users = json.loads(self._get_rest_usermanagement(resource))['users'] - return [x['name'] for x in users] - - def _get_rest_usermanagement(self, resource): - api_url = "/{0}{1}".format(self._uri, resource) - return self._get(api_url) - - def _get(self, api_url): - connection = httplib.HTTPSConnection(self._host, self._port) - connection.request("GET", api_url, headers=self._headers) - resp = connection.getresponse() - - if resp.status == 200: - return resp.read() - elif resp.status == 404: - raise CrowdNotFoundException('Resource not found') - elif resp.status == 401: - raise CrowdUnauthorizedException( - 'Authorization not granted to fulfill the request') - elif resp.status == 403: - raise CrowdForbiddenException( - 'Access forbidden to fulfill the request') - else: - raise CrowdException( - 'Unknown Crowd status {0}'.format(resp.status)) diff --git a/linaro_metrics/migrate_patchmetrics.py b/linaro_metrics/migrate_patchmetrics.py index 2598b2f..cb7d95f 100755 --- a/linaro_metrics/migrate_patchmetrics.py +++ b/linaro_metrics/migrate_patchmetrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import argparse import logging @@ -39,7 +39,7 @@ def table_iterator(cursor, table): old_cursor.execute('SELECT * from %s' % table) for row in old_cursor: yield row - except: + except Exception: if row: print('failed on row: %s' % row) raise diff --git a/linaro_metrics/parsemail.py b/linaro_metrics/parsemail.py index 5c5cd43..18c9065 100644 --- a/linaro_metrics/parsemail.py +++ b/linaro_metrics/parsemail.py @@ -9,8 +9,8 @@ from django.conf import settings from patchwork.models import Patch, Person, Project from patchwork.parser import parse_patch, subject_check -from linaro_metrics.crowd import Crowd -from linaro_metrics.sync_teams import get_or_create_person +from builtins import str +unicode = str log = logging.getLogger('import_emails') @@ -97,21 +97,17 @@ def find_author_submitter(mail, comment_lines): return auth, submitter -def get_linaro_person(crowd, email): - # the author is linaro - if crowd.user_valid(email): - # we need to make sure the "user" exists so that we can apply team - # credits in _patch_saved_callback - return get_or_create_person(crowd, email) +def get_linaro_person(email): + p = Person.objects.filter(email=email) + if p.count() > 0: + pers = p.first() + if pers.user and pers.user.is_active is True: + return pers - # the author has linked a non-linaro email to their User - p = Person.objects.filter(email__iexact=email) - if p.count() == 1 and p[0].user: - if crowd.user_valid(p[0].user.email): - return p[0] + return None -def linaro_parse_mail(crowd, real_parse_mail, mail): +def linaro_parse_mail(real_parse_mail, mail): '''Only track patches authored or submitted by "linaro" people. We determine this by first finding the author's email. Most of the time this is the simply the "from" address. Occasionally the "from" winds up @@ -119,7 +115,7 @@ def linaro_parse_mail(crowd, real_parse_mail, mail): content. Once we have the author email address we check the following logic: - * is this author or submitter a valid crowd email? + * is this author or submitter a User in patchwork? * is there a Person in patchwork by this email address? - if so, is its User part of Linaro? (ie allow tracking of non-linaro emails @@ -136,7 +132,8 @@ def linaro_parse_mail(crowd, real_parse_mail, mail): comment_lines, patch = find_comment_and_patch(mail) author, submitter = find_author_submitter(mail, comment_lines) - person = get_linaro_person(crowd, author) + + person = get_linaro_person(author) if person: # we have a linaro authored patch @@ -148,29 +145,26 @@ def linaro_parse_mail(crowd, real_parse_mail, mail): else: # see if its a linaro submitter, we'll add the patch but not give # out team credits - submitter = get_linaro_person(crowd, submitter) - if submitter: + sub = get_linaro_person(submitter) + if sub: Patch.linaro_author = person return real_parse_mail(mail) - return 0 @contextlib.contextmanager def monkey_patcher(parser): - crwd = Crowd(settings.CROWD_USER, settings.CROWD_PASS, settings.CROWD_URL) def_project, _ = Project.objects.get_or_create( linkname=settings.DEFAULT_PROJECT) - with crwd.cached(settings.CROWD_CACHE): - orig_find = parser.find_project_by_header - orig_parse = parser.parse_mail - try: - parser.find_project_by_header = functools.partial( - linaro_find_project, orig_find, def_project) - parser.parse_mail = functools.partial( - linaro_parse_mail, crwd, orig_parse) - yield - finally: - parser.parse_mail = orig_parse - parser.find_project_by_header = orig_find + orig_find = parser.find_project_by_header + orig_parse = parser.parse_mail + try: + parser.find_project_by_header = functools.partial( + linaro_find_project, orig_find, def_project) + parser.parse_mail = functools.partial( + linaro_parse_mail, orig_parse) + yield + finally: + parser.parse_mail = orig_parse + parser.find_project_by_header = orig_find diff --git a/linaro_metrics/settings.py b/linaro_metrics/settings.py index ee70ad5..b0bf450 100644 --- a/linaro_metrics/settings.py +++ b/linaro_metrics/settings.py @@ -28,11 +28,6 @@ SECRET_KEY = '00000000000000000000000000000000000000000000000000' DEBUG = True TEMPLATE_DEBUG = True -CROWD_USER = None -CROWD_PASS = None -CROWD_URL = None -CROWD_CACHE = '/tmp/crowd_cache.json' - INSTALLED_APPS.append('linaro_metrics') # NOQA F405 DEFAULT_TEAM = 'no-team' DEFAULT_PROJECT = 'no-project' diff --git a/linaro_metrics/sync_gerrit_changes.py b/linaro_metrics/sync_gerrit_changes.py index 6e993b5..f3fbd9f 100755 --- a/linaro_metrics/sync_gerrit_changes.py +++ b/linaro_metrics/sync_gerrit_changes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 import os import sys @@ -10,7 +10,8 @@ import json import logging import textwrap import urllib -import urllib2 +import urllib.request +import urllib.parse from datetime import datetime @@ -24,6 +25,7 @@ log = logging.getLogger('sync_gerrit_changes') GERRIT_PROJECTS = { 'AOSP': 'https://android-review.googlesource.com', + 'openocd': 'http://openocd.zylin.com', 'OpenStack': 'https://review.openstack.org', 'OpenDaylight': 'https://git.opendaylight.org/gerrit', 'westeros': 'https://code.rdkcentral.com/r', @@ -37,10 +39,10 @@ def get_user_changes(email, url_base, offset=0): 'q': 'owner:' + email, 'start': offset, } - url = url_base + '/changes/?' + urllib.urlencode(params) + url = url_base + '/changes/?' + urllib.parse.urlencode(params) log.debug('doing http get on: %s', url) try: - resp = urllib2.urlopen(url) + resp = urllib.request.urlopen(url) resp = resp.read() assert resp.startswith(")]}'") entries = json.loads(resp[4:]) @@ -50,7 +52,7 @@ def get_user_changes(email, url_base, offset=0): if e and e.get('_more_changes'): for e in get_user_changes(email, url_base, offset + len(entries)): yield e - except urllib2.HTTPError as e: + except urllib.request.HTTPError as e: if e.code != 400: log.exception('Unable to GET: %s', url) sys.exit(1) @@ -88,7 +90,7 @@ def create_or_update(url_base, project, email, change): log.warn('non-linaro user should be removed: %s', email) return changed if updated > tcs[0].last_state_change: - for k, v in fields.iteritems(): + for k, v in fields.items(): setattr(p, k, v) changed = True p.save() diff --git a/linaro_metrics/sync_github_changes.py b/linaro_metrics/sync_github_changes.py index e37f55e..98f9ca1 100755 --- a/linaro_metrics/sync_github_changes.py +++ b/linaro_metrics/sync_github_changes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 import contextlib import json @@ -8,7 +8,7 @@ import re import sys import textwrap import time -import urllib2 +from urllib.request import urlopen, HTTPError, Request from datetime import datetime @@ -20,7 +20,6 @@ from django.conf import settings from patchwork.models import Patch, Project, State -from linaro_metrics.crowd import Crowd from linaro_metrics.models import TeamCredit from linaro_metrics.parsemail import get_linaro_person from linaro_metrics import team_project_credit @@ -30,23 +29,70 @@ log = logging.getLogger('sync_github_changes') STATE_MAP = {'closed': 'Accepted', 'open': 'New'} GITHUB_REPOS = [ # tuple of: owner, repo, patchwork-project + ('96boards', 'meta-96boards', 'meta-96boards'), + ('96boards', 'meta-rpb', 'meta-rpb'), + ('apache', 'arrow', 'arrow'), + ('apache', 'ambari', 'ambari'), + ('apache', 'hbase', 'hbase'), + ('apache', 'hadoop', 'hadoop'), + ('apache', 'tvm', 'tvm'), + ('apache', 'bigtop', 'bigtop'), ('ARM-software', 'arm-trusted-firmware', 'arm-trusted-firmware'), + ('ARM-software', 'optimized-routines', 'optimized-routines'), + ('ceph', 'ceph-ansible', 'ceph-ansible'), + ('ceph', 'ceph-isci', 'ceph-isci'), + ('ceph', 'ceph-cm-ansible', 'ceph-cm-ansible'), + ('ceph', 'teuthology', 'teuthology'), + ('flame', 'blis', 'blis'), ('jackmitch', 'libsoc', 'libsoc'), ('OP-TEE', 'optee_os', 'optee_os'), ('OP-TEE', 'optee_test', 'optee_test'), ('OP-TEE', 'optee_client', 'optee_client'), ('OP-TEE', 'build', 'optee_build'), ('OP-TEE', 'manifest', 'optee_manifest'), + ('openhpc', 'ohpc', 'ohpc'), + ('kernelci', 'kernelci-admin', 'kernelci-admin'), + ('kernelci', 'kernelci-build', 'kernelci-build'), + ('kernelci', 'kernelci-backend', 'kernelci-backend'), + ('kernelci', 'kernelci-backend-config', 'kernelci-backend-config'), + ('kernelci', 'kernelci-core', 'kernelci-core'), + ('kernelci', 'kernelci-frontend', 'kernelci-frontend'), + ('kernelci', 'lava-ci', 'lava-ci'), + ('libhugetlbfs', 'libhugetlbfs', 'libhugetlbfs'), + ('Linaro', 'ansible-playbook-for-ohpc', 'ansible-playbook-for-ohpc'), + ('Linaro', 'benchmark_harness', 'benchmark_harness'), + ('Linaro', 'hpc_lab_setup', 'hpc_lab_setup'), + ('Linaro', 'mr-provisioner-client', 'mr-provisioner-client'), + ('Linaro', 'squad', 'squad'), + ('Linaro', 'skipgen', 'skipgen'), + ('Linaro', 'lkft-tools', 'lkft-tools'), + ('Linaro', 'qa-reports-known-issues', 'qa-reports-known-issues'), + ('llvm', 'llvm-project', 'llvm-project'), ('linaro-swg', 'optee_android_manifest', 'optee_android_manifest'), ('linaro-swg', 'optee_benchmark', 'optee_benchmark'), ('linaro-swg', 'linux', 'optee_linux'), ('linaro-swg', 'gen_rootfs', 'optee_gen_rootfs'), ('linaro-swg', 'bios_qemu_tz_arm', 'optee_bios_qemu_tz_arm'), ('linaro-swg', 'hello_world', 'optee_hello_world'), + ('linux-test-project', 'ltp', 'ltp'), + ('longhorn', 'longhorn', 'longhorn'), + ('mlcommons', 'inference', 'MLCommons-inference'), + ('mr-provisioner', 'mr-provisioner', 'mr-provisioner'), + ('openebs', 'mayastor', 'openebs-mayastor'), + ('pmem', 'pmdk', 'pmem-pmdk'), + ('pmem', 'rpma', 'pmem-rpma'), + ('pmem', 'ndctl', 'pmem-ndctl'), + ('pytorch', 'pytorch', 'pytorch'), + ('rook', 'rook', 'rook'), + ('rust-vmm', 'vhost-device', 'vhost-device'), ('scheduler-tools', 'rt-app', 'rt-app'), + ('tensorflow', 'tensorflow', 'TensorFlow'), + ('tom-gall', 'utvm-examples', 'utvm-examples'), + ('tlc-pack', 'tophob', 'tophub'), ('WebPlatformForEmbedded', 'meta-wpe', 'meta-wpe'), ('WebPlatformForEmbedded', 'WPEWebKit', 'WPEWebKit'), ('ndechesne', 'meta-qcom', 'meta-qcom'), + ('xianyi', 'OpenBLAS', 'OpenBLAS'), ('zephyrproject-rtos', 'zephyr', 'Zephyr'), ] @@ -63,13 +109,19 @@ class Commit(object): def _get(url): headers = {'Authorization': 'token %s' % settings.GITHUB_OAUTH_TOKEN} - request = urllib2.Request(url, headers=headers) + request = Request(url, headers=headers) try: - return urllib2.urlopen(request) - except urllib2.HTTPError as e: + return urlopen(request) + except HTTPError as e: + error_code = e.getcode() log.error('HTTP_%d while GETing %s:\n %s', - e.getcode(), url, e.readlines()) - sys.exit(1) + error_code, url, e.readlines()) + # 404 not found shouldn't be fatal.. make this + # a list in case there are others + if error_code in [404]: + return None + else: + sys.exit(1) def get_pull_requests(owner, repo, last_update=None): @@ -78,6 +130,8 @@ def get_pull_requests(owner, repo, last_update=None): url = url % (owner, repo) while url: resp = _get(url) + if resp is None: + return data = json.loads(resp.read()) for x in data: ts = datetime.strptime(x['updated_at'], '%Y-%m-%dT%H:%M:%SZ') @@ -86,13 +140,13 @@ def get_pull_requests(owner, repo, last_update=None): return try: yield x - except: + except Exception: log.error('Unable to process pr(%r)', x) raise url = resp.headers.get('link') if url: # find the <$URL>; rel="next" to get the next page of results - m = re.match('<(\S+)>; rel="next"', url) + m = re.match(r'<(\S+)>; rel="next"', url) url = None if m: url = m.group(1) @@ -103,12 +157,12 @@ def get_commits(pull_request): return json.loads(resp.read()) -def get_author(crowd, commits): +def get_author(commits): if not len(commits): # some PR's have no commits: https://github.com/docker/docker/pull/5894 return email = commits[0]['commit']['author']['email'] - return get_linaro_person(crowd, email) + return get_linaro_person(email) def patchwork_state(github_status): @@ -136,7 +190,7 @@ def create_or_update(proj, owner, repo, author, pr): p = Patch.objects.get(msgid=msgid) tcs = TeamCredit.objects.filter(patch=p) if updated > tcs[0].last_state_change: - for k, v in fields.iteritems(): + for k, v in fields.items(): setattr(p, k, v) p.save() TeamCredit.objects.filter(patch=p).update( @@ -169,25 +223,24 @@ def repo_cache(): data = json.load(f) for repo, dt in data.items(): data[repo] = datetime.strptime(dt, '%Y-%m-%dT%H:%M:%S.%f') - except: + except Exception: log.exception('ignoring') yield data with open(fname, 'w') as f: json.dump(data, f, default=dt_serialize) -def create_tags(crowd, project, commits): +def create_tags(project, commits): for commit in commits: c = Commit(commit['sha'], commit['commit']['message'], commit['commit']['author']) team_project_credit.update_commit_callback( - crowd, project, None, c, False) + project, None, c, False) def main(args): - crwd = Crowd(settings.CROWD_USER, settings.CROWD_PASS, settings.CROWD_URL) - with crwd.cached(settings.CROWD_CACHE), repo_cache() as repos: + with repo_cache() as repos: for owner, repo, proj in GITHUB_REPOS: repo_path = '%s/%s' % (owner, repo) log.info('Looking at: %s', repo_path) @@ -198,12 +251,12 @@ def main(args): for pr in get_pull_requests(owner, repo, last_update): x += 1 commits = get_commits(pr) - auth = get_author(crwd, commits) + auth = get_author(commits) if auth: log.info('checking change: %d', pr['number']) create_or_update(proj, owner, repo, auth, pr) project = Project.objects.get(name=proj) - create_tags(crwd, project, commits) + create_tags(project, commits) repos[repo_path] = now finally: log.info('analayzed %d pull-requests', x) diff --git a/linaro_metrics/sync_teams.py b/linaro_metrics/sync_teams.py index 76eaa97..5ab074e 100755 --- a/linaro_metrics/sync_teams.py +++ b/linaro_metrics/sync_teams.py @@ -1,87 +1,87 @@ -#!/usr/bin/env python +#!/usr/bin/python3 import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append('/srv/linaro-git-tools') from bin import django_setup, add_logging_arguments django_setup() # must be called to get sys.path and django settings in place import logging -from django.conf import settings from django.contrib.auth.models import User -from patchwork.models import Person -from linaro_metrics.crowd import Crowd +from linaro_ldap import do_complex_query, do_query from linaro_metrics.models import Team, TeamMembership log = logging.getLogger('sync_teams') - -def get_or_create_person(crowd, email, save_person=True): - name = None - try: - person = Person.objects.get(email__iexact=email) - except Person.DoesNotExist: - # use crowd to get the "display-name" for the user - name = crowd.get_user_no_cache(email)['display-name'] - log.info('Creating person %s(%s)', name, email) - person = Person(name=name, email=email) - if save_person: - person.save() - - if not person.user: - users = User.objects.filter(person__email=email) - if users.count() == 0: - if not name: - name = crowd.get_user_no_cache(email)['display-name'] - users = User.objects.filter(username=name) - if users.count() == 0: - log.info('Creating user for %s', email) - user = User.objects.create_user(name, email, password=None) - else: - user = users[0] - person.user = user - if save_person: - person.save() - - return person - - -def sync_team(crowd, team, emails, user_memberships): - for email in emails: - user = get_or_create_person(crowd, email).user - user_memberships.setdefault(user, []).append(team) - _, created = TeamMembership.objects.get_or_create(team=team, user=user) - if created: - log.info('New team membership created for: %s', email) - - -def sync_crowd(crowd, teams): - user_memberships = {} - for team in teams: - emails = crowd.get_group(team.name) - log.info('syncing team: %s - (%s)', team, emails) - sync_team(crowd, team, emails, user_memberships) - if len(emails) == 0: - log.warn('empty group definition in crowd for: %s', team) - - for user in User.objects.all(): - memberships = user_memberships.get(user, []) - for tm in TeamMembership.objects.filter(user=user): - if tm.team not in memberships: - log.warn('Deleting %s\'s membership in %s', - user.email, tm.team.name) - tm.delete() +DRY_RUN = False + + +def get_email_by_uid(uid): + ldap_user_entry = do_query('uid', uid, ['mail']) + return(ldap_user_entry[0][1]['mail'][0]) + + +def sync_teams(teams): + + for t in teams: + ldap_results = do_complex_query( + search_filter='(&(objectClass=posixGroup)(cn=%s))' % t.name, + attrlist=['memberUid', 'mail'], + base='ou=security,ou=groups,dc=linaro,dc=org' + ) + + try: + uids_ldap = ldap_results[0][1]['memberUid'] + except KeyError as e: + print("Exception: '%s' for %s" % (e, ldap_results[0][0])) + continue + except IndexError as e: + print("Exception: %s" % e) + sys.exit(1) + + ldap_users = [get_email_by_uid(x) for x in uids_ldap] + memberships = TeamMembership.objects.filter(team=t) + + # look for new Users and add + for ldap_user in ldap_users: + # user should already exist as the sync_users script + # should have been run first + user = User.objects.filter(username=ldap_user).first() + if user is None: + continue + membership = \ + TeamMembership.objects.filter(team=t, user=user).first() + if membership is None: + print("Adding '%s' to team '%s'" % (user.username, t.name)) + if not DRY_RUN: + m = TeamMembership() + m.user = user + m.team = t + m.save() + + # look for Users to remove + for m in memberships: + if m.user.username not in ldap_users: + print("Removing '%s' from team '%s'" % ( + m.user.username, t.name)) + if not DRY_RUN: + m.delete() if __name__ == '__main__': import argparse parser = argparse.ArgumentParser( - description='Synchronize team memberships with info from crowd') + description='Synchronize team memberships with info from LDAP') add_logging_arguments(parser) - parser.parse_args() + parser.add_argument("--dry-run", "-n", action='store_true', + dest='DRY_RUN', default=False, + help="Run the script but do not execute any changes") + args = parser.parse_args() + + DRY_RUN = args.DRY_RUN - crowd = Crowd(settings.CROWD_USER, settings.CROWD_PASS, settings.CROWD_URL) - sync_crowd(crowd, Team.objects.filter(active=True)) + sync_teams(Team.objects.filter(active=True)) diff --git a/linaro_metrics/sync_users.py b/linaro_metrics/sync_users.py new file mode 100755 index 0000000..1808833 --- /dev/null +++ b/linaro_metrics/sync_users.py @@ -0,0 +1,203 @@ +#!/usr/bin/python3 + +import sys +import os +import argparse + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append("/srv/linaro-git-tools") + +from bin import django_setup, add_logging_arguments +django_setup() + +from django.conf import settings + +from django.contrib.auth.models import User +from patchwork.models import Person + +import linaro_ldap + +dry_run = True +verbose = False + + +def sanitize_ldap_entry(ldap_entry): + ''' convert indivicual ldap entry into a dict we can work with''' + e = {} + for key in ldap_entry[1]: + if key not in ['jpegPhoto', 'sshKey']: + e[key] = ldap_entry[1][key].pop() + return e + + +def get_other_scm(labeledUri): + ''' pull out user accounts for other platforms from LDAP ''' + + # ex: labeledURI: ['spoonix IRCCloak', 'spoon GitHub01'] + # returns a dict... key is the service, value is a list of usernames + + accounts = {} + for item in labeledUri: + (acct, service) = item.split(' ') + + if service.lower().find('github') and acct != 'N/A': + if 'github' not in accounts: + accounts['github'] = [] + accounts['github'].append(acct) + + return(accounts) + + +def build_employee_table(ldap_results): + ''' create a dictionary of all employees keyed off of email''' + employees = {} + + for e in ldap_results: + # if employeeType isn't set, it's a bot account + if 'employeeType' not in e: + continue + + # We need to use the 'mail' address as their username + # as we are currently still authenticating with crowd + user = e['mail'] + + employees[user] = {} + + # User objects have first and last name.. + employees[user]['first_name'] = e.get('givenName', '').decode('utf-8') + employees[user]['last_name'] = e['sn'].decode('utf-8') + # But Person objects only have one name field + employees[user]['name'] = "%s %s".decode('utf-8') % ( + employees[user]['first_name'], + employees[user]['last_name']) + # check for other emails the user may be using + if 'otherMailbox' in e: + employees[user]['other_mail'] = \ + e['otherMailbox'].decode('utf-8').split(',') + # we don't currently use this, but it might be handy + # to have available. LDAP uses this field to track + # github usernames, IRC nicks, and possibly other scm + # accounts associated with the user. + if 'labeledURI' in e: + employees[user]['other_scm'] = \ + get_other_scm(e['labeledURI'].decode('utf-8').split(',')) + + return employees + + +def get_ldap_users(): + ''' get a list of all LDAP user entries that are active employees ''' + employees_ldap_raw = linaro_ldap.do_complex_query( + search_filter='(uid=*)', + attrlist=['*'], + base='ou=staff,ou=accounts,dc=linaro,dc=org' + ) + + # ldap results are in tuple format, so let's convert it to a simple dict + return [sanitize_ldap_entry(e) for e in employees_ldap_raw] + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--dry_run', action='store_true') + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + + dry_run = args.dry_run + verbose = args.verbose + + employees = build_employee_table(get_ldap_users()) + + # Let's go through currently existing users and + # see if a User already exists. If it does, make + # sure the username is set to email and that acct + # is_active. If not, then we need to create it. + for email in employees: + + try: + user = User.objects.get(username=email) + + except Exception: + print("User does not exist %s... checking email" % email) + try: + user = User.objects.get(email=email) + except Exception: + print("Email does not exist %s... Creating" % email) + + user = User.objects.create( + username=email, + is_active=True, + email=email, + first_name=employees[email]['first_name'], + last_name=employees[email]['last_name'] + ) + + if user.username != email: + print("Updating username: %s" % email) + user.username = email + + if user.email != email: + print("Updating email: %s" % email) + user.email = email + + if user.is_active is not True: + print("Updating is_active: %s" % True) + user.is_active = True + + if user.first_name != employees[email]['first_name']: + print("Updating first_name: %s" % employees[email]['first_name']) + user.first_name = employees[email]['first_name'] + + if user.last_name != employees[email]['last_name']: + print("Updating first_name: %s" % employees[email]['last_name']) + user.last_name = employees[email]['last_name'] + + if not dry_run: + print("Processed user: %s" % user.email) + user.save() + + # now, let's clean up all active users by + # by 1) deactivating old users, 2) fixing the username and + # first/lastname for existing users, 3) creating missing + # Person objects and making sure existing ones point + # to the correct User + users = User.objects.filter(is_active=True) + + for user in users: + # Deactivate this user if the email isn't in list of + # active employees + if user.username not in employees: + print("DEACTIVATING: %s (%s) not in employees" % ( + user.username.encode('utf-8'), + user.email.encode('utf-8'))) + user.is_active = False + if not dry_run: + user.save() + continue + + # finally, let's check to see if the person objects need updating + other_mail = [user.email] + if 'other_mail' in employees[user.username]: + other_mail += employees[user.username]['other_mail'] + + for email in other_mail: + try: + person = Person.objects.get(email=email) + + except Person.DoesNotExist: + person = Person.objects.create( + email=email, + name="%s %s" % (user.first_name, user.last_name), + user=user + ) + print("CREATED: person <%s>" % person) + + if person.user != user: + person.user = user + + if not dry_run: + person.save() + + if verbose: + print("PROCESSED: %s" % (user.email)) diff --git a/linaro_metrics/team_project_credit.py b/linaro_metrics/team_project_credit.py index d1e973e..5c080b8 100644 --- a/linaro_metrics/team_project_credit.py +++ b/linaro_metrics/team_project_credit.py @@ -6,7 +6,6 @@ import logging from django.conf import settings -from linaro_metrics.crowd import Crowd from linaro_metrics.parsemail import get_linaro_person from linaro_metrics.models import ( CommitTagCredit, @@ -22,10 +21,10 @@ response_re = \ r' .*<(.*@.*)>$', re.M | re.I) -def update_commit_callback(crowd, project, repo, commit, dryrun): +def update_commit_callback(project, repo, commit, dryrun): for match in response_re.finditer(commit.message): tag, email = match.groups() - p = get_linaro_person(crowd, email) + p = get_linaro_person(email) if p: log.debug('User %s found with tag %s', p, tag) if not dryrun: @@ -47,8 +46,5 @@ def update_commit_callback(crowd, project, repo, commit, dryrun): @contextlib.contextmanager def update_commit_callback_constructor(): - crwd = Crowd(settings.CROWD_USER, settings.CROWD_PASS, settings.CROWD_URL) - - with crwd.cached(settings.CROWD_CACHE): - cb = functools.partial(update_commit_callback, crwd) - yield cb + cb = functools.partial(update_commit_callback) + yield cb diff --git a/linaro_metrics/teamcredit.py b/linaro_metrics/teamcredit.py new file mode 100755 index 0000000..9ae0552 --- /dev/null +++ b/linaro_metrics/teamcredit.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 +import datetime +from linaro_metrics.models import Team, TeamMembership, TeamCredit +from patchwork.models import Patch +from bin import django_setup +import logging +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +django_setup() # must be called to get sys.path and django settings in place + +dateb = datetime.datetime.now() - datetime.timedelta(days=180) + +log = logging.getLogger('teamcredit') + +""" +This is a thrown together script to fix the teamcredit statistics. It should +not often be needed, but at times a ldap team can change and the team lead +forgets to inform patches of this change. Hence the teamcredit will go out of +sync. + +The script runs in a default of 180 days. + +""" + +team = 'team-ldcg' +t = Team.objects.get(name=team) +tm = TeamMembership.objects.filter(team=t) + +for p in tm: + logging.info("#### %s %s", p.user, p.id) + # get all patches to that user + patchu = Patch.objects.filter(submitter__user=p.user) + for pa in patchu: + if dateb < pa.date: + tcs = TeamCredit.objects.filter(patch=pa) + if not tcs: + logging.info("No Patch: %s, %s", pa, tcs) + new_values = { + 'patch': pa, 'team': t, 'last_state_change': pa.date + } + tcs = TeamCredit(**new_values) + tcs.save() + else: + tcsno = TeamCredit.objects.filter(patch=pa, + team__name='no-team' + ) + if tcsno: + logging.info("No Credit: %s, %s", tcsno, pa.date) + TeamCredit.objects.filter(patch=pa).update(team=t) diff --git a/linaro_metrics/templates/linaro_metrics/index.html b/linaro_metrics/templates/linaro_metrics/index.html index deb9696..5504a59 100644 --- a/linaro_metrics/templates/linaro_metrics/index.html +++ b/linaro_metrics/templates/linaro_metrics/index.html @@ -135,7 +135,7 @@ function selectTab(item) { When analyzing the charts above, please take into account that the process which allows us to track patches was put in place at the end of January 2011 and that this website went live in June 2011. Check the -<a href="{% url 'linaro_metrics.views.faq_view' %}">FAQ</a> for answers to common +<a href="{% url 'faq_view' %}">FAQ</a> for answers to common questions.<br/><br/> <div class="pure-menu pure-menu-horizontal"> @@ -152,7 +152,9 @@ questions.<br/><br/> {% if cur_teams %} <ul> {% for t in cur_teams %} - <li><a href="{% url 'linaro_metrics.views.team_view' team=t.name %}">{{t}}</a></li> + {% with team_name=t.name|urlencode %} + <li><a href="/team/{{team_name}}/">{{t}}</a></li> + {% endwith %} {% endfor %} </ul> {% else %} @@ -163,16 +165,16 @@ questions.<br/><br/> <div id="projects" style="display:None"> <ul> {% for p in projects %} - <li><a href="{% url 'patchwork.views.patch.patch_list' project_id=p.linkname %}">{{p.name}}</a></li> + <li><a href="{% url 'patch-list' project_id=p.linkname %}">{{p.name}}</a></li> {% endfor %} </ul> </div> -{% if old_teams %} +{% if old_teams and False %} <div id="old-teams" style="display:None"> <ul> {% for t in old_teams %} - <li><a href="{% url 'linaro_metrics.views.team_view' team=t.name %}">{{t.display_name}}</a></li> + <li><a href="{% url 'team_view' team=t.name %}">{{t.display_name}}</a></li> {% endfor %} </ul> </div> diff --git a/linaro_metrics/templates/linaro_metrics/projects.html b/linaro_metrics/templates/linaro_metrics/projects.html new file mode 100644 index 0000000..34ca9b5 --- /dev/null +++ b/linaro_metrics/templates/linaro_metrics/projects.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block title %}Projects{% endblock %} +{% block heading %}Projects{% endblock %} + +{% block body %} + +<ul> + {% for p in projects %} + <li> + <a href="{% url 'patch_list' project_id=p.linkname %}">{{p.name}}</a> + </li> + {% endfor %} +</ul> + +{% endblock %} diff --git a/linaro_metrics/templates/linaro_metrics/report_project_activity.html b/linaro_metrics/templates/linaro_metrics/report_project_activity.html index dc843c2..2232493 100644 --- a/linaro_metrics/templates/linaro_metrics/report_project_activity.html +++ b/linaro_metrics/templates/linaro_metrics/report_project_activity.html @@ -13,4 +13,12 @@ {% endfor %} <table> +<h2>Top Accepted {{ summary|length}} Linaro Project Contributions</h2> +<table class="table"> + <tr><th>Project</th><th>12 months</th><th>6 months</th><th>3 months</th><th>1 month</th></tr> +{% for s in accepted %} + <tr class="{% cycle 'odd' 'even' %}">{% for td in s %}<td>{{td}}</td>{% endfor %}</tr> +{% endfor %} +<table> + {% endblock %} diff --git a/linaro_metrics/templates/linaro_metrics/teams.html b/linaro_metrics/templates/linaro_metrics/teams.html index 6f1697a..6223935 100644 --- a/linaro_metrics/templates/linaro_metrics/teams.html +++ b/linaro_metrics/templates/linaro_metrics/teams.html @@ -9,7 +9,7 @@ <ul> {% for t in teams %} <li> - <a href="{% url 'linaro_metrics.views.team_view' team=t.name %}">{{t.name}}</a> + <a href="{% url 'team_view' team=t.name %}">{{t.name}}</a> </li> {% endfor %} </ul> diff --git a/linaro_metrics/tests/test_crowd.py b/linaro_metrics/tests/test_crowd.py deleted file mode 100644 index 9e5cc34..0000000 --- a/linaro_metrics/tests/test_crowd.py +++ /dev/null @@ -1,76 +0,0 @@ -import json -import os -import tempfile -import time -import unittest - -from linaro_metrics.crowd import Crowd - - -class TestCrowdCache(unittest.TestCase): - def setUp(self): - _, self.tmpfile = tempfile.mkstemp(prefix='crowdtest') - self.addCleanup(os.unlink, self.tmpfile) - self.crowd = Crowd('user', 'pass', 'https://foo/bar/server') - - def fake_get(api_url): - return '''{"email": "email"}''' - self.crowd._get = fake_get - - def test_unicode(self): - '''Ensure we can handle unicode characters in an email address''' - # a real commit in linux.git where the author has a unicode character: - # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ - # commit/?id=e82661e23c60fc41424ca138820d729d8e4a2226 - self.crowd.get_user_no_cache(u'samuel.pitoiset\u0153gmail.com') - - def test_corrupted_cache(self): - with open(self.tmpfile, 'w') as f: - f.write('invalid json') - with self.crowd.cached(self.tmpfile): - self.assertTrue(self.crowd.user_valid('foo@bar.com')) - with open(self.tmpfile) as f: - self.assertIn('foo@bar.com', json.load(f)) - - def test_cache_hit(self): - data = { - 'foo@bar.com': { - 'email': 'foo', - 'valid': False, - 'expires': time.time() + 100, - } - } - with open(self.tmpfile, 'w') as f: - json.dump(data, f) - with self.crowd.cached(self.tmpfile): - self.assertFalse(self.crowd.user_valid('foo@bar.com')) - - def test_cache_miss(self): - data = { - 'foo@bar.com': { - 'email': 'foo', - 'expires': time.time() - 1, - } - } - with open(self.tmpfile, 'w') as f: - json.dump(data, f) - with self.crowd.cached(self.tmpfile): - self.assertTrue(self.crowd.user_valid('foo@bar.com')) - - def test_cache_clean(self): - data = { - 'foo@bar.com': { - 'email': 'email', - 'expires': time.time() - 1, - }, - 'FOO@bar.com': { - 'email': 'email', - 'expires': time.time() + 100, - } - } - with open(self.tmpfile, 'w') as f: - json.dump(data, f) - with self.crowd.cached(self.tmpfile): - pass - with open(self.tmpfile) as f: - self.assertEqual(1, len(json.load(f).keys())) diff --git a/linaro_metrics/tests/test_sync_gerrit_changes.py b/linaro_metrics/tests/test_sync_gerrit_changes.py index 075227e..dcd93a5 100644 --- a/linaro_metrics/tests/test_sync_gerrit_changes.py +++ b/linaro_metrics/tests/test_sync_gerrit_changes.py @@ -15,7 +15,7 @@ from linaro_metrics.models import Team, TeamCredit, TeamMembership class TestSyncGerritChanges(TestCase): fixtures = ['default_states'] - @mock.patch('urllib2.urlopen') + @mock.patch('urllib.request.urlopen') def test_get_user_changes_simple(self, urlopen): resp = mock.Mock() items = [ @@ -26,7 +26,7 @@ class TestSyncGerritChanges(TestCase): changes = list(sync_gerrit_changes.get_user_changes('foo@bar.com', '')) self.assertEqual(items, changes) - @mock.patch('urllib2.urlopen') + @mock.patch('urllib.request.urlopen') def test_get_user_changes_continue(self, urlopen): resp = mock.Mock() responses = [ diff --git a/linaro_metrics/tests/test_sync_teams.py b/linaro_metrics/tests/test_sync_teams.py index f5778d0..7a0c6e5 100644 --- a/linaro_metrics/tests/test_sync_teams.py +++ b/linaro_metrics/tests/test_sync_teams.py @@ -1,74 +1,73 @@ -from django.contrib.auth.models import User -from django.test import TestCase -from patchwork.models import Person - -from linaro_metrics.models import Team, TeamMembership -from linaro_metrics.sync_teams import get_or_create_person, sync_crowd - - -class FakeCrowd(object): - def __init__(self): - self.groups = {} - - def add_group(self, name, users): - self.groups[name] = users - - def get_group(self, name): - return [x[0] for x in self.groups[name]] - - def get_user_no_cache(self, user_email): - for users in self.groups.values(): - for email, display_name in users: - if user_email == email: - return {'display-name': display_name} - - -class TestSyncTeams(TestCase): - def test_user_created(self): - t = Team.objects.create(name='foo') - crowd = FakeCrowd() - crowd.add_group(t.name, [('user@foo.com', 'user name')]) - sync_crowd(crowd, [t]) - - p = Person.objects.get(email='user@foo.com') - self.assertEquals('user name', p.name) - self.assertEquals(p.email, p.user.email) - tm = TeamMembership.objects.all()[0] - self.assertEquals('foo', tm.team.name) - self.assertEquals(p.user, tm.user) - - def test_memberships_change(self): - crowd = FakeCrowd() - crowd.add_group('foo', [('user@foo.com', 'user name')]) - crowd.add_group('bam', [('user@foo.com', 'user name')]) - u = get_or_create_person(crowd, 'user@foo.com').user - - foo = Team.objects.create(name='foo') - bar = Team.objects.create(name='bar') - bam = Team.objects.create(name='bam') - - # put user in foo and bar - TeamMembership.objects.create(team=foo, user=u) - TeamMembership.objects.create(team=bar, user=u) - - # sync - and we should only be in foo and bam - sync_crowd(crowd, [foo, bam]) - teams = [x.team.name for x in TeamMembership.objects.all()] - self.assertEqual(['foo', 'bam'], teams) - - def test_person_no_user(self): - crowd = FakeCrowd() - crowd.add_group('foo', [('user@foo.com', 'user name')]) - Person.objects.create(name='user name', email='user@foo.com') - u = get_or_create_person(crowd, 'user@foo.com').user - p = Person.objects.get(email='user@foo.com') - self.assertEqual(u, p.user) - - def test_user_no_person(self): - crowd = FakeCrowd() - crowd.add_group('foo', [('user@foo.com', 'user name')]) - - orig = User.objects.create_user( - 'user name', 'user@foo.com', password=None) - self.assertEqual( - orig, get_or_create_person(crowd, 'user@foo.com').user) +# from django.contrib.auth.models import User +# from django.test import TestCase +# from patchwork.models import Person + +# from linaro_metrics.models import Team, TeamMembership + + +# class FakeCrowd(object): +# def __init__(self): +# self.groups = {} +# +# def add_group(self, name, users): +# self.groups[name] = users +# +# def get_group(self, name): +# return [x[0] for x in self.groups[name]] +# +# def get_user_no_cache(self, user_email): +# for users in self.groups.values(): +# for email, display_name in users: +# if user_email == email: +# return {'display-name': display_name} + + +# class TestSyncTeams(TestCase): +# def test_user_created(self): +# t = Team.objects.create(name='foo') +# crowd = FakeCrowd() +# crowd.add_group(t.name, [('user@foo.com', 'user name')]) +# sync_crowd(crowd, [t]) +# +# p = Person.objects.get(email='user@foo.com') +# self.assertEquals('user name', p.name) +# self.assertEquals(p.email, p.user.email) +# tm = TeamMembership.objects.all()[0] +# self.assertEquals('foo', tm.team.name) +# self.assertEquals(p.user, tm.user) +# +# def test_memberships_change(self): +# crowd = FakeCrowd() +# crowd.add_group('foo', [('user@foo.com', 'user name')]) +# crowd.add_group('bam', [('user@foo.com', 'user name')]) +# u = get_or_create_person(crowd, 'user@foo.com').user +# +# foo = Team.objects.create(name='foo') +# bar = Team.objects.create(name='bar') +# bam = Team.objects.create(name='bam') +# +# # put user in foo and bar +# TeamMembership.objects.create(team=foo, user=u) +# TeamMembership.objects.create(team=bar, user=u) +# +# # sync - and we should only be in foo and bam +# sync_crowd(crowd, [foo, bam]) +# teams = [x.team.name for x in TeamMembership.objects.all()] +# self.assertEqual(['foo', 'bam'], teams) +# +# def test_person_no_user(self): +# crowd = FakeCrowd() +# crowd.add_group('foo', [('user@foo.com', 'user name')]) +# Person.objects.create(name='user name', email='user@foo.com') +# u = get_or_create_person(crowd, 'user@foo.com').user +# p = Person.objects.get(email='user@foo.com') +# self.assertEqual(u, p.user) +# +# def test_user_no_person(self): +# crowd = FakeCrowd() +# crowd.add_group('foo', [('user@foo.com', 'user name')]) +# +# orig = User.objects.create_user( +# 'user name', 'user@foo.com', password=None) +# self.assertEqual( +# orig, get_or_create_person(crowd, 'user@foo.com').user) diff --git a/linaro_metrics/tests/test_views.py b/linaro_metrics/tests/test_views.py index 9ec0bf0..6fdfb8b 100644 --- a/linaro_metrics/tests/test_views.py +++ b/linaro_metrics/tests/test_views.py @@ -1,25 +1,42 @@ from django.contrib.auth.models import User from django.test import Client, TestCase, skipUnlessDBFeature -from linaro_metrics.models import Team -from linaro_metrics.sync_teams import sync_crowd -from linaro_metrics.tests.test_sync_teams import FakeCrowd +from linaro_metrics.models import Team, TeamMembership class TestTeamsView(TestCase): fixtures = ['default_states'] def setUp(self): - # Create team memberships and project for a patch we'll import. + user1 = User.objects.create( + email='user@foo.com', + username='user', + first_name='user', + last_name='name', + is_active=True + ) + + user2 = User.objects.create( + email='user2@foo.com', + username='user2', + first_name='user2', + last_name='name', + is_active=True + ) + + user3 = User.objects.create( + email='zoltan.kiss@linaro.org', + username='zoltan.kiss', + first_name='Zoltan', + last_name='Kiss', + is_active=True + ) + t = Team.objects.create(name='foo') - crowd = FakeCrowd() - foo_group = [ - ('user@foo.com', 'user name'), - ('user2@foo.com', 'user2 name'), - ('zoltan.kiss@linaro.org', 'Zoltan Kiss'), - ] - crowd.add_group('foo', foo_group) - sync_crowd(crowd, [t]) + + TeamMembership.objects.create(team=t, user=user1) + TeamMembership.objects.create(team=t, user=user2) + TeamMembership.objects.create(team=t, user=user3) # This can't be run against sqlite3, so it won't get hit by unit-test.sh. @skipUnlessDBFeature('supports_mixed_date_datetime_comparisons') diff --git a/linaro_metrics/urls.py b/linaro_metrics/urls.py index 0b3fe4d..4ddcd2e 100644 --- a/linaro_metrics/urls.py +++ b/linaro_metrics/urls.py @@ -1,32 +1,35 @@ -from django.conf.urls import patterns, url, include +from django.conf.urls import url, include from django.contrib import admin - import patchwork.urls +import linaro_metrics.views from linaro_metrics.api import TeamList, TeamDetail, TeamPatchList, TeamTagView admin.autodiscover() -urlpatterns = patterns( - '', +urlpatterns = [ # Provide our override views of things in patchwork - (r'^$', 'linaro_metrics.views.index_view'), - (r'^project/(?P<project_id>[^/]+)/list/$', - 'linaro_metrics.views.project_view'), - url(r'^register/', 'linaro_metrics.views.user_register'), - + url(r'^$', linaro_metrics.views.index_view), # Include all the standard patchwork urls. url(r'^', include(patchwork.urls)), + url(r'^project/(?P<project_id>[^/]+)/list/$', + linaro_metrics.views.project_view), + url(r'^register/', linaro_metrics.views.user_register), + # Now provide our own urls. - (r'^faq$', 'linaro_metrics.views.faq_view'), - (r'^team/$', 'linaro_metrics.views.team_overview'), - (r'^team/(?P<team>[^/]+)/$', 'linaro_metrics.views.team_view'), - (r'^patches/(?P<user>[^/]+)/$', 'linaro_metrics.views.user_view'), - (r'^reports/project_activity$', - 'linaro_metrics.views.report_project_activity'), - (r'^reports/non-author-sign-offs$', - 'linaro_metrics.views.report_signed_off_non_author'), + url(r'^faq$', linaro_metrics.views.faq_view, name='faq_view'), + url(r'^team/$', linaro_metrics.views.team_overview, name='team_overview'), + url(r'^projects/$', linaro_metrics.views.project_overview, + name='project_overview'), + url(r'^team/(?P<team>[^/]+)/$', linaro_metrics.views.team_view, + name='team_view'), + url(r'^patches/(?P<user>[^/]+)/$', linaro_metrics.views.user_view, + name='user_view'), + url(r'^reports/project_activity$', + linaro_metrics.views.report_project_activity, name='project_activity'), + url(r'^reports/non-author-sign-offs$', + linaro_metrics.views.report_signed_off_non_author), url(r'^api/1.0/teams/$', TeamList.as_view(), name='api-team-list'), url(r'^api/1.0/teams/(?P<pk>[^/]+)/$', TeamDetail.as_view(), @@ -37,5 +40,5 @@ urlpatterns = patterns( TeamTagView.as_view(), name='api-team-credits'), # compatibility for old patches - (r'^(?P<patch>\d+)/$', 'linaro_metrics.views.old_patch_link'), -) + url(r'^(?P<patch>\d+)/$', linaro_metrics.views.old_patch_link), +] diff --git a/linaro_metrics/views.py b/linaro_metrics/views.py index 308894f..9fd8f3e 100644 --- a/linaro_metrics/views.py +++ b/linaro_metrics/views.py @@ -6,7 +6,7 @@ import mock import django.template.base from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth.models import User, AnonymousUser from django.core.cache import cache from django.http import StreamingHttpResponse from django.shortcuts import get_object_or_404, redirect, render @@ -23,6 +23,7 @@ from linaro_metrics.models import ( Team, TeamCredit, TeamMembership, + State, ) @@ -71,6 +72,11 @@ def team_overview(request): return render(request, 'linaro_metrics/teams.html', context) +def project_overview(request): + context = {'projects': Project.objects.all()} + return render(request, 'linaro_metrics/projects.html', context) + + # The patchwork version of the "personify" filter requires a project. We don't # have a project for "team". Although personify is a simple function, we can't # directly monkey-patch it due to the way django templates load filters. This @@ -116,12 +122,11 @@ def _non_project_ctx(request, view, view_args, patches): project.is_editable.return_value = False project.tags = [] - orig = request.user.is_authenticated - request.user.is_authenticated = mock.Mock() - request.user.is_authenticated.return_value = False + orig = request.user + request.user = AnonymousUser() context = generic_list( request, project, view, view_args=view_args, patches=patches) - request.user.is_authenticated = orig + request.user = orig # The DelegateFilter won't work for because its tied to a project/user. context['filters']._filters = [ @@ -143,7 +148,7 @@ def team_view(request, team): month = 6 context = _non_project_ctx( - request, 'linaro_metrics.views.team_view', view_args, patches) + request, 'team_view', view_args, patches) context.update({ 'team': team, 'memberships': TeamMembership.objects.filter(team=team), @@ -164,7 +169,7 @@ def user_view(request, user): view_args = {'user': user.id} - context = _non_project_ctx(request, 'linaro_metrics.views.user_view', + context = _non_project_ctx(request, 'user_view', view_args=view_args, patches=patches) context.update({ 'patch_user': user, @@ -181,7 +186,7 @@ def project_view(request, project_id): try: with open(commit) as f: project.last_commit = f.read() - except: + except Exception: project.last_commit = '' view_args = {'project_id': project.linkname} @@ -211,6 +216,8 @@ def report_project_activity(request): last_1 = _subtract_months(last_full_month, 1) summary = {} + accepted = {} + qs = TeamCredit.patch_count_by_month( 12, values=('month', 'patch__project__name'), patch__date__lte=last_full_month) @@ -227,9 +234,30 @@ def report_project_activity(request): if x['month'] >= last_12: e['last_12'] += x['patch__pk__count'] items = [(x, y['last_12'], y['last_6'], y['last_3'], y['last_1']) - for x, y in summary.iteritems()] + for x, y in summary.items()] summary = sorted(items, key=lambda x: x[1], reverse=True)[:20] - context = {'summary': summary} + + qs_a = TeamCredit.patch_count_by_month( + 12, values=('month', 'patch__project__name'), + patch__date__lte=last_full_month, + state=State.objects.get(name='Accepted')) + for x in qs_a: + e = accepted.setdefault( + x['patch__project__name'], { + 'last_12': 0, 'last_6': 0, 'last_3': 0, 'last_1': 0}) + if x['month'] >= last_1: + e['last_1'] += x['patch__pk__count'] + if x['month'] >= last_3: + e['last_3'] += x['patch__pk__count'] + if x['month'] >= last_6: + e['last_6'] += x['patch__pk__count'] + if x['month'] >= last_12: + e['last_12'] += x['patch__pk__count'] + items = [(x, y['last_12'], y['last_6'], y['last_3'], y['last_1']) + for x, y in accepted.items()] + accepted = sorted(items, key=lambda x: x[1], reverse=True)[:20] + + context = {'summary': summary, 'accepted': accepted} return render( request, 'linaro_metrics/report_project_activity.html', context) diff --git a/patch_matcher.py b/patch_matcher.py index e6dfe34..064b922 100644 --- a/patch_matcher.py +++ b/patch_matcher.py @@ -36,8 +36,8 @@ def _patches_similar(name1, diff1, name2, diff2): def _get_patchwork_author_committer(commit): - _, auth_email = email.utils.parseaddr(commit.author) - _, comm_email = email.utils.parseaddr(commit.committer) + _, auth_email = email.utils.parseaddr(commit.author.decode()) + _, comm_email = email.utils.parseaddr(commit.committer.decode()) try: auth = Person.objects.get(email=auth_email) except Person.DoesNotExist: @@ -74,5 +74,5 @@ def get_patches_matching_commit(project, repo, commit): """ persons = [x for x in _get_patchwork_author_committer(commit) if x] patch = repo.get_patch(commit) - name = commit.message.split('\n')[0] + name = commit.message.decode().split('\n')[0] return get_patches_matching(project, persons, name, patch) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3aa3b80 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +Django==1.11.29 +djangorestframework==3.5 +django-filter==1.1 +sqlparse==0.2.2 +dulwich==0.19.6 +mock +flake8 +python-ldap==3.3.1 +django-auth-ldap==1.6.1 +virtualenv +GitPython==3.1.14 diff --git a/run-dev.sh b/run-dev.sh new file mode 100755 index 0000000..f967328 --- /dev/null +++ b/run-dev.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +docker run -d --name patches.linaro.org \ + -e ANSIBLE_GROUP=patchwork \ + -e ANSIBLE_HOST=patches.linaro.org \ + -v $(pwd):/srv/workspace \ + ansible/baseimage:20.04 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..d396e7d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[tool:pytest] +log_cli=True +log_level=INFO +addopts=-x -v + +[flake8] +ignore = E402,W503 +exclude = + linaro_metrics/migrations, + linaro_ldap.py, + .venv +per-file-ignores = + linaro_metrics/cli.py: F401, + linaro_metrics/team_project_credit.py: F401 + linaro_metrics/sync_users.py: F401 + linaro_metrics/sync_teams.py: F401 diff --git a/tests/__init__.py b/tests/__init__.py index ed6ac18..eaabb5e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -37,11 +37,11 @@ class TestRepo(object): def last_commit(self): out = subprocess.check_output( - ['git', 'log', '--format=oneline', '-1'], cwd=self.path) + ['git', 'log', '--format=oneline', '-1'], cwd=self.path, text=True) return out.split(' ')[0] def create_patch(self, commit): patch = subprocess.check_output( ['git', 'format-patch', '%s^..%s' % (commit, commit)], - cwd=self.path) + cwd=self.path, text=True) return os.path.join(self.path, patch.strip()) diff --git a/tests/test_gitrepo.py b/tests/test_gitrepo.py index d1ec23f..505c126 100644 --- a/tests/test_gitrepo.py +++ b/tests/test_gitrepo.py @@ -13,93 +13,100 @@ class TestGitRepo(unittest.TestCase): self.tmpdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmpdir) - self.main = gitrepo.Repo(self.tmpdir, 'main', None) + self.main = gitrepo.Repo(self.tmpdir, "main", None) os.mkdir(self.main.path) - subprocess.check_call(['git', 'init'], cwd=self.main.path) + subprocess.check_call(["git", "init"], cwd=self.main.path) def _add_commit(self, path, content, message): - with open(os.path.join(self.main.path, path), 'w') as f: + with open(os.path.join(self.main.path, path), "w") as f: f.write(content) - subprocess.check_call(['git', 'add', '.'], cwd=self.main.path) subprocess.check_call( - ['git', 'commit', '-a', '-m', message], cwd=self.main.path) + ["git", "add", "."], cwd=self.main.path, text=True) + subprocess.check_call( + ["git", "commit", "-a", "-m", message], cwd=self.main.path, + text=True + ) return self._last_commit() def _last_commit(self): out = subprocess.check_output( - ['git', 'log', '--format=oneline', '-1'], cwd=self.main.path) - return out.split(' ')[0] + ["git", "log", "--format=oneline", "-1"], cwd=self.main.path, + text=True + ) + return out.split(" ")[0] def test_pull_simple(self): - self._add_commit('foo1', 'commit1', 'commit1') - self._add_commit('foo2', 'commit2', 'commit2') + self._add_commit("foo1", "commit1", "commit1") + self._add_commit("foo2", "commit2", "commit2") - repo = gitrepo.Repo(self.tmpdir, 'clone', self.main.path) + repo = gitrepo.Repo(self.tmpdir, "clone", self.main.path) repo._clone() commits = [] - commits.append(self._add_commit('foo3', 'commit3', 'commit3')) - commits.append(self._add_commit('foo4', 'commit4', 'commit4')) + commits.append(self._add_commit("foo3", "commit3", "commit3")) + commits.append(self._add_commit("foo4", "commit4", "commit4")) repo.update() out = subprocess.check_output( - ['git', 'log', '--format=oneline', '-2'], cwd=repo.path) - found = [x.split(' ')[0] for x in out.split('\n') if x] + ["git", "log", "--format=oneline", "-2"], cwd=repo.path, text=True + ) + found = [x.split(" ")[0] for x in out.split("\n") if x] self.assertEqual(commits, list(reversed(found))) def test_pull_rewrite(self): - self._add_commit('foo1', 'commit1', 'commit1') + self._add_commit("foo1", "commit1", "commit1") - repo = gitrepo.Repo(self.tmpdir, 'clone', self.main.path) + repo = gitrepo.Repo(self.tmpdir, "clone", self.main.path) repo._clone() commits = [] - commits.append(self._add_commit('foo3', 'commit3', 'commit3')) + commits.append(self._add_commit("foo3", "commit3", "commit3")) # now add this foo4 to be overwritten - self._add_commit('foo4', 'commit4', 'commit4') + self._add_commit("foo4", "commit4", "commit4") repo.update() subprocess.check_call( - ['git', 'reset', '--hard', 'HEAD^'], cwd=self.main.path) - commits.append(self._add_commit('foo4a', 'commit4a', 'commit4a')) + ["git", "reset", "--hard", "HEAD^"], cwd=self.main.path) + commits.append(self._add_commit("foo4a", "commit4a", "commit4a")) repo.update() out = subprocess.check_output( - ['git', 'log', '--format=oneline', '-2'], cwd=repo.path) - found = [x.split(' ')[0] for x in out.split('\n') if x] + ["git", "log", "--format=oneline", "-2"], cwd=repo.path + ) + found = [x.split(" ")[0] for x in out.decode().split("\n") if x] self.assertEqual(commits, list(reversed(found))) def test_commits_to_check_empty(self): - '''works off an empty repo that's never been analyzed''' + """works off an empty repo that's never been analyzed""" commits = [] - commits.append(self._add_commit('foo', 'foocontent', 'commit1')) - commits.append(self._add_commit('foo', 'foocontent2', 'commit2')) + commits.append(self._add_commit("foo", "foocontent", "commit1")) + commits.append(self._add_commit("foo", "foocontent2", "commit2")) - found = [x.id for x in self.main.process_unchecked_commits()] + found = [x.id.decode() for x in self.main.process_unchecked_commits()] self.assertEqual(commits, found) def test_commits_to_check_previous(self): - '''works off an empty repo that's been analyzed''' - self._add_commit('foo', 'foocontent', 'commit1') + """works off an empty repo that's been analyzed""" + self._add_commit("foo", "foocontent", "commit1") # force last commit file to be updated list(self.main.process_unchecked_commits()) commits = [] - commits.append(self._add_commit('foo', 'foocontent2', 'commit2')) + commits.append(self._add_commit("foo", "foocontent2", "commit2")) - found = [x.id for x in self.main.process_unchecked_commits()] + found = [x.id.decode() for x in self.main.process_unchecked_commits()] self.assertEqual(commits, found) def test_commits_to_check_rewrite(self): - '''can handle a history rewrite''' + """can handle a history rewrite""" commits = [] - commits.append(self._add_commit('foo', 'foocontent', 'commit1')) - commits.append(self._add_commit('foo', 'foocontent2', 'commit2')) + commits.append(self._add_commit("foo", "foocontent", "commit1")) + commits.append(self._add_commit("foo", "foocontent2", "commit2")) - last_commit = os.path.join(self.main.path, 'patchwork-last-commit') - with open(last_commit, 'w') as f: - f.write('11111111111') # invalid sha1, so we'll search back + last_commit = os.path.join(self.main.path, "patchwork-last-commit") + with open(last_commit, "w") as f: + f.write("11111111111") # invalid sha1, so we'll search back - found = [x.id for x in self.main.process_unchecked_commits()] + found = [x.id.decode() for x in self.main.process_unchecked_commits()] self.assertEqual(commits, found) diff --git a/tests/test_import_emails.py b/tests/test_import_emails.py index f576818..d1f57c2 100644 --- a/tests/test_import_emails.py +++ b/tests/test_import_emails.py @@ -6,9 +6,9 @@ import import_emails from django.conf import settings from django.test import TestCase +from django.contrib.auth.models import User from patchwork.models import Comment, Patch, Person, Project, State from linaro_metrics.models import Team, TeamCredit, TeamMembership -from linaro_metrics.sync_teams import get_or_create_person import mock @@ -42,12 +42,69 @@ class ImapFake(object): class TestImportEmail(TestCase): fixtures = ['default_states'] + def tearDown(self): + super(TestImportEmail, self).tearDown() + Person.objects.all().delete() + User.objects.all().delete() + def setUp(self): super(TestImportEmail, self).setUp() self.tmpdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmpdir) + self.user = User.objects.create( + email='user.name@linaro.org', + username='user.name', + first_name='User', + last_name='Name', + is_active=True + ) + + self.user2 = User.objects.create( + email='peter.maydell@linaro.org', + username='peter.maydell', + first_name='Peter', + last_name='Maydell', + is_active=True + ) + + self.person = Person.objects.create( + email='user.name@linaro.org', + name='User Name', + user=self.user + ) + + self.person2 = Person.objects.create( + email='user.name@kernel.org', + name='User Name', + user=self.user + ) + + self.person3 = Person.objects.create( + email='peter.maydell@linaro.org', + name='Peter Maydell', + user=self.user2 + ) + + self.person_other = Person.objects.create( + email='robh@kernel.org', + name='Rob Herring', + user=self.user + ) + + self.person_olof = Person.objects.create( + email='olof@lixom.net', + name='Olof Johansson', + user=self.user2 + ) + + self.notlinaro_person = Person.objects.create( + email='not@notlinaro.org', + name='Not Linaro', + user=None + ) + p = Project.objects.create(listid='lng-odp.lists.linaro.org') self.addCleanup(p.delete) @@ -89,21 +146,25 @@ class TestImportEmail(TestCase): settings.PARSEMAIL_MONKEY_PATCHER = 'tests.test_import_emails.ImapFake' self.assertEqual(ImapFake, import_emails.get_monkey_patcher()) - @mock.patch('linaro_metrics.parsemail.Crowd') - def test_monkey_patch_linaro_only(self, crowd): + def test_monkey_patch_linaro_only(self): '''Test that monkey patching rejects non-linaro patches''' - crowd().user_valid.return_value = False Project.objects.all().delete() with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch('non_linaro.mbox') self.assertEqual(0, Patch.objects.all().count()) - @mock.patch('linaro_metrics.parsemail.Crowd') - def test_monkey_patch_author_check_user_created(self, crowd): + def test_monkey_patch_author_check_user_created(self): '''Test that we can find the author from the patch comment. The author is linaro but doesn't exist locally''' Project.objects.all().delete() - crowd().get_user_no_cache.return_value = {'display-name': 'User Name'} + # junk + self.assertEqual(1, User.objects.filter(username='user.name').count()) + self.assertEqual( + Person.objects.filter(email='user.name@linaro.org').first().user, + User.objects.filter(username='user.name').first() + ) + # junk + with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch('author_submitter_differ.mbox') self.assertEqual(1, Patch.objects.all().count()) @@ -116,26 +177,22 @@ class TestImportEmail(TestCase): p = Project.objects.get(linkname=settings.DEFAULT_PROJECT) self.assertEqual(p, patches[0].project) - @mock.patch('linaro_metrics.parsemail.Crowd') - def test_monkey_patch_submitter_is_linaro(self, crowd): + def test_monkey_patch_submitter_is_linaro(self): '''A valid Linaro User may submit patches on behalf of a user. We have users that want to track these patches in our instance, but we SHOULD NOT contribute these to "team credits"''' Project.objects.all().delete() - crowd().get_user_no_cache.return_value = {'display-name': 'User Name'} with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch('applied-patch.mbox') self.assertEqual(0, Patch.objects.all().count()) tcs = TeamCredit.objects.all() self.assertEqual(0, tcs.count()) - @mock.patch('linaro_metrics.parsemail.Crowd') - def test_monkey_patch_maintainer_applied(self, crowd): + def test_monkey_patch_maintainer_applied(self): '''Don't give a patch credit to a maintainer applying a patch to a tree''' Project.objects.all().delete() - crowd().get_user_no_cache.return_value = {'display-name': 'User Name'} with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch('author_not_linaro.mbox') self.assertEqual(1, Patch.objects.all().count()) @@ -147,17 +204,14 @@ class TestImportEmail(TestCase): p = Project.objects.get(linkname=settings.DEFAULT_PROJECT) self.assertEqual(p, patches[0].project) - @mock.patch('linaro_metrics.parsemail.Crowd') - def _test_patch_auth(self, patch, author, crowd): + def _test_patch_auth(self, patch, author): Project.objects.all().delete() - Person.objects.all().delete() self.auth_found = None - def get_user_no_cache(email): - self.auth_found = email - return {'display-name': 'User Name'} + person = Person.objects.filter(email=author) + if person.count() > 0: + self.auth_found = person[0].email - crowd().get_user_no_cache = get_user_no_cache with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch(patch) self.assertEqual(1, Patch.objects.all().count()) @@ -180,31 +234,24 @@ class TestImportEmail(TestCase): self._test_patch_auth( 'cp8859.mbox', 'user.name@linaro.org') - with mock.patch('linaro_metrics.parsemail.Crowd') as crowd: - crowd().user_valid.return_value = False - with import_emails.get_monkey_patcher()(import_emails.parser): - self._import_patch('cp8859_comment.mbox') - self.assertEqual(1, Comment.objects.all().count()) + with import_emails.get_monkey_patcher()(import_emails.parser): + self._import_patch('cp8859_comment.mbox') + self.assertEqual(1, Comment.objects.all().count()) - @mock.patch('linaro_metrics.parsemail.Crowd') - def test_monkey_patch_user_is_linaro(self, crowd): + def test_monkey_patch_user_is_linaro(self): '''A valid Linaro User may submit patches from a non-linaro looking Person''' Project.objects.all().delete() def user_valid(email): return email.endswith('@linaro.org') - crowd().user_valid = user_valid - crowd().get_user_no_cache.return_value = {'display-name': 'User Name'} - person = get_or_create_person(crowd(), 'user.name@linaro.org') teams = [ Team.objects.create(name='foo'), Team.objects.create(name='bar'), ] for t in teams: - TeamMembership.objects.create(team=t, user=person.user) - Person.objects.create(email='robh@kernel.org', user=person.user) + TeamMembership.objects.create(team=t, user=self.user) with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch('user_linaro_not_person.mbox') @@ -212,15 +259,12 @@ class TestImportEmail(TestCase): tcs = [x.team for x in TeamCredit.objects.all()] self.assertEqual(teams, tcs) - @mock.patch('linaro_metrics.parsemail.Crowd') - def test_monkey_patch_no_listid(self, crowd): + def test_monkey_patch_no_listid(self): Project.objects.all().delete() qemu = Project.objects.create( name='qemu-devel', linkname='qemu-devel', listemail='qemu-devel@nongnu.org', listid='qemu-devel.nongnu.org') - crowd().get_user_no_cache.return_value = {'display-name': 'User Name'} - with import_emails.get_monkey_patcher()(import_emails.parser): self._import_patch('no-list-id.mbox') self.assertEqual(1, Patch.objects.all().count()) diff --git a/tests/test_update_commited_patches.py b/tests/test_update_commited_patches.py index 13f294c..15c97be 100644 --- a/tests/test_update_commited_patches.py +++ b/tests/test_update_commited_patches.py @@ -12,8 +12,9 @@ from linaro_metrics.models import ( CommitTagCredit, TeamMembership, Team, + User, + Person ) -from linaro_metrics.parsemail import get_linaro_person from tests import TestRepo import mock @@ -42,6 +43,21 @@ class TestUpdateCommitedPatches(TestCase): linkname='patchwork_copy') self.addCleanup(self.project.delete) + self.linaro_user = User.objects.create( + email='legit.user@linaro.org', + username='legit.user', + first_name='Legit', + last_name='User', + is_active=True) + self.addCleanup(self.linaro_user.delete) + + self.linaro_person = Person.objects.create( + email='legit.user@linaro.org', + name='Legit User', + user=self.linaro_user + ) + self.addCleanup(self.linaro_person.delete) + commit = self.upstream_repo.add_commit( 'file.txt', 'line1\nline2', 'testing update_commited_patches') self.patchmail = self.upstream_repo.create_patch(commit) @@ -110,24 +126,20 @@ class TestUpdateCommitedPatches(TestCase): commit = mock.Mock() commit.message = 'Commit message\n' \ - 'Signed-off-by: Foo Bar <foo.bar@linaro.org>' + 'Signed-off-by: %s <%s>' % ( + self.linaro_person.name, + self.linaro_person.email) commit.id = '1234' - commit.author = 'Foo Bar <foo.bar@linaro.org>' + commit.author = '%s <%s>' % ( + self.linaro_person.name, + self.linaro_person.email) commit.commit_time = time.time() commit.commit_timezone = -30 - crowd = mock.Mock() - crowd.user_valid.return_value = True - - crowd.get_user_no_cache.return_value = { - 'display-name': 'foo.bar@linaro.org', - 'email': 'foo.bar@linaro.org', - } - p = get_linaro_person(crowd, 'foo.bar@linaro.org').user t = Team.objects.create(name='TeamName') - TeamMembership.objects.create(team=t, user=p) + TeamMembership.objects.create(team=t, user=self.linaro_user) - update_commit_callback(crowd, self.project, None, commit, False) + update_commit_callback(self.project, None, commit, False) credits = CommitTagCredit.objects.all() self.assertEqual(1, credits.count()) diff --git a/unit-test.sh b/unit-test.sh index 3590e90..031e55b 100755 --- a/unit-test.sh +++ b/unit-test.sh @@ -7,40 +7,34 @@ VENV_DIR="${VENV_DIR-$HERE/.venv}" if [ -z $VIRTUAL_ENV ] ; then echo "creating venv: $VENV_DIR ..." - virtualenv --python=`which python2` $VENV_DIR + virtualenv --python=`which python3` $VENV_DIR . $VENV_DIR/bin/activate - pip install Django==1.8.10 - pip install djangorestframework==3.5 - pip install django-filter==1.0 - pip install sqlparse==0.2.2 - pip install dulwich - pip install mock - pip install flake8 + pip install -r requirements.txt + wget -q https://git.linaro.org/infrastructure/linaro-git-tools.git/plain/linaro_ldap.py fi if [ -z $PYTHONPATH ] ; then - git clone https://github.com/getpatchwork/patchwork.git $VENV_DIR/patchwork - cd $VENV_DIR/patchwork - git checkout -b production v2.0.1 - cd $HERE + if [ ! -d $VENV_DIR/patchwork ]; then + git clone https://github.com/getpatchwork/patchwork.git $VENV_DIR/patchwork + cd $VENV_DIR/patchwork + git checkout -b production v2.0.1 + cd $HERE + fi export PYTHONPATH=$VENV_DIR/patchwork fi echo echo "== Running flake8 checks..." echo -FLAKE_EXCLUDE="linaro_metrics/migrations" -if echo $VENV_DIR | grep $HERE ; then - # make sure we don't run tests on the venv if its in our source tree - FLAKE_EXCLUDE="$VENV_DIR,$FLAKE_EXCLUDE" -fi -flake8 --ignore=E402 --show-source --exclude=$FLAKE_EXCLUDE ./ +flake8 --show-source ./ echo echo "== Running test suite..." echo -git config --global user.email "citestbot@example.com" -git config --global user.name "ci test bot" +if [ ! -f ~/.gitconfig ] ; then + git config --global user.email "citestbot@example.com" + git config --global user.name "ci test bot" +fi export DJANGO_SETTINGS_MODULE=linaro_metrics.settings export PYTHONPATH=./:./tests:$PYTHONPATH diff --git a/update_commited_patches.py b/update_commited_patches.py index e295641..3f18efc 100755 --- a/update_commited_patches.py +++ b/update_commited_patches.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import fcntl import logging @@ -37,7 +37,7 @@ def _update_commit(project, repo, commit, dryrun): for i, patch in enumerate(patches): if i == 0: patch.state = accepted - patch.commit_ref = commit.id + patch.commit_ref = commit.id.decode() else: patch.state = superseded log.info('Updating patch %s, commit: %s, state: %s', @@ -66,7 +66,7 @@ def _update_project(cb, repo_dir, project, commits, dryrun): cb(project, repo, commit, dryrun) except MemoryError as e: log.error('Unable to process commit(%s) because of size: %s', - commit.id, e) + commit.id, e) except Exception as e: log.error('Unable to process commit(%s): %s', commit.id, e) @@ -117,7 +117,7 @@ if __name__ == '__main__': _update_project( cb, settings.REPO_DIR, p, args.commit_id, args.dryrun ) - except: + except Exception: log.exception('Error updating commits for: %s', p) if not cb_constructor: @@ -126,5 +126,5 @@ if __name__ == '__main__': _update_project( None, settings.REPO_DIR, p, args.commit_id, args.dryrun ) - except: + except Exception: log.exception('Error updating commits for: %s', p) |