path: root/apps/patchwork/utils.py

# Patchwork - automated patch tracking system
# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
#
# This file is part of the Patchwork package.
#
# Patchwork is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# Patchwork is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Patchwork; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

import StringIO
import atexit
import chardet
import datetime
import os
import re
import subprocess

from difflib import SequenceMatcher
from django.conf import settings
from django.shortcuts import get_object_or_404
from patchwork.models import Bundle, BundlePatch, Patch, Person, State
from patchwork.parser import hash_patch, parse_patch


# Timeout for git operations.
GIT_TIMEOUT = 120 * 60
# FIXME This is a temporary solution, a workardound for bug 1017933.
# Once that bug is fixed or in the process, this should be removed.
REPO_BRANCHES = {'cpufreq': 'origin/next'}

DEVNULL = open(os.devnull, "w")
atexit.register(DEVNULL.close)

# Default date format.
DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f'


class GitException(Exception):
    """General class for git exceptions."""


class FailedToFetchGitRepo(GitException):
    """Failed to fetch the git repo of a project."""


class FailedToCheckoutGitRepo(GitException):
    """Failed to perform a checkout operation on a git repository."""


class GitTimeout(GitException):
    """Timeout when fetching the git repo of a project."""


def get_patch_ids(d, prefix='patch_id'):
    ids = []

    for (k, v) in d.items():
        a = k.split(':')
        if len(a) != 2:
            continue
        if a[0] != prefix:
            continue
        if not v:
            continue
        ids.append(a[1])

    return ids


def parse_name_and_email(text):
    # tuple of (regex, fn)
    #  - where fn returns a (name, email) tuple from the match groups resulting
    #    from re.match().groups()
    from_res = [
        # for "Firstname Lastname" <example@example.com> style addresses
        (re.compile('"?(.*?)"?\s*<([^>]+)>'), (lambda g: (g[0], g[1]))),

        # for example@example.com (Firstname Lastname) style addresses
        (re.compile('"?(.*?)"?\s*\(([^\)]+)\)'), (lambda g: (g[1], g[0]))),

        # everything else
        (re.compile('(.*)'), (lambda g: (None, g[0]))),
    ]

    for regex, fn in from_res:
        match = regex.match(text)
        if match:
            name, email = fn(match.groups())
            if name is not None:
                name = name.strip()
            if email is not None:
                email = email.strip()
            return name, email
    return None, None


class Order(object):
    order_map = {
        'date':         'date',
        'name':         'name',
        'state':        'state__ordering',
        'submitter':    'submitter__name',
        'delegate':     'delegate__username',
    }
    default_order = ('date', True)

    def __init__(self, str=None, editable=False):
        self.reversed = False
        self.editable = editable
        (self.order, self.reversed) = self.default_order

        if self.editable:
            return

        if str is None or str == '':
            return

        reversed = False
        if str[0] == '-':
            str = str[1:]
            reversed = True

        if str not in self.order_map.keys():
            return

        self.order = str
        self.reversed = reversed

    def __str__(self):
        str = self.order
        if self.reversed:
            str = '-' + str
        return str

    def name(self):
        return self.order

    def reversed_name(self):
        if self.reversed:
            return self.order
        else:
            return '-' + self.order

    def query(self):
        q = self.order_map[self.order]
        if self.reversed:
            q = '-' + q
        return q


bundle_actions = ['create', 'add', 'remove']


def set_bundle(user, project, action, data, patches, context):
    # set up the bundle
    bundle = None
    if action == 'create':
        bundle_name = data['bundle_name'].strip()
        if not bundle_name:
            return ['No bundle name was specified']

        bundle = Bundle(owner=user, project=project, name=bundle_name)
        bundle.save()
        context.add_message("Bundle {0} created".format(bundle.name))
    elif action == 'add':
        bundle = get_object_or_404(Bundle, id=data['bundle_id'])
    elif action == 'remove':
        bundle = get_object_or_404(Bundle, id=data['removed_bundle_id'])

    if not bundle:
        return ['no such bundle']

    for patch in patches:
        if action == 'create' or action == 'add':
            bundlepatch_count = BundlePatch.objects.filter(bundle=bundle,
                                                           patch=patch).count()
            if bundlepatch_count == 0:
                bundle.append_patch(patch)
                context.add_message("Patch '{0}' added to bundle {1}".format(
                                    patch.name, bundle.name))
            else:
                context.add_message("Patch '{0}' already in bundle {1}".format(
                                    patch.name, bundle.name))

        elif action == 'remove':
            try:
                bp = BundlePatch.objects.get(bundle=bundle, patch=patch)
                bp.delete()
                context.add_message("Patch '{0}' removed from bundle "
                                    "{1}\n".format(patch.name, bundle.name))
            except Exception:
                pass

    bundle.save()

    return []


def get_child_pids(pid):
    """Recursively get the PIDs of the given process' children."""
    pids = []
    stdout, _ = subprocess.Popen(
        ['pgrep', '-P', str(pid)], stdout=subprocess.PIPE).communicate()
    children = stdout.split()
    while children:
        pids.extend(children)
        new_children = []
        for pid in children:
            args = ['pgrep', '-P', str(pid)]
            stdout, _ = subprocess.Popen(
                args, stdout=subprocess.PIPE).communicate()
            new_children.extend(stdout.split())
        children = new_children
    return pids


def _run(args, cwd=None, timeout=None, stdout=False):
    if timeout:
        args = ['timeout', str(timeout)] + args
    try:
        if stdout:
            return subprocess.check_output(args, cwd=cwd)
        else:
            subprocess.check_call(
                args, cwd=cwd, stdout=DEVNULL, stderr=DEVNULL)
    except subprocess.CalledProcessError:
        raise GitException('Failed to run from(%s):' % cwd + ' '.join(args))


def _smart_pull(root, timeout):
    '''do a pull that's smart enough to handle history re-writes'''
    _run(['git', 'fetch'], root, timeout)
    commit = _run(
        ['git', 'merge-base', 'origin', 'master'], root, timeout, True)
    _run(['git', 'reset', '--hard', commit.strip()], root, timeout)
    _run(['git', 'pull'], root, timeout)


# We run git as a subprocess here instead of using a library like dulwich or
# git-python because dulwich had issues when getting diffs for certain
# commits and the version of git-python that is packaged in most distros is
# not suitable, whereas its latest version has a bunch of non-packaged
# dependencies.
def ensure_source_checkout_for_project(project, timeout=GIT_TIMEOUT):
    """Ensures that we have the code for the project we are working on.

    If there is no code in the PATCHWORK_GIT_REPOS_DIR, it performs a 'clone'
    operation, otherwise we try to 'pull' the latest changes.

    :param project: the project we are working on
    :param timeout: the timeout for the git operation, defaults to GIT_TIMEOUT
    """
    root = os.path.join(settings.PATCHWORK_GIT_REPOS_DIR, project.linkname)

    if os.path.exists(root):
        print "Pulling changes from {0}".format(project.name)
        _smart_pull(root, timeout)
    else:
        print "Cloning repository {0}".format(project.source_tree)
        args = ['git', 'clone']
        branch = REPO_BRANCHES.get(project.name)
        if branch:
            args.extend(['-b', branch])
        args.extend([project.source_tree, root])
        _run(args, None, timeout)
    return root


def get_commits_to_parse(root, start_at):
    if not start_at:
        start_at = 'HEAD~1000'
    args = ['git', 'rev-list', '--reverse', start_at + '..HEAD']
    rc = subprocess.call(
        ['git', 'show', start_at], cwd=root, stdout=DEVNULL, stderr=DEVNULL)
    if rc != 0:
        # we may have had a branch who's history was re-written
        # just try and get changes for past day
        args = ['git', 'rev-list', '--reverse', '--since', '1 day ago', 'HEAD']
    try:
        commits = subprocess.check_output(args, cwd=root).split('\n')
    except subprocess.CalledProcessError:
        raise GitException("Error retrieving revisions list.")
    return [x for x in commits if len(x)]


def get_diff_for_commit(root, commit_id):
    proc = subprocess.Popen(
        ['git', 'show', commit_id], cwd=root, stdout=subprocess.PIPE,
        stderr=DEVNULL, bufsize=4096)

    stdout = ""
    while True:
        for line in iter(proc.stdout.readline, b''):
            stdout += line
        if proc.poll() is not None:
            break

    if proc.returncode != 0:
        print ("Error retrieving diff for commit {0}".format(commit_id))
        return None

    try:
        diff = stdout.decode('utf-8')
    except UnicodeDecodeError:
        try:
            # chardet.detect is rather slow so we only use it when we fail
            # to decode from utf-8.
            encoding = chardet.detect(stdout)['encoding']
            diff = stdout.decode(encoding)
        except UnicodeDecodeError:
            print ("Skipping commit {0} as we can't find the appropriate "
                   "charset to decode it".format(commit_id))
            return None

    patch, _ = parse_patch(diff)
    return patch


def get_diff_for_linaro_commit(root, commit_id, crowd, db):
    """Gets a diff from a commit id for a Linaro made change.

    If the author or the committer of the commit is not a Linaro user, then
    the commit will not be considered.

    This is a slightly different implementation than `get_diff_for_commit`,
    where here it checks if the user is a valid Linaro account before taking
    in account the patch.

    :param root: The path on the file system where the repository is stored.
    :param commit_id: The commmit ID to parse.
    :param crowd: A Crowd object used to determine if the user is from Linaro.
    :param db: A PatchworkDB object used as a cache for users.
    """
    stdout = ""
    patch = None
    author = None
    committer = None

    # We create a custom view on the commit id diff. We extract the author
    # and committer emails, and check if they are from Linaro people, otherwise
    # we do not even consider the commit and move one.
    # Most of the time patchwork/patchmetrics spends a lot of time calculating
    # diffs for patches that might not even be from Linaro people.
    # The first format prints on stdout the author and commiter emails.
    # The rest of the format string is exactly as what is shown with a simple
    # `git show COMMIT_ID`.
    format_string = "%ae%n%ce%n"
    format_string += "commit %H%n"
    format_string += "Author: %aN <%aE>%n"
    format_string += "Date: %ad%n"

    cmd_args = ['git', 'show', '--format=format:{0}'.format(format_string),
                '-M', commit_id]
    proc = subprocess.Popen(cmd_args, cwd=root, bufsize=4096,
                            stdout=subprocess.PIPE, stderr=DEVNULL)

    while True:
        for line in iter(proc.stdout.readline, b''):
            stdout += line
        if proc.poll() is not None:
            break

    if proc.returncode != 0:
        print ("Error retrieving diff for commit {0}".format(commit_id))
    else:
        # Since with big diffs a lot of time is spent decoding the output from
        # the git command, we first read two lines from the output (author and
        # commiter emails), perform validations on those, and in case one of
        # them is a valid Linaro user we continue with output decoding and
        # diff parsing.
        string_file = StringIO.StringIO(stdout)

        author = string_file.readline().strip().decode('utf-8')
        committer = string_file.readline().strip().decode('utf-8')

        valid_patch = False
        if author == committer:
            valid_patch |= is_linaro_user(author, db, crowd)
        else:
            valid_patch |= is_linaro_user(author, db, crowd)
            valid_patch |= is_linaro_user(committer, db, crowd)

        if valid_patch:
            try:
                diff = stdout.decode('utf-8')
            except UnicodeDecodeError:
                try:
                    # chardet.detect is rather slow so we only use it when we
                    # fail # to decode from utf-8.
                    encoding = chardet.detect(stdout)['encoding']
                    diff = stdout.decode(encoding)
                except UnicodeDecodeError:
                    print ("Skipping commit {0} as we can't find the "
                           "appropriate charset to decode "
                           "it".format(commit_id))
                    return None

            # This is a double action, but it is still faster then decoding
            # everytime the output even when the patch does not come from a
            # Linaro user.
            string_file = StringIO.StringIO(diff)
            # We need to pass the StringIO object pointing to the 3rd line
            # since the first two have already been read, and they contains
            # information not needed for the patch.
            for i in range(0, 2):
                string_file.readline()
            patch, _ = parse_patch(string_file)

    return patch


def is_linaro_user(user, db, crowd):
    """Verifies if a user is a valid Linaro account.

    :param user: The email address to verify.
    :param db: A PatchworkDB instance for the cache backend.
    :param crowd: A Crowd instance to retrieve user data if necessary.
    """
    cached_user = db.get_user(user)

    is_valid = False
    if cached_user:
        is_valid |= check_cached_user_validity(cached_user, db, crowd)
    else:
        # XXX: The first time this runs, it might hit the Crowd server hard,
        # since none of the users will be cached.
        valid_user = crowd.is_valid_user(user)
        is_valid |= valid_user
        db.insert_user(user, valid_user, datetime.datetime.now())

    return is_valid


def check_cached_user_validity(cached_user, db, crowd):
    """Checks the user timestamp, and in case re-checks user validity.

    :param chached_user: The user from the local cache.
    :param db: A PatchworkDB instance for the cache backend.
    :param crowd: A Crowd instance to update user data if necessary.
    """
    date_inserted = cached_user[2]
    today = datetime.datetime.now()
    delta = today - date_inserted

    valid_user = cached_user[1]
    if delta.days >= settings.CROWD_USERS_DB_EXPIRY:
        valid_user = crowd.is_valid_user(cached_user[0])
        db.update_user(cached_user[0], valid_user, today)

    return valid_user


def get_patches_matching_commit(project, commit_diff, commit_msg,
                                commit_author):
    """Return the Patch objects that are likely matches to the given commit.

    Patches that are either Accepted or Superseded are never included in the
    results.

    :param project: The Project to which this patch applies.
    :param commit_diff: A string with the diff that was ultimately committed.
    :param commit_msg: The commit message, as a string.
    :param commit_author: The commit author, including the email address, also
                          as a string.
    """
    pending_patches = Patch.objects.filter(project=project).exclude(
        state__in=State.objects.filter(name__in=['Accepted', 'Superseded']))
    # First check if we have patches with a diff hash identical to that of the
    # diff that was ultimately committed.
    patches = list(pending_patches.filter(
        hash=hash_patch(commit_diff).hexdigest()))
    if len(patches) > 0:
        return patches

    # No hashes match that of the committed changes, so lookup using the
    # author's email address and the patch name.
    commit_title = commit_msg.split('\n')[0]
    _, author_email = parse_name_and_email(commit_author)
    if author_email is None:
        # No author found, nothing we can do about it.
        return []

    try:
        author = Person.objects.get(email=author_email)
    except Person.DoesNotExist:
        return []

    # Now lookup patches from the same author that have the exact same name as
    # that of the commit.
    patches = list(pending_patches.filter(name=commit_title, author=author))
    if len(patches) > 0:
        return patches

    patches = []
    for patch in pending_patches.filter(project=project, author=author):
        # Be conservative and use 0.9 as the similarity ratio between the
        # commit's title and content to prevent false-positives.
        # TODO: It'd be nice to factor this out into a separate method for
        # easier testing.
        if SequenceMatcher(None, patch.name, commit_title).ratio() > 0.9:
            matcher = SequenceMatcher(None, patch.content, commit_diff)
            if matcher.ratio() > 0.9:
                # Some commits may be just reverting a previous one and in
                # these cases the title and content ratios will be high and
                # pass the two tests above, so we do one last check here to
                # avoid false-positives in these cases.
                patch_title = re.sub(r'\[[^\]]*\]', '', patch.name).strip()
                commit_title = re.sub(r'\[[^\]]*\]', '', commit_title).strip()
                if (patch_title.startswith('Revert')
                        and not commit_title.startswith('Revert')):
                    continue
                patches.append(patch)
    return patches