gitrepo.py

import datetime
import logging
import os
import subprocess
import tempfile

import dulwich.repo

from django.conf import settings
from patchwork.parser import parse_patch

log = logging.getLogger('gitrepo')


def croncmd(args, cwd='./', timeout=None, get_fail_logger=None):
    if timeout:
        args = ['timeout', str(timeout)] + args

    with tempfile.SpooledTemporaryFile(max_size=4096) as f:
        try:
            subprocess.check_call(args, cwd=cwd, stdout=f, stderr=f)
            if log.level == logging.DEBUG:
                log.debug('Results of cmd(%s): %r', cwd, args)
                f.seek()
                log.debug('COMBINED_OUTPUT\n%s', f.read())
        except subprocess.CalledProcessError as e:
            logger = log.error
            if get_fail_logger:
                logger = get_fail_logger()
            logger('Unable to run command(%s): %r', cwd, args)
            if timeout and e.returncode == 124:
                logger('Command timed out')
            f.seek(0)
            if e.output:
                logger('STDOUT:\n%s', e.output)
                logger('STDERR:\n%s', f.read())
            else:
                logger('COMBINED OUTPUT:\n%s', f.read())
            raise


class Repo(object):
    '''Our patchwork deployments try and automatically update patches by
    looking at the change history on a repository. This class provides a
    simple interface to analyze new commits
    '''
    def __init__(self, repo_dir, name, scm_url):
        self.path = os.path.join(repo_dir, name)
        self.scm_url = scm_url
        self._repo = None

    @property
    def repo(self):
        if not self._repo:
            self._repo = dulwich.repo.Repo(self.path)
        return self._repo

    def __getitem__(self, key):
        return self.repo[key]

    def _clone(self):
        croncmd(['git', 'clone', '--mirror', self.scm_url, self.path])

    def _pull(self):
        fail_file = os.path.join(self.path, 'failures')

        def get_fail_logger():
            with open(fail_file, 'a+') as f:
                f.write('failed at: %s\n' % str(datetime.datetime.now()))
                f.seek(0)
                for count, line in enumerate(f, 1):
                    if count > 3:
                        return log.error
            return log.info

        timeout = str(getattr(settings, 'REPO_TIMEOUT', 120))
        croncmd(
            ['git', 'remote', '-v', 'update', '--prune'], self.path, timeout,
            get_fail_logger)
        if os.path.exists(fail_file):
            # clean out old failures, now that we've succeeded
            os.unlink(fail_file)

    def update(self):
        if not os.path.exists(self.path):
            self._clone()
        else:
            try:
                self._pull()
            except subprocess.CalledProcessError:
                # We've already logged the exception. Code can continue to run
                # because its just going to call process_unchecked_commits and
                # essentially be a no-op
                pass

    def process_unchecked_commits(self, save_state=True):
        last_commit = os.path.join(self.path, 'patchwork-last-commit')
        if os.path.exists(last_commit):
            with open(last_commit) as f:
                start_at = f.read().strip()
        else:
            start_at = 'HEAD~100'

        log.debug('looking for commits since: %s', start_at)
        args = ['git', 'rev-list', '--reverse', start_at + '..HEAD']
        with open('/dev/null', 'w') as f:
            rc = subprocess.call(
                ['git', 'show', start_at], cwd=self.path, stdout=f, stderr=f)
        if rc != 0:
            # we may have had a branch who's history was re-written
            # just try and get changes for past day
            args = ['git', 'rev-list', '--reverse',
                    '--since', '1 day ago', 'HEAD']

        try:
            for x in subprocess.check_output(args, cwd=self.path).split('\n'):
                if x:
                    yield self.repo[x]
                    start_at = x
        finally:
            if save_state:
                with open(last_commit, 'w') as f:
                    f.write(start_at)

    def get_patch(self, commit):
        args = ['git', 'show', '--format=format:%e', '-M', str(commit.id)]
        patch = subprocess.check_output(args, cwd=self.path)
        # the patchwork parser code operates character by character so we must
        # convert to unicode so it can be handled properly
        patch = patch.decode('utf-8', errors='replace')
        # Don't try and process >5Mb patches, they flood the server
        if len(patch) > 5000000:
            raise MemoryError('patch too large to process: %d' % len(patch))
        patch = parse_patch(patch)[0]
        if patch is None:
            # happens for binary only patches like:
            # https://git.linaro.org/uefi/OpenPlatformPkg.git/commit/ \
            # ?id=7ab4bb34b2464a2491868264bdf2931f2acd6452
            patch = ''
        return patch