import codecs import contextlib import email.utils import functools import logging from django.conf import settings from patchwork.models import Patch, Person, Project from patchwork.parser import parse_patch, subject_check from linaro_metrics.crowd import Crowd from linaro_metrics.sync_teams import get_or_create_person log = logging.getLogger('import_emails') def find_project_by_list_address(mail): addrs = mail.get('To', '') + ',' + mail.get('CC', '') for addr in addrs.split(','): _, addr = email.utils.parseaddr(addr) if addr: try: return Project.objects.get(listemail=addr) except Project.DoesNotExist: pass return None def linaro_find_project(real_find_project, def_project, mail): '''parsemail's parse_email discards patches if find_project returns None. In linaro-metrics we assign them to a default project, so we don't loose the work and can come back later and put it in an appropriate project if needed.''' proj = real_find_project(mail) if proj is None: proj = find_project_by_list_address(mail) if not proj: log.info('Using default project for patch') proj = def_project return proj def str_to_unicode(buff, charset): try_charsets = ['utf-8', 'windows-1252', 'iso-8859-1'] if charset: try: codecs.lookup(charset) try_charsets.insert(0, charset) except LookupError: pass for cset in try_charsets: try: return unicode(buff, cset) except UnicodeDecodeError: pass raise RuntimeError( 'Unable to decode buff(%s) with charset(%s)' % (charset, buff)) def find_comment_and_patch(mail): '''A highly simplified version of parsemail.find_content that just gets the comment from a patch.''' patchbuf = None commentbuf = '' for part in mail.walk(): if part.get_content_maintype() != 'text': continue payload = part.get_payload(decode=True) subtype = part.get_content_subtype() if not isinstance(payload, unicode): payload = str_to_unicode(payload, part.get_content_charset()) if not payload: continue if subtype == 'plain': c = payload if not patchbuf: (patchbuf, c) = parse_patch(payload) if c is not None: commentbuf += c.strip() + '\n' return commentbuf.split('\n'), patchbuf def find_author_submitter(mail, comment_lines): _, auth = email.utils.parseaddr(mail.get('From')) submitter = auth for i, line in enumerate(comment_lines): # only look in first 3 lines for author if i > 2: break if line.startswith('From:'): _, auth = email.utils.parseaddr(line[5:]) return auth, submitter def get_linaro_person(crowd, email): # the author is linaro if crowd.user_valid(email): # we need to make sure the "user" exists so that we can apply team # credits in _patch_saved_callback return get_or_create_person(crowd, email) # the author has linked a non-linaro email to their User p = Person.objects.filter(email__iexact=email) if p.count() == 1 and p[0].user: if crowd.user_valid(p[0].user.email): return p[0] def linaro_parse_mail(crowd, real_parse_mail, mail): '''Only track patches authored or submitted by "linaro" people. We determine this by first finding the author's email. Most of the time this is the simply the "from" address. Occasionally the "from" winds up being the submitter and the author is actually listed in the email's content. Once we have the author email address we check the following logic: * is this author or submitter a valid crowd email? * is there a Person in patchwork by this email address? - if so, is its User part of Linaro? (ie allow tracking of non-linaro emails ''' if mail.get('Subject').startswith('Applied "'): # a maintainer has applied a patch to their tree for the user. eg: # https://patches.linaro.org/patch/99942/ return 0 if subject_check(mail.get('Subject')): # this is a comment to a patch return real_parse_mail(mail) comment_lines, patch = find_comment_and_patch(mail) author, submitter = find_author_submitter(mail, comment_lines) person = get_linaro_person(crowd, author) if person: # we have a linaro authored patch try: Patch.linaro_author = person return real_parse_mail(mail) finally: delattr(Patch, 'linaro_author') else: # see if its a linaro submitter, we'll add the patch but not give # out team credits submitter = get_linaro_person(crowd, submitter) if submitter: Patch.linaro_author = person return real_parse_mail(mail) return 0 @contextlib.contextmanager def monkey_patcher(parser): crwd = Crowd(settings.CROWD_USER, settings.CROWD_PASS, settings.CROWD_URL) def_project, _ = Project.objects.get_or_create( linkname=settings.DEFAULT_PROJECT) with crwd.cached(settings.CROWD_CACHE): orig_find = parser.find_project_by_header orig_parse = parser.parse_mail try: parser.find_project_by_header = functools.partial( linaro_find_project, orig_find, def_project) parser.parse_mail = functools.partial( linaro_parse_mail, crwd, orig_parse) yield finally: parser.parse_mail = orig_parse parser.find_project_by_header = orig_find