gitrepo: add logic to only warn after 4 failures

We often find upstream repos are periodically unreachable. This allows
us to fail a few times before we actually log an error. (Error logs
result in cron email so its good to make sure they are real issues)

Change-Id: I8f1a0ae19b36fbd1af4ef26b3d68fa85501577c5

1 files changed, 33 insertions, 8 deletions

diff --git a/gitrepo.py b/gitrepo.py
index ef496ce..b1464f7 100644
--- a/gitrepo.py
+++ b/gitrepo.py
@@ -1,3 +1,4 @@
+import datetime
 import logging
 import os
 import subprocess
@@ -11,7 +12,7 @@ from patchwork.parser import parse_patch
 log = logging.getLogger('gitrepo')
 
 
-def croncmd(args, cwd='./', timeout=None):
+def croncmd(args, cwd='./', timeout=None, get_fail_logger=None):
     if timeout:
         args = ['timeout', str(timeout)] + args
 
@@ -23,15 +24,18 @@ def croncmd(args, cwd='./', timeout=None):
                 f.seek()
                 log.debug('COMBINED_OUTPUT\n%s', f.read())
         except subprocess.CalledProcessError as e:
-            log.error('Unable to run command(%s): %r', cwd, args)
+            logger = log.error
+            if get_fail_logger:
+                logger = get_fail_logger()
+            logger('Unable to run command(%s): %r', cwd, args)
             if timeout and e.returncode == 124:
-                log.error('Command timed out')
+                logger('Command timed out')
             f.seek(0)
             if e.output:
-                log.error('STDOUT:\n%s', e.output)
-                log.error('STDERR:\n%s', f.read())
+                logger('STDOUT:\n%s', e.output)
+                logger('STDERR:\n%s', f.read())
             else:
-                log.error('COMBINED OUTPUT:\n%s', f.read())
+                logger('COMBINED OUTPUT:\n%s', f.read())
             raise
 
 
@@ -58,15 +62,36 @@ class Repo(object):
         croncmd(['git', 'clone', '--mirror', self.scm_url, self.path])
 
     def _pull(self):
+        fail_file = os.path.join(self.path, 'failures')
+
+        def get_fail_logger():
+            with open(fail_file, 'a+') as f:
+                f.write('failed at: %s\n' % str(datetime.datetime.now()))
+                f.seek(0)
+                for count, line in enumerate(f, 1):
+                    if count > 3:
+                        return log.error
+            return log.info
+
         timeout = str(getattr(settings, 'REPO_TIMEOUT', 120))
         croncmd(
-            ['git', 'remote', '-v', 'update', '--prune'], self.path, timeout)
+            ['git', 'remote', '-v', 'update', '--prune'], self.path, timeout,
+            get_fail_logger)
+        if os.path.exists(fail_file):
+            # clean out old failures, now that we've succeeded
+            os.unlink(fail_file)
 
     def update(self):
         if not os.path.exists(self.path):
             self._clone()
         else:
-            self._pull()
+            try:
+                self._pull()
+            except subprocess.CalledProcessError:
+                # We've already logged the exception. Code can continue to run
+                # because its just going to call process_unchecked_commits and
+                # essentially be a no-op
+                pass
 
     def process_unchecked_commits(self, save_state=True):
         last_commit = os.path.join(self.path, 'patchwork-last-commit')