Cache `tools/lint.py` results.

Change-Id: Id9ca69583796e202f68dad847ef640824146fd59

2 files changed, 95 insertions, 11 deletions

diff --git a/tools/lint.py b/tools/lint.py
index 14db8b99..b454bfbb 100755
--- a/tools/lint.py
+++ b/tools/lint.py
@@ -27,7 +27,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import argparse
+import hashlib
 import multiprocessing
+import os
+import pickle
 import re
 import signal
 import subprocess
@@ -49,7 +52,9 @@ def BuildOptions():
       description =
       '''This tool lints C++ files and produces a summary of the errors found.
       If no files are provided on the command-line, all C++ source files in the
-      repository are processed.''',
+      repository are processed.
+      Results are cached to speed up the process.
+      ''',
       # Print default values.
       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
   parser.add_argument('files', nargs = '*')
@@ -59,13 +64,16 @@ def BuildOptions():
                       help='''Runs the tests using N jobs. If the option is set
                       but no value is provided, the script will use as many jobs
                       as it thinks useful.''')
+  parser.add_argument('--no-cache',
+                      action='store_true', default=False,
+                      help='Do not use cached lint results.')
   return parser.parse_args()
 
 
 
 __lint_results_lock__ = multiprocessing.Lock()
 
-# Returns the number of errors in the file linted.
+# Returns a tuple (filename, number of lint errors).
 def Lint(filename, progress_prefix = ''):
   command = ['cpplint.py', filename]
   process = subprocess.Popen(command,
@@ -98,7 +106,7 @@ def Lint(filename, progress_prefix = ''):
       if retcode != None: break;
 
     if retcode == 0:
-      return 0
+      return (filename, 0)
 
     # Return the number of errors in this file.
     res = re.search('\d+$', status_line)
@@ -108,7 +116,8 @@ def Lint(filename, progress_prefix = ''):
         progress_prefix + 'Total errors found in %s : %d' % (filename, n_errors)
     printer.PrintOverwritableLine(status_line, type = printer.LINE_TYPE_LINTER)
     printer.EnsureNewLine()
-    return n_errors
+
+    return (filename, n_errors)
 
 
 # The multiprocessing map_async function does not allow passing multiple
@@ -122,8 +131,24 @@ def LintWrapper(args):
     sys.exit(1)
 
 
+def ShouldLint(filename, cached_results):
+  filename = os.path.realpath(filename)
+  if filename not in cached_results:
+    return True
+  with open(filename, 'rb') as f:
+    file_hash = hashlib.md5(f.read()).hexdigest()
+  return file_hash != cached_results[filename]
+
+
 # Returns the total number of errors found in the files linted.
-def LintFiles(files, jobs = 1, progress_prefix = ''):
+# `cached_results` must be a dictionary, with the format:
+#     { 'filename': file_hash, 'other_filename': other_hash, ... }
+# If not `None`, `cached_results` is used to avoid re-linting files, and new
+# results are stored in it.
+def LintFiles(files,
+              jobs = 1,
+              progress_prefix = '',
+              cached_results = None):
   if not IsCppLintAvailable():
     print(
       printer.COLOUR_RED + \
@@ -134,6 +159,20 @@ def LintFiles(files, jobs = 1, progress_prefix = ''):
       printer.NO_COLOUR)
     return -1
 
+  # Filter out directories.
+  files = filter(os.path.isfile, files)
+
+  # Filter out files for which we have a cached correct result.
+  if cached_results is not None and len(cached_results) != 0:
+    n_input_files = len(files)
+    files = filter(lambda f: ShouldLint(f, cached_results), files)
+    n_skipped_files = n_input_files - len(files)
+    if n_skipped_files != 0:
+      printer.Print(
+        progress_prefix +
+        'Skipping %d correct files that were already processed.' %
+        n_skipped_files)
+
   pool = multiprocessing.Pool(jobs)
   # The '.get(9999999)' is workaround to allow killing the test script with
   # ctrl+C from the shell. This bug is documented at
@@ -148,7 +187,17 @@ def LintFiles(files, jobs = 1, progress_prefix = ''):
   except KeyboardInterrupt:
     pool.terminate()
     sys.exit(1)
-  n_errors = sum(results)
+
+  n_errors = sum(map(lambda (filename, errors): errors, results))
+
+  if cached_results is not None:
+    for filename, errors in results:
+      if errors == 0:
+        with open(filename, 'rb') as f:
+          filename = os.path.realpath(filename)
+          file_hash = hashlib.md5(f.read()).hexdigest()
+          cached_results[filename] = file_hash
+
 
   printer.PrintOverwritableLine(
       progress_prefix + 'Total errors found: %d' % n_errors)
@@ -166,7 +215,8 @@ def is_linter_input(filename):
   # lint all C++ files.
   return CPP_EXT_REGEXP.search(filename) != None
 
-def GetDefaultTrackedFiles():
+
+def GetDefaultFilesToLint():
   if git.is_git_repository_root(config.dir_root):
     default_tracked_files = git.get_tracked_files().split()
     default_tracked_files = filter(is_linter_input, default_tracked_files)
@@ -177,14 +227,48 @@ def GetDefaultTrackedFiles():
                   printer.NO_COLOUR)
     return 1, []
 
+
+cached_results_pkl_filename = \
+  os.path.join(config.dir_tools, '.cached_lint_results.pkl')
+
+
+def ReadCachedResults():
+  cached_results = {}
+  if os.path.isfile(cached_results_pkl_filename):
+    with open(cached_results_pkl_filename, 'rb') as pkl_file:
+      cached_results = pickle.load(pkl_file)
+  return cached_results
+
+
+def CacheResults(results):
+  with open(cached_results_pkl_filename, 'wb') as pkl_file:
+    pickle.dump(results, pkl_file)
+
+
+def RunLinter(files, jobs=1, progress_prefix='', cached=True):
+  results = {} if not cached else ReadCachedResults()
+
+  rc = LintFiles(files,
+                 jobs=jobs,
+                 progress_prefix=progress_prefix,
+                 cached_results=results)
+
+  CacheResults(results)
+  return rc
+
+
+
 if __name__ == '__main__':
   # Parse the arguments.
   args = BuildOptions()
 
   files = args.files
   if not files:
-    retcode, files = GetDefaultTrackedFiles()
+    retcode, files = GetDefaultFilesToLint()
     if retcode:
       sys.exit(retcode)
-  retcode = LintFiles(files, jobs = args.jobs)
+
+  cached = not args.no_cache
+  retcode = RunLinter(files, jobs=args.jobs, cached=cached)
+
   sys.exit(retcode)
diff --git a/tools/test.py b/tools/test.py
index 48f3166c..34c8983c 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -319,10 +319,10 @@ def RunCommand(command, environment_options = None):
 
 
 def RunLinter():
-  rc, default_tracked_files = lint.GetDefaultTrackedFiles()
+  rc, default_tracked_files = lint.GetDefaultFilesToLint()
   if rc:
     return rc
-  return lint.LintFiles(map(lambda x: join(dir_root, x), default_tracked_files),
+  return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files),
                         jobs = args.jobs, progress_prefix = 'cpp lint: ')