Refactor idiff for imagerepo.json capability. (SciTools#2166)

* Refactor idiff for imagerepo.json capability.

Author: bjlittle

.gitignore

@@ -33,6 +33,9 @@ lib/iris/std_names.py
 lib/iris/tests/result_image_comparison/
 iris_image_test_output/
 
+# Iris test lock file
+lib/iris/tests/results/imagerepo.lock
+
 # Pydev/Eclipse files
 .project
 .pydevproject

conda-requirements.txt

@@ -20,6 +20,7 @@ nose
 pep8=1.5.7
 sphinx
 iris_sample_data
+filelock
 
 # Optional iris dependencies
 nc_time_axis

lib/iris/tests/idiff.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# (C) British Crown Copyright 2010 - 2015, Met Office
+# (C) British Crown Copyright 2010 - 2016, Met Office
 #
 # This file is part of Iris.
 #
@@ -25,17 +25,54 @@
 from __future__ import (absolute_import, division, print_function)
 from six.moves import (filter, input, map, range, zip)  # noqa
 
+import argparse
+import codecs
+import contextlib
+from glob import glob
+import hashlib
+import json
 import os.path
-import shutil
 import sys
+import warnings
 
+from PIL import Image
+import filelock
 import matplotlib.pyplot as plt
 import matplotlib.image as mimg
+import matplotlib.testing.compare as mcompare
 import matplotlib.widgets as mwidget
-
-
-def diff_viewer(expected_fname, result_fname, diff_fname):
-    plt.figure(figsize=(16, 16))
+import requests
+
+# Force iris.tests to use the ```tkagg``` backend by using the '-d'
+# command-line argument as idiff is an interactive tool that requires a
+# gui interface.
+sys.argv.append('-d')
+import iris.tests
+import iris.util as iutil
+
+
+_POSTFIX_DIFF = '-failed-diff.png'
+_POSTFIX_JSON = os.path.join('results', 'imagerepo.json')
+_POSTFIX_LOCK = os.path.join('results', 'imagerepo.lock')
+_PREFIX_URI = 'https://scitools.github.io/test-images-scitools/image_files'
+_TIMEOUT = 30
+_TOL = 0
+
+
+@contextlib.contextmanager
+def temp_png(suffix=''):
+    if suffix:
+        suffix = '-{}'.format(suffix)
+    fname = iutil.create_temp_filename(suffix+'.png')
+    try:
+        yield fname
+    finally:
+        os.remove(fname)
+
+
+def diff_viewer(repo, key, repo_fname,
+                expected_fname, result_fname, diff_fname):
+    plt.figure(figsize=(14, 12))
     plt.suptitle(os.path.basename(expected_fname))
     ax = plt.subplot(221)
     ax.imshow(mimg.imread(expected_fname))
@@ -44,49 +81,126 @@ def diff_viewer(expected_fname, result_fname, diff_fname):
     ax = plt.subplot(223, sharex=ax, sharey=ax)
     ax.imshow(mimg.imread(diff_fname))
 
+    # Determine the new image hash.png name.
+    with open(result_fname, 'rb') as fi:
+        sha1 = hashlib.sha1(fi.read())
+    result_dir = os.path.dirname(result_fname)
+    fname = sha1.hexdigest() + '.png'
+    uri = os.path.join(_PREFIX_URI, fname)
+    hash_fname = os.path.join(result_dir, fname)
+
     def accept(event):
-        # removes the expected result, and move the most recent result in
-        print('ACCEPTED NEW FILE: %s' % (os.path.basename(expected_fname), ))
-        os.remove(expected_fname)
-        shutil.copy2(result_fname, expected_fname)
+        if uri not in repo[key]:
+            # Ensure to maintain strict time order where the first uri
+            # associated with the repo key is the oldest, and the last
+            # uri is the youngest
+            repo[key].append(uri)
+            # Update the image repo.
+            with open(repo_fname, 'wb') as fo:
+                json.dump(repo, fo, indent=4, sort_keys=True)
+            os.rename(result_fname, hash_fname)
+            msg = 'ACCEPTED:  {} -> {}'
+            print(msg.format(os.path.basename(result_fname),
+                             os.path.basename(hash_fname)))
+        else:
+            msg = 'DUPLICATE: {} -> {} (ignored)'
+            print(msg.format(os.path.basename(result_fname),
+                             os.path.basename(hash_fname)))
+            os.remove(result_fname)
         os.remove(diff_fname)
         plt.close()
 
     def reject(event):
-        print('REJECTED: %s' % (os.path.basename(expected_fname), ))
+        if uri not in repo[key]:
+            print('REJECTED:  {}'.format(os.path.basename(result_fname)))
+        else:
+            msg = 'DUPLICATE: {} -> {} (ignored)'
+            print(msg.format(os.path.basename(result_fname),
+                             os.path.basename(hash_fname)))
+        os.remove(result_fname)
+        os.remove(diff_fname)
         plt.close()
 
-    ax_accept = plt.axes([0.7, 0.05, 0.1, 0.075])
-    ax_reject = plt.axes([0.81, 0.05, 0.1, 0.075])
-    bnext = mwidget.Button(ax_accept, 'Accept change')
-    bnext.on_clicked(accept)
-    bprev = mwidget.Button(ax_reject, 'Reject')
-    bprev.on_clicked(reject)
+    def skip(event):
+        print('SKIPPED:   {}'.format(os.path.basename(result_fname)))
+        os.remove(diff_fname)
+        plt.close()
 
+    ax_accept = plt.axes([0.59, 0.05, 0.1, 0.075])
+    ax_reject = plt.axes([0.7, 0.05, 0.1, 0.075])
+    ax_skip = plt.axes([0.81, 0.05, 0.1, 0.075])
+    baccept = mwidget.Button(ax_accept, 'Accept')
+    baccept.on_clicked(accept)
+    breject = mwidget.Button(ax_reject, 'Reject')
+    breject.on_clicked(reject)
+    bskip = mwidget.Button(ax_skip, 'Skip')
+    bskip.on_clicked(skip)
     plt.show()
 
 
-def step_over_diffs():
-    import iris.tests
-    image_dir = os.path.join(os.path.dirname(iris.tests.__file__),
-                             'results', 'visual_tests')
-    diff_dir = os.path.join(os.path.dirname(iris.tests.__file__),
-                            'result_image_comparison')
-
-    for expected_fname in sorted(os.listdir(image_dir)):
-        result_path = os.path.join(diff_dir, 'result-' + expected_fname)
-        diff_path = result_path[:-4] + '-failed-diff.png'
-
-        # if the test failed, there will be a diff file
-        if os.path.exists(diff_path):
-            expected_path = os.path.join(image_dir, expected_fname)
-            diff_viewer(expected_path, result_path, diff_path)
+def step_over_diffs(result_dir):
+    processed = False
+    dname = os.path.dirname(iris.tests.__file__)
+    lock = filelock.FileLock(os.path.join(dname, _POSTFIX_LOCK))
+
+    # Remove old image diff results.
+    target = os.path.join(result_dir, '*{}'.format(_POSTFIX_DIFF))
+    for fname in glob(target):
+        os.remove(fname)
+
+    with lock.acquire(timeout=_TIMEOUT):
+        # Load the imagerepo.
+        repo_fname = os.path.join(dname, _POSTFIX_JSON)
+        with open(repo_fname, 'rb') as fi:
+            repo = json.load(codecs.getreader('utf-8')(fi))
+
+        target = os.path.join(result_dir, 'result-*.png')
+        for result_fname in sorted(glob(target)):
+            # We only care about PNG images.
+            try:
+                im = Image.open(result_fname)
+                if im.format != 'PNG':
+                    continue
+            except IOError:
+                continue
+            key = os.path.splitext('-'.join(result_fname.split('-')[1:]))[0]
+            try:
+                uri = repo[key][0]
+            except KeyError:
+                wmsg = 'Ignoring unregistered test result {!r}.'
+                warnings.warn(wmsg.format(key))
+                continue
+            with temp_png(key) as expected_fname:
+                processed = True
+                resource = requests.get(uri)
+                with open(expected_fname, 'wb') as fo:
+                    fo.write(resource.content)
+                mcompare.compare_images(expected_fname, result_fname, tol=_TOL)
+                diff_fname = os.path.splitext(result_fname)[0] + _POSTFIX_DIFF
+                diff_viewer(repo, key, repo_fname, expected_fname,
+                            result_fname, diff_fname)
+        if not processed:
+            msg = '\n{}: There are no iris test result images to process.\n'
+            print(msg.format(os.path.splitext(sys.argv[0])[0]))
 
 
 if __name__ == '__main__':
-    # Force iris.tests to use the ```tkagg``` backend by using the '-d'
-    # command-line argument as idiff is an interactive tool that requires a
-    # gui interface.
-    sys.argv.append('-d')
-
-    step_over_diffs()
+    default = os.path.join(os.path.dirname(iris.tests.__file__),
+                           'result_image_comparison')
+    description = 'Iris graphic test difference tool.'
+    parser = argparse.ArgumentParser(description=description)
+    help = 'Path to iris tests result image directory (default: %(default)s)'
+    parser.add_argument('--resultdir', '-r',
+                        default=default,
+                        help=help)
+    help = 'Force "iris.tests" to use the tkagg backend (default: %(default)s)'
+    parser.add_argument('-d',
+                        action='store_true',
+                        default=True,
+                        help=help)
+    args = parser.parse_args()
+    result_dir = args.resultdir
+    if not os.path.isdir(result_dir):
+        emsg = 'Invalid results directory: {}'
+        raise ValueError(emsg.format(result_dir))
+    step_over_diffs(result_dir)

minimal-conda-requirements.txt

@@ -19,6 +19,7 @@ mock
 nose
 pep8=1.5.7
 sphinx
+filelock
 
 # Optional iris dependencies
 gdal