Expose package metadata from wheels in .par files (#49)

* Expose package metadata from wheels in .par files

* Update pkg_resources's default distribution metadata object.

`pkg_resources` creates a default WorkingSet as soon as it is
imported, so even though we add hooks for it to find the .dist-info
metadata inside .par files, it is too late.  So we manually update the
default WorkingSet, trying not to disturb existing entries.

* Cut down unnecessary test data.

* Fix lint errors

WORKSPACE

@@ -5,6 +5,14 @@ local_repository(
     name = "test_workspace",
     path = "tests/test_workspace",
 )
+local_repository(
+    name = "pypi__portpicker_1_2_0",
+    path = "third_party/pypi__portpicker_1_2_0",
+)
+local_repository(
+    name = "pypi__yapf_0_19_0",
+    path = "third_party/pypi__yapf_0_19_0",
+)
 
 # Not actually referenced anywhere, but must be marked as a separate
 # repository so that things like "bazel test //..." don't get confused

runtime/support.py

@@ -33,8 +33,10 @@
 """
 
 import os
+import pkgutil
 import sys
 import warnings
+import zipimport
 
 
 def _log(msg):
@@ -61,6 +63,97 @@ def _find_archive():
     return archive_path
 
 
+def _setup_pkg_resources(pkg_resources_name):
+    """Setup hooks into the `pkg_resources` module
+
+    This enables the pkg_resources module to find metadata from wheels
+    that have been included in this .par file.
+
+    The functions and classes here are scoped to this function, since
+    we might have multitple pkg_resources modules, or none.
+    """
+
+    try:
+        __import__(pkg_resources_name)
+        pkg_resources = sys.modules.get(pkg_resources_name)
+        if pkg_resources is None:
+            return
+    except ImportError:
+        # Skip setup
+        return
+
+    class DistInfoMetadata(pkg_resources.EggMetadata):
+        """Metadata provider for zip files containing .dist-info
+
+        In find_dist_info_in_zip(), we call
+        metadata.resource_listdir(directory_name).  However, it doesn't
+        work with EggMetadata, because _zipinfo_name() expects the
+        directory name to end with a /, but metadata._listdir() which
+        expects the directory to _not_ end with a /.
+
+        Therefore this class exists.
+        """
+
+        def _listdir(self, fspath):
+            """List of resource names in the directory (like ``os.listdir()``)
+
+            Overrides EggMetadata._listdir()
+            """
+
+            zipinfo_name = self._zipinfo_name(fspath)
+            while zipinfo_name.endswith('/'):
+                zipinfo_name = zipinfo_name[:-1]
+            result = self._index().get(zipinfo_name, ())
+            return list(result)
+
+    def find_dist_info_in_zip(importer, path_item, only=False):
+        """Find dist-info style metadata in zip files.
+
+        We ignore the `only` flag because it's not clear what it should
+        actually do in this case.
+        """
+        metadata = DistInfoMetadata(importer)
+        for subitem in metadata.resource_listdir('/'):
+            if subitem.lower().endswith('.dist-info'):
+                subpath = os.path.join(path_item, subitem)
+                submeta = pkg_resources.EggMetadata(
+                    zipimport.zipimporter(subpath))
+                submeta.egg_info = subpath
+                dist = pkg_resources.Distribution.from_location(
+                    path_item, subitem, submeta)
+                yield dist
+        return
+
+    def find_eggs_and_dist_info_in_zip(importer, path_item, only=False):
+        """Chain together our finder and the standard pkg_resources finder
+
+        For simplicity, and since pkg_resources doesn't provide a public
+        interface to do so, we hardcode the chaining (find_eggs_in_zip).
+        """
+        # Our finder
+        for dist in find_dist_info_in_zip(importer, path_item, only):
+            yield dist
+        # The standard pkg_resources finder
+        for dist in pkg_resources.find_eggs_in_zip(importer, path_item, only):
+            yield dist
+        return
+
+    # This overwrites the existing registered finder.
+    pkg_resources.register_finder(zipimport.zipimporter,
+                                  find_eggs_and_dist_info_in_zip)
+
+    # Note that the default WorkingSet has already been created, and
+    # there is no public interface to easily refresh/reload it that
+    # doesn't also have a "Don't use this" warning.  So we manually
+    # add just the entries we know about to the existing WorkingSet.
+    for entry in sys.path:
+        importer = pkgutil.get_importer(entry)
+        if isinstance(importer, zipimport.zipimporter):
+            for dist in find_dist_info_in_zip(importer, entry, only=True):
+                pkg_resources.working_set.add(dist, entry, insert=False,
+                                              replace=False)
+
+
 def setup(import_roots=None):
     """Initialize subpar run-time support"""
     # Add third-party library entries to sys.path
@@ -75,3 +168,7 @@ def setup(import_roots=None):
         new_path = os.path.join(archive_path, import_root)
         _log('# adding %s to sys.path' % new_path)
         sys.path.insert(1, new_path)
+
+    # Add hook for package metadata
+    _setup_pkg_resources('pkg_resources')
+    _setup_pkg_resources('pip._vendor.pkg_resources')

runtime/support_test.py

@@ -21,7 +21,7 @@
 
 class SupportTest(unittest.TestCase):
 
-    def test_log(self):
+    def test__log(self):
         old_stderr = sys.stderr
         try:
             mock_stderr = io.StringIO()
@@ -36,17 +36,35 @@ def test_log(self):
         finally:
             sys.stderr = old_stderr
 
-    def test_find_archive(self):
+    def test__find_archive(self):
         # pylint: disable=protected-access
         path = support._find_archive()
         self.assertNotEqual(path, None)
 
     def test_setup(self):
-        support.setup(import_roots=['some_root', 'another_root'])
-        self.assertTrue(sys.path[1].endswith('subpar/runtime/some_root'),
-                        sys.path)
-        self.assertTrue(sys.path[2].endswith('subpar/runtime/another_root'),
-                        sys.path)
+        # `import pip` can cause arbitrary sys.path changes,
+        # especially if using the Debian `python-pip` package or
+        # similar.  Get that lunacy out of the way before starting
+        # test
+        try:
+            import pip  # noqa
+        except ImportError:
+            pass
+
+        old_sys_path = sys.path
+        try:
+            mock_sys_path = list(sys.path)
+            sys.path = mock_sys_path
+            support.setup(import_roots=['some_root', 'another_root'])
+        finally:
+            sys.path = old_sys_path
+        self.assertTrue(mock_sys_path[1].endswith('subpar/runtime/some_root'),
+                        mock_sys_path)
+        self.assertTrue(
+            mock_sys_path[2].endswith('subpar/runtime/another_root'),
+            mock_sys_path)
+        self.assertEqual(mock_sys_path[0], sys.path[0])
+        self.assertEqual(mock_sys_path[3:], sys.path[1:])
 
 
 if __name__ == '__main__':

tests/BUILD

@@ -104,6 +104,19 @@ par_binary(
     srcs_version = "PY2AND3",
 )
 
+par_binary(
+    name = "package_pkg_resources/main",
+    srcs = [
+        "package_pkg_resources/main.py",
+    ],
+    data = [
+        "@pypi__portpicker_1_2_0//:files",
+        "@pypi__yapf_0_19_0//:files",
+    ],
+    main = "package_pkg_resources/main.py",
+    srcs_version = "PY2AND3",
+)
+
 # Test targets
 [(
     # Run test without .par file as a control
@@ -154,6 +167,7 @@ par_binary(
     ),
     ("indirect_dependency", "//tests:package_c/c", "tests/package_c/c"),
     ("main_boilerplate", "//tests:package_g/g", "tests/package_g/g"),
+    ("pkg_resources", "//tests:package_pkg_resources/main", "tests/package_pkg_resources/main"),
     ("shadow", "//tests:package_shadow/main", "tests/package_shadow/main"),
     ("version", "//tests:package_f/f", "tests/package_f/f"),
 ]]

tests/package_pkg_resources/main.py

@@ -0,0 +1,45 @@
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration test program for Subpar.
+
+Test that pkg_resources correctly identifies distribution packages
+inside a .par file.
+"""
+
+
+def main():
+    print('In pkg_resources test main()')
+    try:
+        import pkg_resources
+    except ImportError:
+        print('Skipping test, pkg_resources module is not available')
+        return
+
+    ws = pkg_resources.working_set
+
+    # Informational for debugging
+    distributions = list(ws)
+    print('Resources found: %s' % distributions)
+
+    # Check for the packages we provided metadata for.  There will
+    # also be metadata for whatever other packages happen to be
+    # installed in the current Python interpreter.
+    for spec in ['portpicker==1.2.0', 'yapf==0.19.0']:
+        dist = ws.find(pkg_resources.Requirement.parse(spec))
+        assert dist, (spec, distributions)
+
+
+if __name__ == '__main__':
+    main()

tests/package_pkg_resources/main_PY2_filelist.txt

@@ -0,0 +1,12 @@
+__main__.py
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
+subpar/__init__.py
+subpar/runtime/__init__.py
+subpar/runtime/support.py
+subpar/tests/__init__.py
+subpar/tests/package_pkg_resources/__init__.py
+subpar/tests/package_pkg_resources/main
+subpar/tests/package_pkg_resources/main.py

tests/package_pkg_resources/main_PY3_filelist.txt

@@ -0,0 +1,12 @@
+__main__.py
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
+subpar/__init__.py
+subpar/runtime/__init__.py
+subpar/runtime/support.py
+subpar/tests/__init__.py
+subpar/tests/package_pkg_resources/__init__.py
+subpar/tests/package_pkg_resources/main
+subpar/tests/package_pkg_resources/main.py

third_party/README.md

@@ -0,0 +1,9 @@
+# Wheels
+
+This directory contains code, metadata files, and wheels, from other projects,
+used for testing.  The projects were chosen to match the licence and ownership
+of this project, i.e. Apache License 2.0, copyright owned by Google, from the
+Google organization repository.  Do not add any other type of thing here.
+
+- [python_portpicker](https://github.com/google/python_portpicker)
+- [yapf](https://github.com/google/yapf)

third_party/pypi__portpicker_1_2_0/BUILD

@@ -0,0 +1,10 @@
+# Test package for Subpar
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "files",
+    srcs = [],
+    data = glob(["portpicker-1.2.0.dist-info/**"]),
+    imports = ["."],
+)

third_party/pypi__portpicker_1_2_0/WORKSPACE

@@ -0,0 +1 @@
+workspace(name = "pypi__portpicker_1_2_0")

third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA

@@ -0,0 +1,29 @@
+Metadata-Version: 2.0
+Name: portpicker
+Version: 1.2.0
+Summary: A library to choose unique available network ports.
+Home-page: https://github.com/google/python_portpicker
+Author: Google
+Author-email: [email protected]
+License: Apache 2.0
+Description-Content-Type: UNKNOWN
+Platform: POSIX
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: Jython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+
+Portpicker provides an API to find and return an available network
+port for an application to bind to.  Ideally suited for use from
+unittests or for test harnesses that launch local servers.
+

third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json

@@ -0,0 +1 @@
+{"classifiers": ["Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: Jython", "Programming Language :: Python :: Implementation :: PyPy"], "description_content_type": "UNKNOWN", "extensions": {"python.details": {"contacts": [{"email": "[email protected]", "name": "Google", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/google/python_portpicker"}}}, "generator": "bdist_wheel (0.30.0)", "license": "Apache 2.0", "metadata_version": "2.0", "name": "portpicker", "platform": "POSIX", "summary": "A library to choose unique available network ports.", "version": "1.2.0"}

third_party/pypi__yapf_0_19_0/BUILD

@@ -0,0 +1,10 @@
+# Test package for Subpar
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "files",
+    srcs = [],
+    data = glob(["yapf-0.19.0.dist-info/**"]),
+    imports = ["."],
+)

third_party/pypi__yapf_0_19_0/WORKSPACE

@@ -0,0 +1 @@
+workspace(name = "pypi__yapf_0_19_0")

third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA

@@ -0,0 +1,24 @@
+Metadata-Version: 2.0
+Name: yapf
+Version: 0.19.0
+Summary: A formatter for Python code.
+Home-page: UNKNOWN
+Author: Bill Wendling
+Author-email: [email protected]
+License: Apache License, Version 2.0
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Quality Assurance
+
+Text from original package has been elided.

third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json

@@ -0,0 +1 @@
+{"classifiers": ["Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance"], "extensions": {"python.commands": {"wrap_console": {"yapf": "yapf:run_main"}}, "python.details": {"contacts": [{"email": "[email protected]", "name": "Bill Wendling", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}}, "python.exports": {"console_scripts": {"yapf": "yapf:run_main"}}}, "generator": "bdist_wheel (0.29.0)", "license": "Apache License, Version 2.0", "metadata_version": "2.0", "name": "yapf", "summary": "A formatter for Python code.", "version": "0.19.0"}