diff --git a/WORKSPACE b/WORKSPACE index 89cbb3c..09df09f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,6 +5,14 @@ local_repository( name = "test_workspace", path = "tests/test_workspace", ) +local_repository( + name = "pypi__portpicker_1_2_0", + path = "third_party/pypi__portpicker_1_2_0", +) +local_repository( + name = "pypi__yapf_0_19_0", + path = "third_party/pypi__yapf_0_19_0", +) # Not actually referenced anywhere, but must be marked as a separate # repository so that things like "bazel test //..." don't get confused diff --git a/runtime/support.py b/runtime/support.py index d9e957a..0132e39 100644 --- a/runtime/support.py +++ b/runtime/support.py @@ -33,8 +33,10 @@ """ import os +import pkgutil import sys import warnings +import zipimport def _log(msg): @@ -61,6 +63,97 @@ def _find_archive(): return archive_path +def _setup_pkg_resources(pkg_resources_name): + """Setup hooks into the `pkg_resources` module + + This enables the pkg_resources module to find metadata from wheels + that have been included in this .par file. + + The functions and classes here are scoped to this function, since + we might have multitple pkg_resources modules, or none. + """ + + try: + __import__(pkg_resources_name) + pkg_resources = sys.modules.get(pkg_resources_name) + if pkg_resources is None: + return + except ImportError: + # Skip setup + return + + class DistInfoMetadata(pkg_resources.EggMetadata): + """Metadata provider for zip files containing .dist-info + + In find_dist_info_in_zip(), we call + metadata.resource_listdir(directory_name). However, it doesn't + work with EggMetadata, because _zipinfo_name() expects the + directory name to end with a /, but metadata._listdir() which + expects the directory to _not_ end with a /. + + Therefore this class exists. + """ + + def _listdir(self, fspath): + """List of resource names in the directory (like ``os.listdir()``) + + Overrides EggMetadata._listdir() + """ + + zipinfo_name = self._zipinfo_name(fspath) + while zipinfo_name.endswith('/'): + zipinfo_name = zipinfo_name[:-1] + result = self._index().get(zipinfo_name, ()) + return list(result) + + def find_dist_info_in_zip(importer, path_item, only=False): + """Find dist-info style metadata in zip files. + + We ignore the `only` flag because it's not clear what it should + actually do in this case. + """ + metadata = DistInfoMetadata(importer) + for subitem in metadata.resource_listdir('/'): + if subitem.lower().endswith('.dist-info'): + subpath = os.path.join(path_item, subitem) + submeta = pkg_resources.EggMetadata( + zipimport.zipimporter(subpath)) + submeta.egg_info = subpath + dist = pkg_resources.Distribution.from_location( + path_item, subitem, submeta) + yield dist + return + + def find_eggs_and_dist_info_in_zip(importer, path_item, only=False): + """Chain together our finder and the standard pkg_resources finder + + For simplicity, and since pkg_resources doesn't provide a public + interface to do so, we hardcode the chaining (find_eggs_in_zip). + """ + # Our finder + for dist in find_dist_info_in_zip(importer, path_item, only): + yield dist + # The standard pkg_resources finder + for dist in pkg_resources.find_eggs_in_zip(importer, path_item, only): + yield dist + return + + # This overwrites the existing registered finder. + pkg_resources.register_finder(zipimport.zipimporter, + find_eggs_and_dist_info_in_zip) + + # Note that the default WorkingSet has already been created, and + # there is no public interface to easily refresh/reload it that + # doesn't also have a "Don't use this" warning. So we manually + # add just the entries we know about to the existing WorkingSet. + for entry in sys.path: + importer = pkgutil.get_importer(entry) + if isinstance(importer, zipimport.zipimporter): + for dist in find_dist_info_in_zip(importer, entry, only=True): + pkg_resources.working_set.add(dist, entry, insert=False, + replace=False) + + def setup(import_roots=None): """Initialize subpar run-time support""" # Add third-party library entries to sys.path @@ -75,3 +168,7 @@ def setup(import_roots=None): new_path = os.path.join(archive_path, import_root) _log('# adding %s to sys.path' % new_path) sys.path.insert(1, new_path) + + # Add hook for package metadata + _setup_pkg_resources('pkg_resources') + _setup_pkg_resources('pip._vendor.pkg_resources') diff --git a/runtime/support_test.py b/runtime/support_test.py index ab25bd6..3ec1063 100644 --- a/runtime/support_test.py +++ b/runtime/support_test.py @@ -21,7 +21,7 @@ class SupportTest(unittest.TestCase): - def test_log(self): + def test__log(self): old_stderr = sys.stderr try: mock_stderr = io.StringIO() @@ -36,17 +36,35 @@ def test_log(self): finally: sys.stderr = old_stderr - def test_find_archive(self): + def test__find_archive(self): # pylint: disable=protected-access path = support._find_archive() self.assertNotEqual(path, None) def test_setup(self): - support.setup(import_roots=['some_root', 'another_root']) - self.assertTrue(sys.path[1].endswith('subpar/runtime/some_root'), - sys.path) - self.assertTrue(sys.path[2].endswith('subpar/runtime/another_root'), - sys.path) + # `import pip` can cause arbitrary sys.path changes, + # especially if using the Debian `python-pip` package or + # similar. Get that lunacy out of the way before starting + # test + try: + import pip # noqa + except ImportError: + pass + + old_sys_path = sys.path + try: + mock_sys_path = list(sys.path) + sys.path = mock_sys_path + support.setup(import_roots=['some_root', 'another_root']) + finally: + sys.path = old_sys_path + self.assertTrue(mock_sys_path[1].endswith('subpar/runtime/some_root'), + mock_sys_path) + self.assertTrue( + mock_sys_path[2].endswith('subpar/runtime/another_root'), + mock_sys_path) + self.assertEqual(mock_sys_path[0], sys.path[0]) + self.assertEqual(mock_sys_path[3:], sys.path[1:]) if __name__ == '__main__': diff --git a/tests/BUILD b/tests/BUILD index 0d35eb7..0d215bc 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -104,6 +104,19 @@ par_binary( srcs_version = "PY2AND3", ) +par_binary( + name = "package_pkg_resources/main", + srcs = [ + "package_pkg_resources/main.py", + ], + data = [ + "@pypi__portpicker_1_2_0//:files", + "@pypi__yapf_0_19_0//:files", + ], + main = "package_pkg_resources/main.py", + srcs_version = "PY2AND3", +) + # Test targets [( # Run test without .par file as a control @@ -154,6 +167,7 @@ par_binary( ), ("indirect_dependency", "//tests:package_c/c", "tests/package_c/c"), ("main_boilerplate", "//tests:package_g/g", "tests/package_g/g"), + ("pkg_resources", "//tests:package_pkg_resources/main", "tests/package_pkg_resources/main"), ("shadow", "//tests:package_shadow/main", "tests/package_shadow/main"), ("version", "//tests:package_f/f", "tests/package_f/f"), ]] diff --git a/tests/package_pkg_resources/main.py b/tests/package_pkg_resources/main.py new file mode 100644 index 0000000..dea801d --- /dev/null +++ b/tests/package_pkg_resources/main.py @@ -0,0 +1,45 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration test program for Subpar. + +Test that pkg_resources correctly identifies distribution packages +inside a .par file. +""" + + +def main(): + print('In pkg_resources test main()') + try: + import pkg_resources + except ImportError: + print('Skipping test, pkg_resources module is not available') + return + + ws = pkg_resources.working_set + + # Informational for debugging + distributions = list(ws) + print('Resources found: %s' % distributions) + + # Check for the packages we provided metadata for. There will + # also be metadata for whatever other packages happen to be + # installed in the current Python interpreter. + for spec in ['portpicker==1.2.0', 'yapf==0.19.0']: + dist = ws.find(pkg_resources.Requirement.parse(spec)) + assert dist, (spec, distributions) + + +if __name__ == '__main__': + main() diff --git a/tests/package_pkg_resources/main_PY2_filelist.txt b/tests/package_pkg_resources/main_PY2_filelist.txt new file mode 100644 index 0000000..7b93f00 --- /dev/null +++ b/tests/package_pkg_resources/main_PY2_filelist.txt @@ -0,0 +1,12 @@ +__main__.py +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json +subpar/__init__.py +subpar/runtime/__init__.py +subpar/runtime/support.py +subpar/tests/__init__.py +subpar/tests/package_pkg_resources/__init__.py +subpar/tests/package_pkg_resources/main +subpar/tests/package_pkg_resources/main.py diff --git a/tests/package_pkg_resources/main_PY3_filelist.txt b/tests/package_pkg_resources/main_PY3_filelist.txt new file mode 100644 index 0000000..7b93f00 --- /dev/null +++ b/tests/package_pkg_resources/main_PY3_filelist.txt @@ -0,0 +1,12 @@ +__main__.py +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json +subpar/__init__.py +subpar/runtime/__init__.py +subpar/runtime/support.py +subpar/tests/__init__.py +subpar/tests/package_pkg_resources/__init__.py +subpar/tests/package_pkg_resources/main +subpar/tests/package_pkg_resources/main.py diff --git a/third_party/README.md b/third_party/README.md new file mode 100644 index 0000000..6cbf2d2 --- /dev/null +++ b/third_party/README.md @@ -0,0 +1,9 @@ +# Wheels + +This directory contains code, metadata files, and wheels, from other projects, +used for testing. The projects were chosen to match the licence and ownership +of this project, i.e. Apache License 2.0, copyright owned by Google, from the +Google organization repository. Do not add any other type of thing here. + +- [python_portpicker](https://github.com/google/python_portpicker) +- [yapf](https://github.com/google/yapf) diff --git a/third_party/pypi__portpicker_1_2_0/BUILD b/third_party/pypi__portpicker_1_2_0/BUILD new file mode 100644 index 0000000..6cff5d5 --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/BUILD @@ -0,0 +1,10 @@ +# Test package for Subpar + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "files", + srcs = [], + data = glob(["portpicker-1.2.0.dist-info/**"]), + imports = ["."], +) diff --git a/third_party/pypi__portpicker_1_2_0/WORKSPACE b/third_party/pypi__portpicker_1_2_0/WORKSPACE new file mode 100644 index 0000000..57c31df --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "pypi__portpicker_1_2_0") diff --git a/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA new file mode 100644 index 0000000..8e852c1 --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA @@ -0,0 +1,29 @@ +Metadata-Version: 2.0 +Name: portpicker +Version: 1.2.0 +Summary: A library to choose unique available network ports. +Home-page: https://github.com/google/python_portpicker +Author: Google +Author-email: greg@krypto.org +License: Apache 2.0 +Description-Content-Type: UNKNOWN +Platform: POSIX +Classifier: Development Status :: 5 - Production/Stable +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy + +Portpicker provides an API to find and return an available network +port for an application to bind to. Ideally suited for use from +unittests or for test harnesses that launch local servers. + diff --git a/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json new file mode 100644 index 0000000..6ec9c43 --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"classifiers": ["Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: Jython", "Programming Language :: Python :: Implementation :: PyPy"], "description_content_type": "UNKNOWN", "extensions": {"python.details": {"contacts": [{"email": "greg@krypto.org", "name": "Google", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/google/python_portpicker"}}}, "generator": "bdist_wheel (0.30.0)", "license": "Apache 2.0", "metadata_version": "2.0", "name": "portpicker", "platform": "POSIX", "summary": "A library to choose unique available network ports.", "version": "1.2.0"} \ No newline at end of file diff --git a/third_party/pypi__yapf_0_19_0/BUILD b/third_party/pypi__yapf_0_19_0/BUILD new file mode 100644 index 0000000..cd12f69 --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/BUILD @@ -0,0 +1,10 @@ +# Test package for Subpar + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "files", + srcs = [], + data = glob(["yapf-0.19.0.dist-info/**"]), + imports = ["."], +) diff --git a/third_party/pypi__yapf_0_19_0/WORKSPACE b/third_party/pypi__yapf_0_19_0/WORKSPACE new file mode 100644 index 0000000..eae31db --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "pypi__yapf_0_19_0") diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA new file mode 100644 index 0000000..067f12a --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA @@ -0,0 +1,24 @@ +Metadata-Version: 2.0 +Name: yapf +Version: 0.19.0 +Summary: A formatter for Python code. +Home-page: UNKNOWN +Author: Bill Wendling +Author-email: morbo@google.com +License: Apache License, Version 2.0 +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Quality Assurance + +Text from original package has been elided. diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json new file mode 100644 index 0000000..71e9541 --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"classifiers": ["Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance"], "extensions": {"python.commands": {"wrap_console": {"yapf": "yapf:run_main"}}, "python.details": {"contacts": [{"email": "morbo@google.com", "name": "Bill Wendling", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}}, "python.exports": {"console_scripts": {"yapf": "yapf:run_main"}}}, "generator": "bdist_wheel (0.29.0)", "license": "Apache License, Version 2.0", "metadata_version": "2.0", "name": "yapf", "summary": "A formatter for Python code.", "version": "0.19.0"} \ No newline at end of file