build: add support for pip3_import from rules_python. (envoyproxy#11108)

Provide an Envoy build system styled way of loading arbitrary pip3 packages as py_{library,binary} dependencies. Part of envoyproxy#11085 (to fetch PyYAML). Risk level: Low (build system) Testing: Manual validation that the test utility loads PyYAML at the correct version (different to my system version). Signed-off-by: Harvey Tuch <[email protected]>
lemonlinger · May 12, 2020 · 568e139 · 568e139
1 parent 45726b7
commit 568e139
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 8 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -12,6 +12,10 @@ load("//bazel:repositories.bzl", "envoy_dependencies")
 
 envoy_dependencies()
 
+load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra")
+
+envoy_dependencies_extra()
+
 load("//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 
 envoy_dependency_imports()
diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md
@@ -6,7 +6,9 @@ values can change when Github change their tar/gzip libraries breaking
 builds. Maintainer provided tarballs are more stable and the maintainer
 can provide the SHA256.
 
-# Adding external dependencies to Envoy (native Bazel)
+# Adding external dependencies to Envoy (C++)
+
+## Native Bazel
 
 This is the preferred style of adding dependencies that use Bazel for their
 build process.
@@ -17,7 +19,7 @@ build process.
    `external_deps` attribute.
 3. `bazel test //test/...`
 
-# Adding external dependencies to Envoy (external CMake)
+## External CMake (preferred)
 
 This is the preferred style of adding dependencies that use CMake for their build system.
 
@@ -29,7 +31,8 @@ This is the preferred style of adding dependencies that use CMake for their buil
    `external_deps` attribute.
 4. `bazel test //test/...`
 
-# Adding external dependencies to Envoy (genrule repository)
+
+## genrule repository
 
 This is the newer style of adding dependencies with no upstream Bazel configs.
 It wraps the dependency's native build tooling in a Bazel-aware shell script,
@@ -54,6 +57,24 @@ Dependencies between external libraries can use the standard Bazel dependency
 resolution logic, using the `$(location)` shell extension to resolve paths
 to binaries, libraries, headers, etc.
 
+# Adding external dependencies to Envoy (Python)
+
+Python dependencies should be added via `pip3` and `rules_python`. The process
+is:
+
+1. Define a `pip3_import()` pointing at your target `requirements.txt` in
+   [`bazel/repositories_extra.bzl`](repositories_extra.bzl)
+
+2. Add a `pip_install()` invocation in
+   [`bazel/dependency_imports.bzl`](dependency_imports.bzl).
+
+3. Add a `requirements("<package name")` in the `BUILD` file that depends on
+   this package.
+
+You can use [`tools/config_validation/BUILD`](../tools/config_validation/BUILD) as an example
+for this flow. See also the [`rules_python`](https://github.com/bazelbuild/rules_python)
+documentation for further references.
+
 # Updating an external dependency version
 
 1. Update the corresponding entry in

diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl
@@ -4,6 +4,7 @@ load("@envoy_build_tools//toolchains:rbe_toolchains_config.bzl", "rbe_toolchains
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
 load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
 load("@upb//bazel:repository_defs.bzl", upb_bazel_version_repository = "bazel_version_repository")
+load("@config_validation//:requirements.bzl", config_validation_pip_install = "pip_install")
 
 # go version for rules_go
 GO_VERSION = "1.13.5"
@@ -38,3 +39,5 @@ def envoy_dependency_imports(go_version = GO_VERSION):
         sum = "h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=",
         version = "v0.3.0",
     )
+
+    config_validation_pip_install()
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -68,9 +68,9 @@ _default_envoy_build_config = repository_rule(
     },
 )
 
-# Python dependencies. If these become non-trivial, we might be better off using a virtualenv to
-# wrap them, but for now we can treat them as first-class Bazel.
+# Python dependencies.
 def _python_deps():
+    # TODO(htuch): convert these to pip3_import.
     _repository_impl(
         name = "com_github_pallets_markupsafe",
         build_file = "@envoy//bazel/external:markupsafe.BUILD",

diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl
@@ -0,0 +1,16 @@
+load("@rules_python//python:repositories.bzl", "py_repositories")
+load("@rules_python//python:pip.bzl", "pip3_import", "pip_repositories")
+
+# Python dependencies.
+def _python_deps():
+    py_repositories()
+    pip_repositories()
+
+    pip3_import(
+        name = "config_validation",
+        requirements = "@envoy//tools/config_validation:requirements.txt",
+    )
+
+# Envoy deps that rely on a first stage of dependency loading in envoy_dependencies().
+def envoy_dependencies_extra():
+    _python_deps()
diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -365,8 +365,11 @@ DEPENDENCY_REPOSITORIES = dict(
         use_category = ["build"],
     ),
     rules_python = dict(
-        sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
-        urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz"],
+        sha256 = "76a8fd4e7eca2a3590f816958faa0d83c9b2ce9c32634c5c375bcccf161d3bb5",
+        strip_prefix = "rules_python-a0fbf98d4e3a232144df4d0d80b577c7a693b570",
+        # 2020-04-09
+        # TODO(htuch): revert back to a point releases when pip3_import appears.
+        urls = ["https://github.com/bazelbuild/rules_python/archive/a0fbf98d4e3a232144df4d0d80b577c7a693b570.tar.gz"],
         use_category = ["build"],
     ),
     six = dict(

diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example
@@ -17,6 +17,10 @@ load("@envoy//bazel:repositories.bzl", "envoy_dependencies")
 
 envoy_dependencies()
 
+load("@envoy//bazel:repositories_extra.bzl", "envoy_dependencies_extra")
+
+envoy_dependencies_extra()
+
 load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 
 envoy_dependency_imports()
diff --git a/ci/run_envoy_docker_windows.sh b/ci/run_envoy_docker_windows.sh
@@ -4,7 +4,7 @@ set -e
 
 # TODO(sunjayBhatia, wrowe): update this note once we have an RBE toolchain generated for Windows
 # NOTE: Update this from the latest pushed image here: https://hub.docker.com/r/envoyproxy/envoy-build-windows2019/tags
-ENVOY_BUILD_SHA="3cbc11e373dc4e3a523b9273ed010c5e0f197874"
+ENVOY_BUILD_SHA="9b7dc527351b9888805377a05e5975d6ef8d6ae1"
 
 [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019"
 # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker

diff --git a/tools/config_validation/BUILD b/tools/config_validation/BUILD
@@ -0,0 +1,10 @@
+licenses(["notice"])  # Apache 2
+
+load("@config_validation//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "validate_yaml_fragment",
+    srcs = ["validate_yaml_fragment.py"],
+    visibility = ["//visibility:public"],
+    deps = [requirement("PyYAML")],
+)
diff --git a/tools/config_validation/requirements.txt b/tools/config_validation/requirements.txt
@@ -0,0 +1 @@
+PyYAML==5.3.1
diff --git a/tools/config_validation/validate_yaml_fragment.py b/tools/config_validation/validate_yaml_fragment.py
@@ -0,0 +1,3 @@
+import yaml
+
+print('YAML version is %s' % yaml.__version__)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		import yaml

		print('YAML version is %s' % yaml.__version__)