tooling: Cleaup dep tools (envoyproxy#19816)

Signed-off-by: Ryan Northey <[email protected]>

tools/dependency/BUILD

@@ -1,4 +1,4 @@
-load("@rules_python//python:defs.bzl", "py_binary", "py_library")
+load("@rules_python//python:defs.bzl", "py_binary")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
 load("//tools/base:envoy_python.bzl", "envoy_entry_point")
 load("@base_pip3//:requirements.bzl", "requirement")
@@ -8,11 +8,6 @@ licenses(["notice"])  # Apache 2
 
 envoy_package()
 
-py_library(
-    name = "utils",
-    srcs = ["utils.py"],
-)
-
 envoy_entry_point(
     name = "check",
     args = [

tools/dependency/ossf_scorecard.py

@@ -24,9 +24,8 @@
 import os
 import subprocess as sp
 import sys
-
-import exports
-import utils
+from importlib.util import spec_from_loader, module_from_spec
+from importlib.machinery import SourceFileLoader
 
 Scorecard = namedtuple(
     'Scorecard', [
@@ -42,6 +41,35 @@
     ])
 
 
+# Obtain GitHub project URL from a list of URLs.
+def get_github_project_url(urls):
+    for url in urls:
+        if not url.startswith('https://github.com/'):
+            continue
+        components = url.split('/')
+        return f'https://github.com/{components[3]}/{components[4]}'
+    return None
+
+
+# Shared Starlark/Python files must have a .bzl suffix for Starlark import, so
+# we are forced to do this workaround.
+def load_module(name, path):
+    spec = spec_from_loader(name, SourceFileLoader(name, path))
+    module = module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# this is the relative path in a bazel build
+# to call this module outside of a bazel build set the `API_PATH` first,
+# for example, if running from the envoy repo root: `export API_PATH=api/`
+api_path = os.getenv("API_PATH", "external/envoy_api")
+
+# Modules
+repository_locations_utils = load_module(
+    'repository_locations_utils', os.path.join(api_path, 'bazel/repository_locations_utils.bzl'))
+
+
 # Thrown on errors related to release date.
 class OssfScorecardError(Exception):
     pass
@@ -63,7 +91,7 @@ def score(scorecard_path, repository_locations):
             continue
         results_key = metadata['project_name']
         formatted_name = '=HYPERLINK("%s", "%s")' % (metadata['project_url'], results_key)
-        github_project_url = utils.get_github_project_url(metadata['urls'])
+        github_project_url = get_github_project_url(metadata['urls'])
         if not github_project_url:
             na = 'Not Scorecard compatible'
             results[results_key] = Scorecard(
@@ -139,8 +167,8 @@ def print_csv_results(csv_output_path, results):
     path = sys.argv[1]
     scorecard_path = sys.argv[2]
     csv_output_path = sys.argv[3]
-    spec_loader = exports.repository_locations_utils.load_repository_locations_spec
-    path_module = exports.load_module('repository_locations', path)
+    spec_loader = repository_locations_utils.load_repository_locations_spec
+    path_module = load_module('repository_locations', path)
     try:
         results = score(scorecard_path, spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC))
         print_csv_results(csv_output_path, results)

tools/docs/BUILD

@@ -22,7 +22,6 @@ py_binary(
     ],
     args = ["$(location //bazel:all_repository_locations)"],
     data = ["//bazel:all_repository_locations"],
-    deps = ["//tools/dependency:utils"],
 )
 
 py_binary(

tools/docs/generate_external_deps_rst.py

@@ -10,7 +10,38 @@
 import urllib.parse
 from collections import defaultdict, namedtuple
 
-from tools.dependency import utils as dep_utils
+# Information releated to a GitHub release version.
+GitHubRelease = namedtuple('GitHubRelease', ['organization', 'project', 'version', 'tagged'])
+
+
+# Search through a list of URLs and determine if any contain a GitHub URL. If
+# so, use heuristics to extract the release version and repo details, return
+# this, otherwise return None.
+def get_github_release_from_urls(urls):
+    for url in urls:
+        if not url.startswith('https://github.com/'):
+            continue
+        components = url.split('/')
+        if components[5] == 'archive':
+            # Only support .tar.gz, .zip today. Figure out the release tag from this
+            # filename.
+            if components[-1].endswith('.tar.gz'):
+                github_version = components[-1][:-len('.tar.gz')]
+            else:
+                assert (components[-1].endswith('.zip'))
+                github_version = components[-1][:-len('.zip')]
+        else:
+            # Release tag is a path component.
+            assert (components[5] == 'releases')
+            github_version = components[7]
+        # If it's not a GH hash, it's a tagged release.
+        tagged_release = len(github_version) != 40
+        return GitHubRelease(
+            organization=components[3],
+            project=components[4],
+            version=github_version,
+            tagged=tagged_release)
+    return None
 
 
 # Render a CSV table given a list of table headers, widths and list of rows
@@ -56,7 +87,7 @@ def render_title(title):
 # SHA. Otherwise, return the tarball download.
 def get_version_url(metadata):
     # Figure out if it's a GitHub repo.
-    github_release = dep_utils.get_github_release_from_urls(metadata['urls'])
+    github_release = get_github_release_from_urls(metadata['urls'])
     # If not, direct download link for tarball
     if not github_release:
         return metadata['urls'][0]