Merge pull request #480 from claranet/fix/github_releases_search

fix: GitHub releases page no longer support `after` parameter, use tags page

Author: pdecat

src/claranet_tfwrapper/__init__.py

@@ -65,7 +65,8 @@ def get_architecture():
 RC_KO = 1
 RC_UNK = 2
 
-GITHUB_RELEASES = "https://github.com/{}/releases"
+LIMIT_GITHUB_RELEASES = 42
+GITHUB_RELEASES = "https://github.com/{}/tags"
 
 TERRAFORM_RELEASES_URL = "https://releases.hashicorp.com/terraform/index.json"
 TERRAFORM_MINOR_VERSION_REGEX = r"[0-9]+\.[0-9]+"
@@ -569,21 +570,32 @@ def bootstrap(wrapper_config):
 # FIXME: use https://pygithub.readthedocs.io/en/stable/github_objects/Repository.html#github.Repository.Repository.get_releases
 def search_on_github(repo, minor_version, patch_regex, patch):
     """Search release on github."""
-    # FIXME: the github UI seems to use a cache that does not return recent versions
-    result = CachedRequestsSession.get("{}?q={}".format(GITHUB_RELEASES.format(repo), minor_version))
-    releases = re.findall(r"<a href=\"/{}/releases/tag/.*\">(.*)</a>".format(repo), result.text)
-
-    for release in releases:
-        logger.debug(
-            'Release found: "{}", checking with "^v{}.{}$"'.format(release, minor_version, patch if patch else patch_regex)
-        )
-        if re.match(
-            r"^v{}\.{}$".format(minor_version, patch if patch else patch_regex),
-            release,
-        ):
-            patch = patch or release.split(".")[-1]
-            return patch
+    # Start search from the next incremented minor version
+    # Note: the github UI serves the first page if the requested version does not exist
+    release = "v{}{}.0".format(minor_version[:-1], int(minor_version[-1]) + 1)
+    releases_count = 0
+    while True:
+        result = CachedRequestsSession.get("{}?after={}".format(GITHUB_RELEASES.format(repo), release))
+        releases = re.findall(r"<a href=\"/{}/releases/tag/.*\">(.*)</a>".format(repo), result.text)
+        releases_count += len(releases)
+        for release in releases:
+            logger.debug(
+                'Release found: "{}", checking with "^v{}.{}$"'.format(release, minor_version, patch if patch else patch_regex)
+            )
+            if re.match(
+                r"^v{}\.{}$".format(minor_version, patch if patch else patch_regex),
+                release,
+            ):
+                patch = patch or release.split(".")[-1]
+                return patch
+            # hard limit or it will take too long
+            if releases_count > LIMIT_GITHUB_RELEASES:
+                return None
+        if len(releases) < 1:
+            # no more version available
+            break
 
+        release = releases[-1:][0]
     return None
 
 
@@ -1190,7 +1202,7 @@ def parse_args(args):
         "--pipe-plan",
         action="store_true",
         default=False,
-        help=("Pipe plan output to the command set in config" " or passed in --pipe-plan-command argument (cat by default)."),
+        help=("Pipe plan output to the command set in config or passed in --pipe-plan-command argument (cat by default)."),
     )
     parser_apply.add_argument(
         "--pipe-plan-command",
@@ -1249,7 +1261,7 @@ def parse_args(args):
         "--pipe-plan",
         action="store_true",
         default=False,
-        help=("Pipe plan output to the command set in config" " or passed in --pipe-plan-command argument (cat by default)."),
+        help=("Pipe plan output to the command set in config or passed in --pipe-plan-command argument (cat by default)."),
     )
     parser_plan.add_argument(
         "--pipe-plan-command",
@@ -1364,6 +1376,9 @@ def main(argv=None):
     )
     stack_config = load_stack_config_from_file(stack_config_file)
 
+    # parse all args
+    args = parse_args(argv or sys.argv[1:])
+
     # select tool version for the stack if selected, with a fallback on v1.0
     tool_version = (tf_config := stack_config.get(TOOL_TERRAFORM, {})).get(
         "version", tf_config.get("vars", {}).get("version", "1.0")
@@ -1382,9 +1397,6 @@ def main(argv=None):
     else:
         select_terraform_version(tool_version)
 
-    # parse all args
-    args = parse_args(argv or sys.argv[1:])
-
     # convert args to dict
     wrapper_config.update(vars(args))
 

tests/test_cache_requests.py

@@ -12,7 +12,7 @@
 
 
 @pytest.fixture
-def terraform_releases_html_q_0_12():
+def terraform_releases_html_after_v0_13_0():
     return """
 <!DOCTYPE html>
 <html lang="en">
@@ -43,7 +43,7 @@ def terraform_releases_html_q_0_12():
 #   CacheControl at the same time as it also works at the Adapter level.
 # - pook intercepts requests at a lower level by replacing urllib3.urlopen() by its
 #   own handler. This allows to actually verify what network requests would happen.
-# See See https://github.com/h2non/pook/blob/v1.0.2/pook/interceptors/urllib3.py#L24-L27
+# See https://github.com/h2non/pook/blob/v1.0.2/pook/interceptors/urllib3.py#L24-L27
 
 
 class AutoclosingBytesIO(io.BytesIO):
@@ -74,19 +74,19 @@ def read(self, size=-1):
 
 def test_search_on_github_cache_terraform_releases_200(
     tmp_working_dir,
-    terraform_releases_html_q_0_12,
+    terraform_releases_html_after_v0_13_0,
 ):
     with mock.patch("io.BytesIO", AutoclosingBytesIO):
         with pook.use():
             repo = "hashicorp/terraform"
-            releases_url = "https://github.com/{}/releases?q=0.12".format(repo)
+            releases_url = "https://github.com/{}/tags?after=v0.13.0".format(repo)
 
             # A volatile mock that can only be invoked once
             pook.get(
                 releases_url,
                 reply=200,
                 response_type="text/plain",
-                response_body=terraform_releases_html_q_0_12,
+                response_body=terraform_releases_html_after_v0_13_0,
                 response_headers={
                     "Status": "200 OK",
                     "ETag": 'W/"df0474ebd25f223a95926ba58e11e77b"',
@@ -110,11 +110,13 @@ def test_search_on_github_cache_terraform_releases_200(
     assert not pook.isactive()
 
 
-def test_search_on_github_cache_terraform_releases_does_not_cache_error_429(tmp_working_dir, terraform_releases_html_q_0_12):
+def test_search_on_github_cache_terraform_releases_does_not_cache_error_429(
+    tmp_working_dir, terraform_releases_html_after_v0_13_0
+):
     with mock.patch("io.BytesIO", AutoclosingBytesIO):
         with pook.use():
             repo = "hashicorp/terraform"
-            releases_url = "https://github.com/{}/releases?q=0.12".format(repo)
+            releases_url = "https://github.com/{}/tags?after=v0.13.0".format(repo)
 
             # volatile mocks that can only be invoked once each
             pook.get(
@@ -127,7 +129,7 @@ def test_search_on_github_cache_terraform_releases_does_not_cache_error_429(tmp_
                 releases_url,
                 reply=200,
                 response_type="text/plain",
-                response_body=terraform_releases_html_q_0_12,
+                response_body=terraform_releases_html_after_v0_13_0,
                 response_headers={
                     "Status": "200 OK",
                     "ETag": 'W/"df0474ebd25f223a95926ba58e11e77b"',
@@ -157,12 +159,12 @@ def test_search_on_github_cache_terraform_releases_does_not_cache_error_429(tmp_
 
 def test_search_on_github_cache_terraform_releases_does_not_cache_error_403(
     tmp_working_dir,
-    terraform_releases_html_q_0_12,
+    terraform_releases_html_after_v0_13_0,
 ):
     with mock.patch("io.BytesIO", AutoclosingBytesIO):
         with pook.use():
             repo = "hashicorp/terraform"
-            releases_url = "https://github.com/{}/releases?q=0.12".format(repo)
+            releases_url = "https://github.com/{}/tags?after=v0.13.0".format(repo)
 
             # volatile mocks that can only be invoked once each
             pook.get(
@@ -175,7 +177,7 @@ def test_search_on_github_cache_terraform_releases_does_not_cache_error_403(
                 releases_url,
                 reply=200,
                 response_type="text/plain",
-                response_body=terraform_releases_html_q_0_12,
+                response_body=terraform_releases_html_after_v0_13_0,
                 response_headers={
                     "Status": "200 OK",
                     "ETag": 'W/"df0474ebd25f223a95926ba58e11e77b"',
@@ -205,12 +207,12 @@ def test_search_on_github_cache_terraform_releases_does_not_cache_error_403(
 
 def test_search_on_github_cache_terraform_releases_does_not_cache_error_404(
     tmp_working_dir,
-    terraform_releases_html_q_0_12,
+    terraform_releases_html_after_v0_13_0,
 ):
     with mock.patch("io.BytesIO", AutoclosingBytesIO):
         with pook.use():
             repo = "hashicorp/terraform"
-            releases_url = "https://github.com/{}/releases?q=0.12".format(repo)
+            releases_url = "https://github.com/{}/tags?after=v0.13.0".format(repo)
 
             # volatile mocks that can only be invoked once each
             pook.get(
@@ -223,7 +225,7 @@ def test_search_on_github_cache_terraform_releases_does_not_cache_error_404(
                 releases_url,
                 reply=200,
                 response_type="text/plain",
-                response_body=terraform_releases_html_q_0_12,
+                response_body=terraform_releases_html_after_v0_13_0,
                 response_headers={
                     "Status": "200 OK",
                     "ETag": 'W/"df0474ebd25f223a95926ba58e11e77b"',

tests/test_search_on_github.py

@@ -7,7 +7,7 @@
 
 
 @pytest.fixture
-def provider_releases_html_q_2_5():
+def provider_releases_html_after_v2_6_0():
     return """
 <!DOCTYPE html>
 <html lang="en">
@@ -34,7 +34,7 @@ def provider_releases_html_q_2_5():
 
 
 @pytest.fixture
-def provider_releases_html_q_1_2():
+def provider_releases_html_after_v1_3_0():
     return """
 <!DOCTYPE html>
 <html lang="en">
@@ -62,28 +62,28 @@ def provider_releases_html_q_1_2():
 
 def test_search_on_github_provider_releases(
     requests_mock,
-    provider_releases_html_q_2_5,
-    provider_releases_html_q_1_2,
+    provider_releases_html_after_v2_6_0,
+    provider_releases_html_after_v1_3_0,
 ):
     from claranet_tfwrapper import GITHUB_RELEASES
 
     repo = "claranet/terraform-provider-gitlab"
     releases_url = GITHUB_RELEASES.format(repo)
-    releases_url_q_2_5 = releases_url + "?q=2.5"
-    releases_url_q_1_2 = releases_url + "?q=1.2"
+    releases_url_after_v2_6_0 = releases_url + "?after=v2.6.0"
+    releases_url_after_v1_3_0 = releases_url + "?after=v1.3.0"
 
     requests_mock.get(
-        releases_url_q_2_5,
+        releases_url_after_v2_6_0,
         complete_qs=True,
-        text=provider_releases_html_q_2_5,
+        text=provider_releases_html_after_v2_6_0,
     )
     requests_mock.get(
-        releases_url_q_1_2,
+        releases_url_after_v1_3_0,
         complete_qs=True,
-        text=provider_releases_html_q_1_2,
+        text=provider_releases_html_after_v1_3_0,
     )
 
-    assert provider_releases_html_q_2_5 == requests.get(releases_url_q_2_5).text
+    assert provider_releases_html_after_v2_6_0 == requests.get(releases_url_after_v2_6_0).text
 
     patch_regex = r"[0-9]+(((-alpha|-beta|-rc)[0-9]+)|(?P<dev>-dev))?"
     patch = tfwrapper.search_on_github(repo, "2.5", patch_regex, "")