Merge pull request #687 from anaik91/main

Adding Proxy Endpoint Unifier

CODEOWNERS

@@ -36,4 +36,5 @@
 /tools/oas-configurable-proxy           @danistrebel
 /tools/pipeline-linter                  @seymen @danistrebel
 /tools/pipeline-runner                  @seymen @danistrebel
-/tools/sf-dependency-list               @yuriylesyuk
+/tools/sf-dependency-list               @yuriylesyuk
+/tools/proxy-endpoint-unifier           @anaik91

README.md

@@ -91,6 +91,8 @@ Apigee products.
   A tool to generate topologically sorted Shared Flow dependencies.
 - [Apigee Envoy Quickstart Toolkit](tools/apigee-envoy-quickstart) -
   A tool to set up the sample deployments of Apigee Envoy.
+- [Apigee API Proxy Endpoint Unifier](tools/proxy-endpoint-unifier) -
+  A tool to unify/split proxy endpoints based on API basepath.
 
 ## Labs
 

tools/pipeline-runner/Dockerfile

@@ -33,7 +33,9 @@ RUN apk add --no-cache \
   freetype-dev \
   harfbuzz \
   ca-certificates \
-  ttf-freefont
+  ttf-freefont \
+  py-pip \
+  zip
 
 # Reduce nighly log (note: -ntp requires maven 3.6.1+)
 RUN mv /usr/bin/mvn /usr/bin/_mvn &&\

tools/proxy-endpoint-unifier/README.md

@@ -0,0 +1,44 @@
+# Apigee API Proxy Endpoint Unifier
+
+Apigee X and hybrid have a limitation of hosting up to 5 Proxy Endpoints per API Proxy. Apigee Edge has no such limitation.
+The objective of this tool is to take a proxy bundle and intelligently convert its proxy endpoints into logically
+grouped conditional flows, in order to stay within the Proxy Endpoint limit.
+
+## Disclaimer
+This is not an officially supported Google product.
+
+## Prerequisites
+* `python3`
+* Please install the required Python dependencies
+```
+    python3 -m pip install -r requirements.txt
+```
+* Please fill in `input.properties`
+```
+    [common]
+    input_apis=apis                                     # Folder Containing exported & unzipped Proxy Bundles
+    processed_apis=transformed                          # Folder to export transformed Proxies to 
+    proxy_bundle_directory=transformed_zipped_bundles   # Folder to export transformed Proxies Bundles (zip) to 
+    proxy_endpoint_count=4                              # Number of Proxy Endpoints to retain while transforming (1-5)
+    debug=false                                         # Flag to export debug logs
+
+    [validate]
+    enabled=true                                        # Flag to enable proxy validation
+    gcp_project_id=xxx-xxx-xxx                          # Apigee Project for proxy validation
+```
+
+* If enabling validation, please run the following command to authenticate against Apigee APIs:
+
+```
+    export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)
+```
+
+
+## Usage
+Run the script as below
+```
+python3 main.py
+```
+
+## Limitations
+* This tool does not currently handle the resources within API proxies.

tools/proxy-endpoint-unifier/apigee.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+import os
+
+
+class Apigee:
+    def __init__(self, org):
+        self.baseurl = f"https://apigee.googleapis.com/v1/organizations/{org}"
+        self.auth_header = {}
+
+    def set_auth_header(self, token):
+        self.auth_header = {
+            'Authorization': f"Bearer {token}"
+        }
+
+    def validate_api(self, api_type, proxy_bundle_path):
+        api_name = os.path.basename(proxy_bundle_path).split('.zip')[0]
+        url = f"{self.baseurl}/{api_type}?name={api_name}&action=validate&validate=true"  # noqa
+        files = [
+            ('data', (api_name, open(proxy_bundle_path,'rb'), 'application/zip'))  # noqa
+        ]
+        response = requests.request("POST", url, headers=self.auth_header,
+                                    data={}, files=files)
+        if response.status_code == 200:
+            return True
+        else:
+            return response.json()

tools/proxy-endpoint-unifier/input.properties

@@ -0,0 +1,24 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[common]
+input_apis=test/api_bundles
+processed_apis=test/transformed
+proxy_bundle_directory=test/transformed_zipped_bundles
+proxy_endpoint_count=4
+debug=false
+
+[validate]
+enabled=true
+gcp_project_id=xxx-xxx-xxx

tools/proxy-endpoint-unifier/main.py

@@ -0,0 +1,132 @@
+#!/usr/bin/python
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from apigee import Apigee
+import utils
+
+
+def main():
+    cfg = utils.parse_config('input.properties')
+    proxy_dir = cfg['common']['input_apis']
+    proxy_dest_dir = cfg['common']['processed_apis']
+    proxy_bundle_directory = cfg['common']['proxy_bundle_directory']
+    export_debug_file = cfg.getboolean('common', 'debug')
+    validation_enabled = cfg.getboolean('validate', 'enabled')
+    utils.delete_folder(proxy_dest_dir)
+    utils.delete_folder(proxy_bundle_directory)
+    utils.create_dir(proxy_bundle_directory)
+    proxy_endpoint_count = utils.get_proxy_endpoint_count(cfg)
+    proxies = utils.list_dir(proxy_dir)
+
+    final_dict = {}
+    processed_dict = {}
+
+    for each_dir in proxies:
+        each_proxy_dict = utils.read_proxy_artifacts(
+                        f"{proxy_dir}/{each_dir}",
+                        utils.parse_proxy_root(f"{proxy_dir}/{each_dir}")
+                            )
+        if len(each_proxy_dict) > 0:
+            each_proxy_rel = utils.get_proxy_objects_relationships(
+                each_proxy_dict)
+            final_dict[each_dir] = each_proxy_dict
+            processed_dict[each_dir] = each_proxy_rel
+
+    processing_final_dict = final_dict.copy()
+
+    path_group_map = {}
+    for each_api, each_api_info in processed_dict.items():
+        path_group_map[each_api] = utils.get_api_path_groups(each_api_info)
+
+    grouped_apis = {}
+    for each_api, base_path_info in path_group_map.items():
+        grouped_apis[each_api] = utils.group_paths_by_path(
+            base_path_info, proxy_endpoint_count)
+
+    bundled_group = {}
+    for each_api, grouped_api in grouped_apis.items():
+        bundled_group[each_api] = utils.bundle_path(grouped_api)
+
+    merged_pes = {}
+    merged_objects = {}
+    for each_api, grouped_api in bundled_group.items():
+        print(f'Processing API => {each_api} with {len(grouped_api)} groups')
+        for index, each_group in enumerate(grouped_api):
+            merged_objects[f"{each_api}_{index}"] = {
+                'Policies': [],
+                'TargetEndpoints': [],
+                'ProxyEndpoints': []
+            }
+            for each_path, pes in each_group.items():
+                each_pe = '-'.join(pes)
+                merged_pes[each_pe] = utils.merge_proxy_endpoints(
+                    processing_final_dict[each_api],
+                    each_path,
+                    pes
+                )
+                merged_objects[f"{each_api}_{index}"]['Name'] = f"{final_dict[each_api]['proxyName']}_{index}"  # noqa
+                merged_objects[f"{each_api}_{index}"]['Policies'].extend(  # noqa
+                    [ item for pe in pes for item in processed_dict[each_api][pe]['Policies']])  # noqa
+                merged_objects[f"{each_api}_{index}"]['TargetEndpoints'].extend(  # noqa
+                    [ item for pe in pes for item in processed_dict[each_api][pe]['TargetEndpoints']])  # noqa
+                merged_objects[f"{each_api}_{index}"]['Policies'] = list(set(merged_objects[f"{each_api}_{index}"]['Policies']))  # noqa
+                merged_objects[f"{each_api}_{index}"]['TargetEndpoints'] = list(set(merged_objects[f"{each_api}_{index}"]['TargetEndpoints']))  # noqa
+                merged_objects[f"{each_api}_{index}"]['ProxyEndpoints'].append(each_pe)  # noqa
+
+    for each_api, grouped_api in bundled_group.items():
+        for index, each_group in enumerate(grouped_api):
+            utils.clone_proxies(
+                    f"{proxy_dir}/{each_api}",
+                    f"{proxy_dest_dir}/{each_api}_{index}",
+                    merged_objects[f"{each_api}_{index}"],
+                    merged_pes,
+                    proxy_bundle_directory
+            )
+
+    files = {
+        'final_dict': final_dict,
+        'processed_dict': processed_dict,
+        'path_group_map': path_group_map,
+        'grouped_apis': grouped_apis,
+        'bundled_group': bundled_group,
+        'merged_pes': merged_pes,
+        'merged_objects': merged_objects,
+    }
+    if export_debug_file:
+        utils.export_debug_log(files)
+
+    if validation_enabled:
+        errors = {}
+        gcp_project_id = cfg['validate']['gcp_project_id']
+        x = Apigee(gcp_project_id)
+        x.set_auth_header(os.getenv('APIGEE_ACCESS_TOKEN'))
+        result = {}
+        bundled_proxies = utils.list_dir(proxy_bundle_directory)
+        for each_bundle in bundled_proxies:
+            validation = x.validate_api('apis',f"{proxy_bundle_directory}/{each_bundle}")  # noqa
+            if not isinstance(validation, bool):
+                errors[each_bundle] = validation
+            result[each_bundle] = validation
+            print(f"{each_bundle}  ==> Validation : {validation}")
+        if len(errors) > 0:
+            print('ERROR: Some Validations have failed')
+            sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tools/proxy-endpoint-unifier/pipeline.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SCRIPTPATH="$( cd "$(dirname "$0")" || exit >/dev/null 2>&1 ; pwd -P )"
+
+# Clean up previously generated files
+rm -rf "$SCRIPTPATH/input.properties"
+rm -rf "$SCRIPTPATH/transformed"
+rm -rf "$SCRIPTPATH/transformed_bundles"
+
+# Generate input file
+cat > "$SCRIPTPATH/input.properties" << EOF
+[common]
+input_apis=$SCRIPTPATH/test/api_bundles
+processed_apis=$SCRIPTPATH/transformed
+proxy_bundle_directory=$SCRIPTPATH/transformed_bundles
+proxy_endpoint_count=4
+debug=true
+
+[validate]
+enabled=true
+gcp_project_id=$APIGEE_X_ORG
+EOF
+
+# Install Dependencies
+python3 -m pip install -r "$SCRIPTPATH/requirements.txt"
+
+# Generate Gcloud Acccess Token
+APIGEE_ACCESS_TOKEN="$(gcloud config config-helper --force-auth-refresh --format json | jq -r '.credential.access_token')"
+export APIGEE_ACCESS_TOKEN
+
+# Building API Proxy Bundle for Proxy containing more than 5 Proxy Endpoints
+cd "$SCRIPTPATH/test/api_bundles"
+rm -rf "$SCRIPTPATH/test/api_bundles/test.zip"
+echo "Building original proxy bundle"
+zip -q -r test.zip apiproxy/
+cd "$SCRIPTPATH"
+
+# Validating API Proxy Bundle for Proxy containing more than 5 Proxy Endpoints
+echo "Validating the original proxy bundle"
+python3 -c "import os, sys ,json; \
+            from apigee import Apigee; \
+            x = Apigee(os.getenv('APIGEE_X_ORG')); \
+            x.set_auth_header(os.getenv('APIGEE_ACCESS_TOKEN')); \
+            r=x.validate_api('apis','test/api_bundles/test.zip'); \
+            print(json.dumps(r,indent=2))"
+rm -rf "$SCRIPTPATH/test/api_bundles/test.zip"
+
+# Running and Validating API Proxy Bundle after splitting the proxies
+python3 "$SCRIPTPATH/main.py"

tools/proxy-endpoint-unifier/requirements.txt

@@ -0,0 +1,16 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+xmltodict==0.13.0
+requests==2.28.1

tools/proxy-endpoint-unifier/test/api_bundles/apiproxy/policies/ExtractVariables-3.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+    Copyright 2023 Google LLC
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+        http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<ExtractVariables name="ExtractVariables-3">
+    <Source>response</Source>
+    <JSONPayload>
+        <Variable name="latitude" type="float">
+            <JSONPath>$.results[0].geometry.location.lat</JSONPath>
+        </Variable>
+        <Variable name="longitude" type="float">
+            <JSONPath>$.results[0].geometry.location.lng</JSONPath>
+        </Variable>
+    </JSONPayload>
+    <VariablePrefix>geocoderesponse</VariablePrefix>
+</ExtractVariables>