Merge branch 'release-2.0.0' into 'master'

Release 2.0.0

See merge request automation-sdk/f5-cloud-failover!393

.gitlab-ci.yml

@@ -4,11 +4,12 @@ variables:
   GIT_SECRETS_VERSION: "1.3.0"
   GIT_SECRETS_PKG_URL: "https://github.com/awslabs/git-secrets/archive/${GIT_SECRETS_VERSION}.tar.gz"
   ARTIFACTORY_SERVER: "$ARTIFACTORY_SERVER"
+  ATG_IMG_REGISTRY_URL: ${ARTIFACTORY_SERVER}/dockerhub-remote
   DOCS_BUILD_DIR: "docs/_build/html"
   POSTMAN_EXAMPLES: "postman-collection"
   MONGO_INITDB_ROOT_PASSWORD: root
   MONGO_INITDB_ROOT_USERNAME: root
-  DT_IMAGE_V2: ${ARTIFACTORY_SERVER}/ecosystems-cloudsolutions-docker-dev/deployment-tool-v2:latest
+  DT_IMAGE_V3: ${ARTIFACTORY_SERVER}/ecosystems-cloudsolutions-docker-dev/deployment-tool-v3:latest
   # azure vars - normalize env var options between ansible and terraform
   AZURE_TENANT: "$AZURE_TENANT_ID"
   AZURE_SECRET: "$AZURE_CLIENT_SECRET"
@@ -27,9 +28,11 @@ variables:
   F5_SDK_LOG_LEVEL: "INFO"
   F5_DISABLE_CERT_VERIFY: "true"
   PYTHONWARNINGS: "ignore:Unverified HTTPS request"
+  SEC_TOML_BRANCH: main
 
 stages:
   - check_content
+  - atg-shared-checks
   - test
   - build_package
   - test_functional_init
@@ -39,6 +42,10 @@ stages:
   - build_docs
   - publish
 
+include:
+  - project: automation-toolchain/atg-shared-templates
+    file: security/base.gitlab-ci.yml
+
 .helpers: &helpers |
   function validate () {
       "$@"
@@ -155,16 +162,18 @@ build_package:
 # Functional Tests - Initialization phase (with 1 retries in a case of any failures)
 .test-install:
   stage: test_functional_init
-  image: $DT_IMAGE_V2
+  image: $DT_IMAGE_V3
   services:
     - alias: mongo
       name: ${ARTIFACTORY_SERVER}/dockerhub-remote/mongo:4.0-xenial
   tags:
     - azure-autoscale-runner
   script:
-    - export DEPLOYMENT_TOOL_VERSION=latest; curl -L https://cdn.f5.com/product/cloudsolutions/deployment-tool/${DEPLOYMENT_TOOL_VERSION}/install.sh | bash
-    - source ~/.bash_profile
-    - deployment_tool --command create --plan $PLAN --name ci-$CI_JOB_ID --database export --parameters "$PARAMETERS"
+    - deployment-tool create ci-$CI_JOB_ID --plan $PLAN $PARAMETERS
+    - deployment-tool database export --uri "mongodb://root:root@mongo:27017/porter?authSource=admin" --destination ./deployment/
+    - deployment-tool describe ci-$CI_JOB_ID
+    - deployment-tool get ci-$CI_JOB_ID --output-format json | jq > ./deployment/deployment_info.json
+    - cat ./deployment/deployment_info.json | jq
     - echo "ci-$CI_JOB_ID" > ./deployment/name.txt
   only:
     variables:
@@ -189,7 +198,7 @@ test_functional_init_azure:
   extends: .test-install
   variables:
     PLAN: "azure"
-    PARAMETERS: "location=westus2 bigip_version=16.1.303000"
+    PARAMETERS: "--param location=westus2 --param bigip_version=16.1.303000"
   except:
     variables:
       - $TESTS_TIER == "2"
@@ -202,7 +211,7 @@ test_functional_init_azure_1nic:
   extends: .test-install
   variables:
     PLAN: "azure"
-    PARAMETERS: "nic_count=1 location=westus2 bigip_version=16.1.303000"
+    PARAMETERS: "--param nic_count=1 --param location=westus2 --param bigip_version=16.1.303000"
   when: manual
   except:
     variables:
@@ -217,7 +226,7 @@ test_functional_init_aws_across_net:
   extends: .test-install
   variables:
     PLAN: "aws"
-    PARAMETERS: "region=us-west-2"
+    PARAMETERS: "--param region=us-west-2"
   except:
     variables:
       - $TESTS_TIER == "2"
@@ -230,7 +239,7 @@ test_functional_init_aws_across_net_1nic:
   extends: .test-install
   variables:
     PLAN: "aws"
-    PARAMETERS: "region=us-west-2 nic_count=1"
+    PARAMETERS: "--param region=us-west-2 --param nic_count=1"
   when: manual
   except:
     variables:
@@ -245,7 +254,7 @@ test_functional_init_aws:
   extends: .test-install
   variables:
     PLAN: "aws"
-    PARAMETERS: "region=us-west-2 use_availability_zones=false"
+    PARAMETERS: "--param region=us-west-2 --param use_availability_zones=false"
   except:
     variables:
       - $TESTS_TIER == "2"
@@ -259,7 +268,7 @@ test_functional_init_aws_1nic:
   extends: .test-install
   variables:
     PLAN: "aws"
-    PARAMETERS: "region=us-west-2 use_availability_zones=false nic_count=1"
+    PARAMETERS: "--param region=us-west-2 --param use_availability_zones=false --param nic_count=1"
   when: manual
   except:
     variables:
@@ -274,7 +283,7 @@ test_functional_init_gcp:
   extends: .test-install
   variables:
     PLAN: "gcp"
-    PARAMETERS: "region=us-west1 bigip_version=f5-bigip-16-1-3-3-0-0-3-payg-best-plus-25mbps-221222234728"
+    PARAMETERS: "--param region=us-west1 --param bigip_version=f5-bigip-16-1-3-3-0-0-3-payg-best-plus-25mbps-221222234728"
   except:
     variables:
       - $TESTS_TIER == "2"
@@ -287,7 +296,7 @@ test_functional_init_gcp_no_forwarding:
   extends: .test-install
   variables:
     PLAN: "gcp"
-    PARAMETERS: "region=us-west1 use_forwarding_rule=false bigip_version=f5-bigip-16-1-3-3-0-0-3-payg-best-plus-25mbps-221222234728"
+    PARAMETERS: "--param region=us-west1 --param use_forwarding_rule=false --param bigip_version=f5-bigip-16-1-3-3-0-0-3-payg-best-plus-25mbps-221222234728"
   except:
     variables:
       - $TESTS_TIER == "1"
@@ -463,7 +472,7 @@ test_functional_execute_gcp_no_forwarding:
 # Functional Tests - Cleanup phase (executes always with 1 retry in a case of any failures)
 .test-uninstall:
   stage: test_functional_cleanup
-  image: $DT_IMAGE_V2
+  image: $DT_IMAGE_V3
   retry: 1
   when: always
   services:
@@ -473,7 +482,8 @@ test_functional_execute_gcp_no_forwarding:
     - azure-autoscale-runner
   script:
     - mkdir -p /deployment-tool/deployment && cp -a ./deployment/. /deployment-tool/deployment
-    - cd /deployment-tool && ./deployment_tool.sh --command delete --plan $PLAN --name $(cat ./deployment/name.txt) --database import --parameters "$PARAMETERS"
+    - deployment-tool database import --uri "mongodb://root:root@mongo:27017/porter?authSource=admin" --backup-path ./deployment/porter
+    - deployment-tool delete $(cat ./deployment/name.txt)
   only:
     variables:
       - $CI_COMMIT_REF_NAME == "master"

docs/revision-history.rst

@@ -11,7 +11,11 @@ Document Revision History
         - Description
         - Date
 
-        * - 2.2.7
+      * - 2.3.0
+        - Updated the documentation for Cloud Failover Extension with the following changes: |br| • Updated documentation in :ref:`troubleshooting`and :ref:`azure`.
+        - 12-13-23
+
+      * - 2.2.7
         - Updated the documentation for Cloud Failover Extension with the following changes: |br| • Updated documentation in :ref:`troubleshooting`.
         - 03-14-23
 

docs/userguide/aws-same-az.rst

@@ -468,9 +468,14 @@ Define Remote Storage for State File in AWS
    - The property ``scopingName`` is available in Cloud Failover Extension v1.7.0 and later.
    - Beginning v1.13.0, CFE supports Serverside Encryption on the S3 Bucket using Amazon S3-Managed Keys (SSE-S3) or KMS keys Stored in AWS Key Management Service (SSE-KMS) with either the default AWS managed key or a customer managed key. See `AWS Documentation <https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html>`_ for more details on how to enable server-side encryption on the S3 bucket.
 
+
+
 1. Create an `S3 bucket in AWS <https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html>`_ for Cloud Failover Extension cluster-wide file(s). 
 
-   .. WARNING:: To avoid a potential data breach, ensure the required S3 buckets are properly secured and do not have public access. See your cloud provider for best practices.
+   .. WARNING:: 
+      - Amazon S3 now applies server-side encryption with Amazon S3 managed keys (SSE-S3) as the base level of encryption for every bucket in Amazon S3. Starting January 5, 2023, all new object uploads to Amazon S3 are automatically encrypted.
+      - Although SSE-S3 is applied automatcially, the IAM policy to require encrypted uploads must be created separately. To avoid a potential data breach, ensure the required S3 buckets are properly secured per the IAM role examples provided above, and do not allow public access. See your cloud provider for best practices.
+      - The ``encryption`` configuration block is required when an IAM policy that blocks unencrypted uploads is applied to the S3 bucket (recommended).
 
 
 2. Update/modify the Cloud Failover ``scopingName`` value with name of your S3 bucket:

docs/userguide/aws.rst

@@ -447,12 +447,15 @@ Define Remote Storage for State File in AWS
 
    - The property ``scopingName`` is available in Cloud Failover Extension v1.7.0 and later.
    - Beginning v1.13.0, CFE supports Serverside Encryption on the S3 Bucket using Amazon S3-Managed Keys (SSE-S3) or KMS keys Stored in AWS Key Management Service (SSE-KMS) with either the default AWS managed key or a customer managed key. See `AWS Documentation <https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html>`_ for more details on how to enable server-side encryption on the S3 bucket.
-   
+
 
 
 1. Create an `S3 bucket in AWS <https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html>`_ for Cloud Failover Extension cluster-wide file(s). 
 
-   .. WARNING:: To avoid a potential data breach, ensure the required S3 buckets are properly secured and do not have public access. See your cloud provider for best practices.
+   .. WARNING:: 
+      - Amazon S3 now applies server-side encryption with Amazon S3 managed keys (SSE-S3) as the base level of encryption for every bucket in Amazon S3. Starting January 5, 2023, all new object uploads to Amazon S3 are automatically encrypted.
+      - Although SSE-S3 is applied automatcially, the IAM policy to require encrypted uploads must be created separately. To avoid a potential data breach, ensure the required S3 buckets are properly secured per the IAM role examples provided above, and do not allow public access. See your cloud provider for best practices.
+      - The ``encryption`` configuration block is required when an IAM policy that blocks unencrypted uploads is applied to the S3 bucket (recommended).
 
 
 2. Update/modify the Cloud Failover ``scopingName`` value with **name** of your S3 bucket:

docs/userguide/azure.rst

@@ -133,23 +133,26 @@ RBAC Role Definition
 
 Below is an example Azure role definition with permissions required by CFE.
 
-======================================================== ==================================== ======================= ======================================================================================== 
- Name                                                     Scope                                CFE Component           Description   
-======================================================== ==================================== ======================= ======================================================================================== 
- Microsoft.Authorization/\*/read                          Roles and Role Assignments - Read    All                     To authenticate using the managed identity.        
- Microsoft.Compute/locations/\*/read                      Locations - Read                     All                     To get the Azure location.
- Microsoft.Compute/virtualMachines/\*/read                Virtual Machines - Read              All                     To get information about a BIG-IP instance.   
- Microsoft.Network/\*/join/action                         Network Provider - Join              All                     To update network. 
- Microsoft.Network/networkInterfaces/read                 Network Interfaces - Read            All                     To get information about a network interface.
- Microsoft.Network/networkInterfaces/write                Network Interfaces - Write           All                     To update a network interface to use the active BIG-IP instance.
- Microsoft.Network/publicIPAddresses/read                 Public IP Addresses - Read           failoverAddresses       To get information about a public IP address.
- Microsoft.Network/publicIPAddresses/write                Public IP Addresses - Write          failoverAddresses       To update a public IP address to use the active BIG-IP instance.
- Microsoft.Network/routeTables/\*/read                    Route Tables - Read                  failoverRoutes          To get information about a route table.
- Microsoft.Network/routeTables/\*/write                   Route Tables - Write                 failoverRoutes          To update route next hop to use the active BIG-IP instance.
- Microsoft.Resources/subscriptions/resourceGroups/read    Resource Groups - Read               All                     To get information about a resource group.
- Microsoft.Storage/storageAccounts/listKeys/action        Storage Account Keys - List          externalStorage         To get the key for creating/writing/deleting the failover state file.
- Microsoft.Storage/storageAccounts/read                   Storage Accounts - Read              externalStorage         To get information about the storage account used for the failover state file.
-======================================================== ==================================== ======================= ======================================================================================== 
+======================================================================  ==================================== ======================= ======================================================================================== 
+ Name                                                                   Scope                                CFE Component           Description   
+======================================================================  ==================================== ======================= ======================================================================================== 
+ Microsoft.Authorization/\*/read                                        Roles and Role Assignments - Read    All                     To authenticate using the managed identity.        
+ Microsoft.Compute/locations/\*/read                                    Locations - Read                     All                     To get the Azure location.
+ Microsoft.Compute/virtualMachines/\*/read                              Virtual Machines - Read              All                     To get information about a BIG-IP instance.   
+ Microsoft.Network/\*/join/action                                       Network Provider - Join              All                     To update network. 
+ Microsoft.Network/networkInterfaces/read                               Network Interfaces - Read            All                     To get information about a network interface.
+ Microsoft.Network/networkInterfaces/write                              Network Interfaces - Write           All                     To update a network interface to use the active BIG-IP instance.
+ Microsoft.Network/publicIPAddresses/read                               Public IP Addresses - Read           failoverAddresses       To get information about a public IP address.
+ Microsoft.Network/publicIPAddresses/write                              Public IP Addresses - Write          failoverAddresses       To update a public IP address to use the active BIG-IP instance.
+ Microsoft.Network/routeTables/\*/read                                  Route Tables - Read                  failoverRoutes          To get information about a route table.
+ Microsoft.Network/routeTables/\*/write                                 Route Tables - Write                 failoverRoutes          To update route next hop to use the active BIG-IP instance.
+ Microsoft.Resources/subscriptions/resourceGroups/read                  Resource Groups - Read               All                     To get information about a resource group.
+ Microsoft.Storage/storageAccounts/read                                 Storage Accounts - Read              externalStorage         To get information about the storage account used for the failover state file.
+ Microsoft.Storage/storageAccounts/blobServices/containers/read         Storage Containers - Read            externalStorage         To get information about the storage container used for the failover state file.
+ Microsoft.Storage/storageAccounts/blobServices/containers/write        Storage Containers - Write           externalStorage         To create the storage container used for the failover state file.
+ Microsoft.Storage/storageAccounts/blobServices/containers/blobs/read   Storage Blobs - Read                 externalStorage         To get information about the storage blob used for the failover state file.
+ Microsoft.Storage/storageAccounts/blobServices/containers/blobs/write  Storage Blobs - Write                externalStorage         To create the storage blob used for the failover state file.
+======================================================================  ==================================== ======================= ======================================================================================== 
 
    |
 
@@ -160,6 +163,7 @@ Below is an example Azure role definition with permissions required by CFE.
    - Certain resources such as the virtual network are commonly deployed in a separate resource group; ensure the correct scopes are applied to all applicable resource groups.
    - Certain resources such as route tables may be deployed in a separate subscription, ensure the assignable scopes applies to all relevant subscriptions.
    - CFE supports only **one** Managed Service Identity assigned to each Azure Virtual Machine instance; failover will not function when multiple identities are assigned. You must create a single identity with all of the permissions listed above, as well as any other required permissions. You can create a managed identity manually, or by using the F5 access template. See `Deploying Access Template <https://github.com/F5Networks/f5-azure-arm-templates-v2/tree/main/examples/modules/access>`_ for more information.
+   - Previous versions of CFE required the **Microsoft.Storage/storageAccounts/listKeys/action** role definition permission; this requirement has been superseded by the **Microsoft.Storage/storageAccounts/blobServices/containers/** data actions permissions listed above.
 
 |
 

docs/userguide/example-declarations.rst

@@ -110,7 +110,10 @@ Azure Route Tables in Multiple Subscriptions
 --------------------------------------------
 This example shows a BIG-IP cluster managing route tables in multiple subscriptions. The identity (MSI) assigned to each BIG-IP instance must have appropriate access to the additional subscriptions, see :ref:`azure-msi` for more details. 
 
-.. Note:: By default, the cloud failover extension looks in the subscription in which the instances are deployed. The example below looks in three different subscriptions, the one the instances are deployed in as well as 1111 and 2222.
+.. Note:: 
+
+   - By default, the cloud failover extension looks in the subscription in which the instances are deployed. The example below looks in three different subscriptions, the one the instances are deployed in as well as 1111 and 2222.
+   - Beginning with v1.14, the Route Table **not** the routes are updated.
 
 .. literalinclude:: ../../examples/declarations/azureRouteTablesInMutipleSubscriptions.json
    :language: json

docs/userguide/prereqs.rst

@@ -11,7 +11,7 @@ The following are prerequisites for using Cloud Failover:
   role.
 - You should be familiar with the F5 BIG-IP and F5 terminology. For
   general information and documentation on the BIG-IP system, see the
-  `F5 Knowledge Center <https://support.f5.com/csp/knowledge-center/software/BIG-IP?module=BIG-IP%20LTM&version=13.1.0>`_.
+  `F5 Knowledge Center <https://techdocs.f5.com/en-us/bigip-14-1-0/big-ip-local-traffic-management-basics-14-1-0.html>`_.
 - Each provider has their own prerequisites, see the invididual provider sections for more information.
 - Your BIG-IPs must have DNS and NTP setup.