From d74b03f28980d7506e0586050af70d252c3bb992 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Wed, 7 Feb 2024 20:55:56 +1000 Subject: [PATCH] fix: deployment scripts, add .envtemplate, add init environment variables --- .../adf_cicd_auto_publish/.envtemplate | 33 +++++++++++ .../adf_cicd_auto_publish/README.md | 6 +- .../{scripts => }/deploy_infra.sh | 55 ++++++++++++------- .../deploy_azdo_service_connections_azure.sh | 23 ++++---- .../scripts/deploy_azdo_variables.sh | 6 +- .../scripts/init_environment.sh | 51 +++++++++++++++++ 6 files changed, 136 insertions(+), 38 deletions(-) create mode 100644 single_tech_samples/datafactory/adf_cicd_auto_publish/.envtemplate rename single_tech_samples/datafactory/adf_cicd_auto_publish/{scripts => }/deploy_infra.sh (81%) create mode 100644 single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/init_environment.sh diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/.envtemplate b/single_tech_samples/datafactory/adf_cicd_auto_publish/.envtemplate new file mode 100644 index 000000000..10b991836 --- /dev/null +++ b/single_tech_samples/datafactory/adf_cicd_auto_publish/.envtemplate @@ -0,0 +1,33 @@ +### +# REQUIRED. Azure DevOps details used in "az devops configure" when devcontainer is built + +# Target Azure DevOps project where Azure Pipelines and Variable groups will be deploy +export AZDO_PROJECT= + +# Target Azure DevOps Organization of Azure DevOps project in this form `https://dev.azure.com//`. (ei. https://dev.azure.com/my_azdo_org/) +export AZDO_ORGANIZATION_URL= + + +### +# REQUIRED. Environment variables for deployment script + +# a Azure DevOps PAT token. More info [here](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows). This requires "repo" scope. +export AZURE_DEVOPS_EXT_PAT= + + +### +# OPTIONAL. Environment variables for deployment script. +# Uncomment to set values. + +# Environment name. Default: dev +# export ENV_NAME= + +# Azure location to deploy resources. Default: westus +# export AZURE_LOCATION= + +# Azure subscription id to use to deploy resources. Default: default azure subscription. +# export AZURE_SUBSCRIPTION_ID= + +# String appended to all resource names to ensure uniqueness of azure resource names. Default: random five character string +# export DEPLOYMENT_ID= + diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/README.md b/single_tech_samples/datafactory/adf_cicd_auto_publish/README.md index 233e7f602..cf960ee46 100644 --- a/single_tech_samples/datafactory/adf_cicd_auto_publish/README.md +++ b/single_tech_samples/datafactory/adf_cicd_auto_publish/README.md @@ -12,14 +12,14 @@ In ADF, CI/CD essentially means deploying the various data factory entities such ### ADF Modes -1) ADF consists of two modes namely `live mode` and `Git Mode`. +1. ADF consists of two modes namely `live mode` and `Git Mode`. - Live mode is the standalone ADF which is not connected to Git. All operations on ADF happens via Portal. - Git Mode is when ADF is connected to Git and all the ADF entities are stored as code in Git. -2)Git integration in ADF consists of selecting two branches Collaboration branch and Publish branch. -3)Collaboration branch is where all the feature branch is merged (will be mapped to ‘develop’ branch for our case). Publish branch is where all the changes including auto generated ARM templates gets published (by default, ADF creates ‘adf_publish’ branch for that). +2. Git integration in ADF consists of selecting two branches Collaboration branch and Publish branch. +3. Collaboration branch is where all the feature branch is merged (will be mapped to ‘develop’ branch for our case). Publish branch is where all the changes including auto generated ARM templates gets published (by default, ADF creates ‘adf_publish’ branch for that). > NOTE: Only the ADF instance in the DEV environment should be linked to git. The deployment are propagated to the other environments by CI/CD pipelines. diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_infra.sh b/single_tech_samples/datafactory/adf_cicd_auto_publish/deploy_infra.sh similarity index 81% rename from single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_infra.sh rename to single_tech_samples/datafactory/adf_cicd_auto_publish/deploy_infra.sh index 530b4c77e..13edab0f7 100644 --- a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_infra.sh +++ b/single_tech_samples/datafactory/adf_cicd_auto_publish/deploy_infra.sh @@ -28,23 +28,25 @@ set -o errexit set -o pipefail set -o nounset -# set -o xtrace # For debugging +set -o xtrace # For debugging ################### # REQUIRED ENV VARIABLES: # # ENV_NAME -# RESOURCE_GROUP_NAME -# RESOURCE_GROUP_LOCATION -# AZURE_SUBSCRIPTION_ID +# AZDO_PROJECT +# AZDO_ORGANIZATION_URL +# AZURE_DEVOPS_EXT_PAT + +. ./scripts/init_environment.sh # add by Bo for test # step 1 TODO Add by Bo -export ENV_NAME="botestadf" -export RESOURCE_GROUP_NAME="botestadf" -export RESOURCE_GROUP_LOCATION="eastus" -export AZURE_SUBSCRIPTION_ID="" -export DEPLOYMENT_ID="2w4" +# export ENV_NAME="botestadf" +# export RESOURCE_GROUP_NAME="botestadf" +# export AZURE_LOCATION="eastus" +# export AZURE_SUBSCRIPTION_ID="" +# export DEPLOYMENT_ID="2w4" #step 2 variable for deploy_azdo_service_connections_azure.sh # export AZURE_DEVOPS_ORG="bwa0800" @@ -58,13 +60,10 @@ export DEPLOYMENT_ID="2w4" ##################### # DEPLOY ARM TEMPLATE -# Set account to where ARM template will be deployed to -echo "Deploying to Subscription: $AZURE_SUBSCRIPTION_ID" -az account set --subscription $AZURE_SUBSCRIPTION_ID - # Create resource group -echo "Creating resource group: $RESOURCE_GROUP_NAME" -az group create --name "$RESOURCE_GROUP_NAME" --location "$RESOURCE_GROUP_LOCATION" --tags Environment=$ENV_NAME +resource_group_name="adf-$DEPLOYMENT_ID-$ENV_NAME-rg" +echo "Creating resource group: $resource_group_name" +az group create --name "$resource_group_name" --location "$AZURE_LOCATION" --tags Environment=$ENV_NAME # By default, set all KeyVault permission to deployer # Retrieve KeyVault User Id @@ -75,7 +74,7 @@ kv_owner_object_id=$(az ad signed-in-user show --output json | jq -r '.id') # add by bo ===> The default parameter can be used for this parameter and default value at ../infra/azuredeploy.json file # --parameters @"azuredeploy.parameters.${ENV_NAME}.json" \ # Deploy arm template -echo "Deploying resources into $RESOURCE_GROUP_NAME" +echo "Deploying resources into $resource_group_name" arm_output=$(az deployment group create \ --resource-group "$RESOURCE_GROUP_NAME" \ --template-file "../infra/azuredeploy.json" \ @@ -146,8 +145,22 @@ export KV_URL=https://$kv_name.vault.azure.net/ #################### # Set up AZDO Azure Service Connection and Variables Groups -. ./deploy_azdo_service_connections_azure.sh -. ./deploy_azdo_variables.sh +# AzDO Azure Service Connections +PROJECT=$PROJECT \ +ENV_NAME=$ENV_NAME \ +RESOURCE_GROUP_NAME=$resource_group_name \ +AZDO_ORGANIZATION_URL=$AZDO_ORGANIZATION_URL \ +AZURE_DEVOPS_EXT_PAT=$AZURE_DEVOPS_EXT_PAT \ + bash -c "./scripts/deploy_azdo_service_connections_azure.sh" + +ENV_NAME=$ENV_NAME +AZURE_LOCATION=$AZURE_LOCATION +RESOURCE_GROUP_NAME=$RESOURCE_GROUP_NAME +DATAFACTORY_NAME=$DATAFACTORY_NAME +AZURE_SUBSCRIPTION_ID=$AZURE_SUBSCRIPTION_ID +DL_STORAGE_ACCOUNT=$DL_STORAGE_ACCOUNT + bash -c "./scripts/deploy_azdo_variables.sh" + #################### # BUILD ENV FILE FROM CONFIG INFORMATION @@ -156,8 +169,8 @@ env_file=".env.${ENV_NAME}" echo "Appending configuration to .env file." cat << EOF >> $env_file -RESOURCE_GROUP_NAME=${RESOURCE_GROUP_NAME} -RESOURCE_GROUP_LOCATION=${RESOURCE_GROUP_LOCATION} +RESOURCE_GROUP_NAME=${resource_group_name} +AZURE_LOCATION=${AZURE_LOCATION} DL_STORAGE_ACCOUNT=${DL_STORAGE_ACCOUNT} DL_STORAGE_KEY=${DL_STORAGE_KEY} DATAFACTORY_NAME=${DATAFACTORY_NAME} @@ -165,4 +178,4 @@ KV_URL=${KV_URL} EOF -echo "Completed deploying Azure resources $RESOURCE_GROUP_NAME ($ENV_NAME)" \ No newline at end of file +echo "Completed deploying Azure resources $resource_group_name ($ENV_NAME)" \ No newline at end of file diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_service_connections_azure.sh b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_service_connections_azure.sh index dd779be32..f0095369e 100644 --- a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_service_connections_azure.sh +++ b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_service_connections_azure.sh @@ -38,20 +38,19 @@ set -o nounset # RESOURCE_GROUP_NAME # DEPLOYMENT_ID # AZURE_DEVOPS_ORG -# AZURE_DEVOPS_PROJECT +# AZDO_ORGANIZATION_URL # AZURE_DEVOPS_EXT_PAT # add by Bo for test -export ENV_NAME="botestadf" -export RESOURCE_GROUP_NAME="botestadf" -export RESOURCE_GROUP_LOCATION="eastus" -export AZURE_SUBSCRIPTION_ID="" -export DEPLOYMENT_ID="2w4" - -export AZURE_DEVOPS_ORG="bwa0800" -export AZURE_DEVOPS_PROJECT="adf-test" -export AZURE_DEVOPS_EXT_PAT="" +# export ENV_NAME="botestadf" +# export RESOURCE_GROUP_NAME="botestadf" +# export RESOURCE_GROUP_LOCATION="eastus" +# export AZURE_SUBSCRIPTION_ID="" +# export DEPLOYMENT_ID="2w4" +# export AZURE_DEVOPS_ORG="bwa0800" +# export AZDO_ORGANIZATION_URL="adf-test" +# export AZURE_DEVOPS_EXT_PAT="" az_service_connection_name="mdws-adf-serviceconnection-$ENV_NAME" @@ -75,10 +74,10 @@ az_sp=$(az ad sp create-for-rbac \ export SERVICE_PRINCIPAL_ID=$(echo $az_sp | jq -r '.appId') # error ---> Failed to authenticate using the supplied token. (hard-coded) -echo $AZURE_DEVOPS_EXT_PAT | az devops login --organization https://dev.azure.com/$AZURE_DEVOPS_ORG +echo $AZURE_DEVOPS_EXT_PAT | az devops login --organization $AZDO_ORGANIZATION_URL az_sp_tenant_id=$(echo $az_sp | jq -r '.tenant') -az devops configure --defaults organization=https://dev.azure.com/$AZURE_DEVOPS_ORG project=$AZURE_DEVOPS_PROJECT +az devops configure --defaults organization=$AZDO_ORGANIZATION_URL project=$AZURE_DEVOPS_PROJECT # Create Azure Service connection in Azure DevOps diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_variables.sh b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_variables.sh index 199cab712..1fcaec068 100644 --- a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_variables.sh +++ b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_variables.sh @@ -36,11 +36,13 @@ set -o nounset # REQUIRED ENV VARIABLES: # # ENV_NAME -# RESOURCE_GROUP_LOCATION +# AZURE_LOCATION # RESOURCE_GROUP_NAME # DATAFACTORY_NAME # AZURE_SUBSCRIPTION_ID # DL_STORAGE_ACCOUNT + + # DL_STORAGE_KEY # FS_STORAGE_ACCOUNT # FS_STORAGE_KEY @@ -52,7 +54,7 @@ az pipelines variable-group create \ --name "$vargroup_name" \ --authorize "true" \ --variables \ - location="$RESOURCE_GROUP_LOCATION" \ + location="$AZURE_LOCATION" \ resource_group_name="$RESOURCE_GROUP_NAME" \ azure_data_factory_name="$DATAFACTORY_NAME" \ azure_subscription_id="$AZURE_SUBSCRIPTION_ID" \ diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/init_environment.sh b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/init_environment.sh new file mode 100644 index 000000000..e812484d6 --- /dev/null +++ b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/init_environment.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# check required variables are specified. + +if [ -z "$AZDO_PROJECT" ] +then + echo "Please specify a target Azure DevOps project where Azure Pipelines and Variable groups will be deploy using AZDO_PROJECT environment variable" + exit 1 +fi + +if [ -z "$AZDO_ORGANIZATION_URL" ] +then + echo "Please specify a Azure DevOps organization url using the AZDO_ORGANIZATION_URL environment variable in this form: https://dev.azure.com//" + exit 1 +fi + +if [ -z "$AZURE_DEVOPS_EXT_PAT" ] +then + echo "Please specify a Azure DevOps PAT token using the AZURE_DEVOPS_EXT_PAT environment variable." + exit 1 +fi + +# initialise optional variables. + +ENV_NAME=${ENV_NAME:-} +if [ -z "$ENV_NAME" ] +then + export ENV_NAME="dev" + echo "No environment name [ENV_NAME] specified, defaulting to $ENV_NAME" +fi + +DEPLOYMENT_ID=${DEPLOYMENT_ID:-} +if [ -z "$DEPLOYMENT_ID" ] +then + export DEPLOYMENT_ID="$(random_str 5)" + echo "No deployment id [DEPLOYMENT_ID] specified, defaulting to $DEPLOYMENT_ID" +fi + +AZURE_LOCATION=${AZURE_LOCATION:-} +if [ -z "$AZURE_LOCATION" ] +then + export AZURE_LOCATION="westus" + echo "No resource group location [AZURE_LOCATION] specified, defaulting to $AZURE_LOCATION" +fi + +AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID:-} +if [ -z "$AZURE_SUBSCRIPTION_ID" ] +then + export AZURE_SUBSCRIPTION_ID=$(az account show --output json | jq -r '.id') + echo "No Azure subscription id [AZURE_SUBSCRIPTION_ID] specified. Using default subscription id." +fi