Azure-Samples · devlace · Feb 7, 2024
diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/.envtemplate b/single_tech_samples/datafactory/adf_cicd_auto_publish/.envtemplate
@@ -0,0 +1,33 @@
+###
+# REQUIRED. Azure DevOps details used in "az devops configure" when devcontainer is built
+
+# Target Azure DevOps project where Azure Pipelines and Variable groups will be deploy
+export AZDO_PROJECT=
+
+# Target Azure DevOps Organization of Azure DevOps project in this form `https://dev.azure.com/<organization>/`. (ei. https://dev.azure.com/my_azdo_org/)
+export AZDO_ORGANIZATION_URL=
+
+
+###
+# REQUIRED. Environment variables for deployment script
+
+# a Azure DevOps PAT token. More info [here](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows). This requires "repo" scope.
+export AZURE_DEVOPS_EXT_PAT=
+
+
+###
+# OPTIONAL. Environment variables for deployment script.
+# Uncomment to set values.
+
+# Environment name. Default: dev
+# export ENV_NAME=
+
+# Azure location to deploy resources. Default: westus
+# export AZURE_LOCATION=
+
+# Azure subscription id to use to deploy resources. Default: default azure subscription.
+# export AZURE_SUBSCRIPTION_ID=
+
+# String appended to all resource names to ensure uniqueness of azure resource names. Default: random five character string
+# export DEPLOYMENT_ID=
+
diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/README.md b/single_tech_samples/datafactory/adf_cicd_auto_publish/README.md
@@ -12,14 +12,14 @@ In ADF, CI/CD essentially means deploying the various data factory entities such
 
 ### ADF Modes
 
-1) ADF consists of two modes namely `live mode` and `Git Mode`.
+1. ADF consists of two modes namely `live mode` and `Git Mode`.
 
 - Live mode is the standalone ADF which is not connected to Git. All operations on ADF happens via Portal.
 
 - Git Mode is when ADF is connected to Git and all the ADF entities are stored as code in Git.
 
-2)Git integration in ADF consists of selecting two branches Collaboration branch and Publish branch.
-3)Collaboration branch is where all the feature branch is merged (will be mapped to ‘develop’ branch for our case). Publish branch is where all the changes including auto generated ARM templates gets published (by default, ADF creates ‘adf_publish’ branch for that).
+2. Git integration in ADF consists of selecting two branches Collaboration branch and Publish branch.
+3. Collaboration branch is where all the feature branch is merged (will be mapped to ‘develop’ branch for our case). Publish branch is where all the changes including auto generated ARM templates gets published (by default, ADF creates ‘adf_publish’ branch for that).
 
 > NOTE: Only the ADF instance in the DEV environment should be linked to git. The deployment are propagated to the other environments by CI/CD pipelines.
 

diff --git a/...cicd_auto_publish/scripts/deploy_infra.sh → ...ory/adf_cicd_auto_publish/deploy_infra.sh b/...cicd_auto_publish/scripts/deploy_infra.sh → ...ory/adf_cicd_auto_publish/deploy_infra.sh
@@ -28,23 +28,25 @@
 set -o errexit
 set -o pipefail
 set -o nounset
-# set -o xtrace # For debugging
+set -o xtrace # For debugging
 
 ###################
 # REQUIRED ENV VARIABLES:
 #
 # ENV_NAME
-# RESOURCE_GROUP_NAME
-# RESOURCE_GROUP_LOCATION
-# AZURE_SUBSCRIPTION_ID
+# AZDO_PROJECT
+# AZDO_ORGANIZATION_URL
+# AZURE_DEVOPS_EXT_PAT
+
+. ./scripts/init_environment.sh
 
 # add by Bo for test  
 # step 1 TODO Add by Bo
-export ENV_NAME="botestadf"
-export RESOURCE_GROUP_NAME="botestadf"
-export RESOURCE_GROUP_LOCATION="eastus"
-export AZURE_SUBSCRIPTION_ID=""
-export DEPLOYMENT_ID="2w4"
+# export ENV_NAME="botestadf"
+# export RESOURCE_GROUP_NAME="botestadf"
+# export AZURE_LOCATION="eastus"
+# export AZURE_SUBSCRIPTION_ID=""
+# export DEPLOYMENT_ID="2w4"
 
 #step 2 variable for deploy_azdo_service_connections_azure.sh
 # export AZURE_DEVOPS_ORG="bwa0800"
@@ -58,13 +60,10 @@ export DEPLOYMENT_ID="2w4"
 #####################
 # DEPLOY ARM TEMPLATE
 
-# Set account to where ARM template will be deployed to
-echo "Deploying to Subscription: $AZURE_SUBSCRIPTION_ID"
-az account set --subscription $AZURE_SUBSCRIPTION_ID
-
 # Create resource group
-echo "Creating resource group: $RESOURCE_GROUP_NAME"
-az group create --name "$RESOURCE_GROUP_NAME" --location "$RESOURCE_GROUP_LOCATION" --tags Environment=$ENV_NAME
+resource_group_name="adf-$DEPLOYMENT_ID-$ENV_NAME-rg"
+echo "Creating resource group: $resource_group_name"
+az group create --name "$resource_group_name" --location "$AZURE_LOCATION" --tags Environment=$ENV_NAME
 
 # By default, set all KeyVault permission to deployer
 # Retrieve KeyVault User Id
@@ -75,7 +74,7 @@ kv_owner_object_id=$(az ad signed-in-user show --output json | jq -r '.id')
 # add by bo ===>  The default parameter can be used for this parameter and default value at ../infra/azuredeploy.json file
 #  --parameters @"azuredeploy.parameters.${ENV_NAME}.json" \
 # Deploy arm template
-echo "Deploying resources into $RESOURCE_GROUP_NAME"
+echo "Deploying resources into $resource_group_name"
 arm_output=$(az deployment group create \
     --resource-group "$RESOURCE_GROUP_NAME" \
     --template-file "../infra/azuredeploy.json" \
@@ -146,8 +145,22 @@ export KV_URL=https://$kv_name.vault.azure.net/
 
 ####################
 # Set up AZDO Azure Service Connection and Variables Groups
-. ./deploy_azdo_service_connections_azure.sh
-. ./deploy_azdo_variables.sh
+# AzDO Azure Service Connections
+PROJECT=$PROJECT \
+ENV_NAME=$ENV_NAME \
+RESOURCE_GROUP_NAME=$resource_group_name \
+AZDO_ORGANIZATION_URL=$AZDO_ORGANIZATION_URL \
+AZURE_DEVOPS_EXT_PAT=$AZURE_DEVOPS_EXT_PAT \
+    bash -c "./scripts/deploy_azdo_service_connections_azure.sh"
+
+ENV_NAME=$ENV_NAME
+AZURE_LOCATION=$AZURE_LOCATION
+RESOURCE_GROUP_NAME=$RESOURCE_GROUP_NAME
+DATAFACTORY_NAME=$DATAFACTORY_NAME
+AZURE_SUBSCRIPTION_ID=$AZURE_SUBSCRIPTION_ID
+DL_STORAGE_ACCOUNT=$DL_STORAGE_ACCOUNT
+    bash -c "./scripts/deploy_azdo_variables.sh"
+
 
 ####################
 # BUILD ENV FILE FROM CONFIG INFORMATION
@@ -156,13 +169,13 @@ env_file=".env.${ENV_NAME}"
 echo "Appending configuration to .env file."
 cat << EOF >> $env_file
 
-RESOURCE_GROUP_NAME=${RESOURCE_GROUP_NAME}
-RESOURCE_GROUP_LOCATION=${RESOURCE_GROUP_LOCATION}
+RESOURCE_GROUP_NAME=${resource_group_name}
+AZURE_LOCATION=${AZURE_LOCATION}
 DL_STORAGE_ACCOUNT=${DL_STORAGE_ACCOUNT}
 DL_STORAGE_KEY=${DL_STORAGE_KEY}
 DATAFACTORY_NAME=${DATAFACTORY_NAME}
 KV_URL=${KV_URL}
 
 EOF
 
-echo "Completed deploying Azure resources $RESOURCE_GROUP_NAME ($ENV_NAME)"
+echo "Completed deploying Azure resources $resource_group_name ($ENV_NAME)"
diff --git a/...amples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_service_connections_azure.sh b/...amples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_service_connections_azure.sh
@@ -38,20 +38,19 @@ set -o nounset
 # RESOURCE_GROUP_NAME
 # DEPLOYMENT_ID
 # AZURE_DEVOPS_ORG
-# AZURE_DEVOPS_PROJECT
+# AZDO_ORGANIZATION_URL
 # AZURE_DEVOPS_EXT_PAT
 
 
 # add by Bo for test
-export ENV_NAME="botestadf"
-export RESOURCE_GROUP_NAME="botestadf"
-export RESOURCE_GROUP_LOCATION="eastus"
-export AZURE_SUBSCRIPTION_ID=""
-export DEPLOYMENT_ID="2w4"
-
-export AZURE_DEVOPS_ORG="bwa0800"
-export AZURE_DEVOPS_PROJECT="adf-test"
-export AZURE_DEVOPS_EXT_PAT=""
+# export ENV_NAME="botestadf"
+# export RESOURCE_GROUP_NAME="botestadf"
+# export RESOURCE_GROUP_LOCATION="eastus"
+# export AZURE_SUBSCRIPTION_ID=""
+# export DEPLOYMENT_ID="2w4"
+# export AZURE_DEVOPS_ORG="bwa0800"
+# export AZDO_ORGANIZATION_URL="adf-test"
+# export AZURE_DEVOPS_EXT_PAT=""
 
 
 az_service_connection_name="mdws-adf-serviceconnection-$ENV_NAME"
@@ -75,10 +74,10 @@ az_sp=$(az ad sp create-for-rbac \
 
 export SERVICE_PRINCIPAL_ID=$(echo $az_sp | jq -r '.appId')
 # error ---> Failed to authenticate using the supplied token. (hard-coded)
-echo $AZURE_DEVOPS_EXT_PAT | az devops login --organization https://dev.azure.com/$AZURE_DEVOPS_ORG
+echo $AZURE_DEVOPS_EXT_PAT | az devops login --organization $AZDO_ORGANIZATION_URL
 az_sp_tenant_id=$(echo $az_sp | jq -r '.tenant')
 
-az devops configure --defaults organization=https://dev.azure.com/$AZURE_DEVOPS_ORG project=$AZURE_DEVOPS_PROJECT
+az devops configure --defaults organization=$AZDO_ORGANIZATION_URL project=$AZURE_DEVOPS_PROJECT
 
 # Create Azure Service connection in Azure DevOps
 

diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_variables.sh b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/deploy_azdo_variables.sh
@@ -36,11 +36,13 @@ set -o nounset
 # REQUIRED ENV VARIABLES:
 #
 # ENV_NAME
-# RESOURCE_GROUP_LOCATION
+# AZURE_LOCATION
 # RESOURCE_GROUP_NAME
 # DATAFACTORY_NAME
 # AZURE_SUBSCRIPTION_ID
 # DL_STORAGE_ACCOUNT
+
+
 # DL_STORAGE_KEY
 # FS_STORAGE_ACCOUNT
 # FS_STORAGE_KEY
@@ -52,7 +54,7 @@ az pipelines variable-group create \
     --name "$vargroup_name" \
     --authorize "true" \
     --variables \
-        location="$RESOURCE_GROUP_LOCATION" \
+        location="$AZURE_LOCATION" \
         resource_group_name="$RESOURCE_GROUP_NAME" \
         azure_data_factory_name="$DATAFACTORY_NAME" \
         azure_subscription_id="$AZURE_SUBSCRIPTION_ID" \

diff --git a/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/init_environment.sh b/single_tech_samples/datafactory/adf_cicd_auto_publish/scripts/init_environment.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# check required variables are specified.
+
+if [ -z "$AZDO_PROJECT" ]
+then 
+    echo "Please specify a target Azure DevOps project where Azure Pipelines and Variable groups will be deploy using AZDO_PROJECT environment variable"
+    exit 1
+fi
+
+if [ -z "$AZDO_ORGANIZATION_URL" ]
+then 
+    echo "Please specify a Azure DevOps organization url using the AZDO_ORGANIZATION_URL environment variable in this form: https://dev.azure.com/<organization>/"
+    exit 1
+fi
+
+if [ -z "$AZURE_DEVOPS_EXT_PAT" ]
+then 
+    echo "Please specify a Azure DevOps PAT token using the AZURE_DEVOPS_EXT_PAT environment variable."
+    exit 1
+fi
+
+# initialise optional variables.
+
+ENV_NAME=${ENV_NAME:-}
+if [ -z "$ENV_NAME" ]
+then 
+    export ENV_NAME="dev"
+    echo "No environment name [ENV_NAME] specified, defaulting to $ENV_NAME"
+fi
+
+DEPLOYMENT_ID=${DEPLOYMENT_ID:-}
+if [ -z "$DEPLOYMENT_ID" ]
+then 
+    export DEPLOYMENT_ID="$(random_str 5)"
+    echo "No deployment id [DEPLOYMENT_ID] specified, defaulting to $DEPLOYMENT_ID"
+fi
+
+AZURE_LOCATION=${AZURE_LOCATION:-}
+if [ -z "$AZURE_LOCATION" ]
+then    
+    export AZURE_LOCATION="westus"
+    echo "No resource group location [AZURE_LOCATION] specified, defaulting to $AZURE_LOCATION"
+fi
+
+AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID:-}
+if [ -z "$AZURE_SUBSCRIPTION_ID" ]
+then
+    export AZURE_SUBSCRIPTION_ID=$(az account show --output json | jq -r '.id')
+    echo "No Azure subscription id [AZURE_SUBSCRIPTION_ID] specified. Using default subscription id."
+fi