arc_k8s.azcli

########################################################
# Script to deploy different features of Arc-enabled
#    k8s clusters, for demo or training purposes.
#
# Jose Moreno
# March 2021
########################################################

# Variables
k8s_rg=aksengine
location=westeurope
arc_rg=k8sarc
arc_name=myaksengine
deployment_mode=aksengine   # Whether the cluster is to be deployed with aksengine, capi or kubeadm (only aksengine supported)

# Create RG for AKS engine cluster
echo "Creating resource group ${k8s_rg}..."
az group create -n $k8s_rg -l $location -o none

# Retrieve Service Principal form your AKV, required for AKS engine or CAPI
purpose=aksengine
keyvault_appid_secret_name=$purpose-sp-appid
keyvault_password_secret_name=$purpose-sp-secret
keyvault_name=erjositoKeyvault
keyvault_appid_secret_name=$purpose-sp-appid
keyvault_password_secret_name=$purpose-sp-secret
sp_app_id=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_appid_secret_name --query 'value' -o tsv 2>/dev/null)
sp_app_secret=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_password_secret_name --query 'value' -o tsv 2>/dev/null)

# If a secret was found, verify whether SP has expired, and if so, renew it
if [[ -n "$sp_app_id" ]] || [[ -n "$sp_app_secret" ]]
then
    sp_end_date=$(az ad app show --id $sp_app_id --query 'passwordCredentials[0].endDate' -o tsv)
    sp_end_date=$(date --date="$sp_end_date" +%s)
    now=$(date +%s)
    if [[ $sp_end_date < $now ]]
    then
        echo "SP expired, extending one year"
        new_password=$(az ad app credential reset --id $sp_app_id --years 1 --query password -o tsv)
        az keyvault secret set --vault-name $keyvault_name --name $keyvault_password_secret_name --value $new_password -o none
        sp_app_secret=$new_password
    else
        echo "SP not expired"
    fi
fi

# If they could not be retrieved, generate new ones
if [[ -z "$sp_app_id" ]] || [[ -z "$sp_app_secret" ]]
then
    echo "No SP for AKS-engine could be found in AKV $keyvault_name, generating new ones..."
    sp_name=$purpose
    sp_output=$(az ad sp create-for-rbac --name $sp_name --skip-assignment 2>/dev/null)
    sp_app_id=$(echo $sp_output | jq -r '.appId')
    sp_app_secret=$(echo $sp_output | jq -r '.password')
    # Store the created app ID and secret in an AKV
    az keyvault secret set --vault-name $keyvault_name -n $keyvault_appid_secret_name --value $sp_app_id -o none
    az keyvault secret set --vault-name $keyvault_name -n $keyvault_password_secret_name --value $sp_app_secret -o none
else
    echo "Service Principal $sp_app_id and secret successfully retrieved from AKV $keyvault_name"
fi

# Grant access to the SP to the new RG
scope=$(az group show -n $k8s_rg --query id -o tsv)
# assignee=$(az ad sp show --id $sp_app_id --query objectId -o tsv)
echo "Granting access for the SP ${sp_app_id} to the resource group ${k8s_rg}..."
az role assignment create --scope $scope --role Contributor --assignee $sp_app_id -o none

#############################
# Deploy aks-engine cluster #
#############################

if [[ "$deployment_mode" == "aksengine" ]]; then

  # File containing the description of the AKS engine cluster to create
  aksengine_cluster_file="/tmp/aksengine_cluster.json" 

  # Option1: Retrieve example JSON file describing a basic cluster
  url=https://raw.githubusercontent.com/Azure/aks-engine/master/examples/kubernetes.json
  wget $url -O $aksengine_cluster_file
  # You can modify the kubernetes.json file, for example with smaller VM sizes such as Standard_B2ms

  # Option 2: Create a cluster file from scratch:
  aksengine_vm_size=Standard_B2ms
  cat <<EOF > $aksengine_cluster_file
{
  "apiVersion": "vlabs",
  "properties": {
    "orchestratorProfile": {
      "orchestratorType": "Kubernetes",
      "kubernetesConfig": {
        "networkPlugin": "antrea",
        "networkPolicy": "antrea"
      }
    },
    "masterProfile": {
      "count": 1,
      "dnsPrefix": "",
      "vmSize": "$aksengine_vm_size"
    },
    "agentPoolProfiles": [
      {
        "name": "agentpool1",
        "count": 2,
        "vmSize": "$aksengine_vm_size"
      }
    ],
    "linuxProfile": {
      "adminUsername": "azureuser",
      "ssh": {
        "publicKeys": [
          {
            "keyData": ""
          }
        ]
      }
    },
    "servicePrincipalProfile": {
      "clientId": "",
      "secret": ""
    }
  }
}
EOF

  # Wait 30 seconds (the role assignment might need some time to propagate)
  sleep 30

  # Create AKS-engine cluster
  # You might need to install aks-engine from https://github.com/Azure/aks-engine/blob/master/docs/tutorials/quickstart.md
  subscription=$(az account show --query id -o tsv)
  domain=abc$RANDOM
  output_dir=/tmp/aksengine
  rm -rf $output_dir   # The output directory cannot exist
  aks-engine deploy --subscription-id $subscription \
      --dns-prefix $domain \
      --resource-group $k8s_rg \
      --location $location \
      --api-model $aksengine_cluster_file \
      --client-id $sp_app_id \
      --client-secret $sp_app_secret \
      --set servicePrincipalProfile.clientId=$sp_app_id \
      --set servicePrincipalProfile.secret="$sp_app_secret" \
      --output-directory $output_dir

  # There are different ways to access the cluster
  # Exporting the KUBECONFIG variable is required by the command "az k8s-configuration create"
  export KUBECONFIG="$output_dir/kubeconfig/kubeconfig.$location.json" 
  kubectl get node
  # The alias is a little dirty trick for lazy persons like me
  # alias ke="kubectl --kubeconfig ./_output/$domain/kubeconfig/kubeconfig.$location.json" 
fi


####################################
#   Deploy k8s cluster with CAPI   #
####################################
# NOT WORKING YET!!!


if [[ "$deployment_mode" == "capi" ]]; then
    # Install kind if required
    kind_exec=$(which kind 2>/dev/null)
    if [[ "$kind_exec" == "kind not found" ]]
    then
        echo "Downloading and installing kind..."
        curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
    else
        echo "kind executable found in ${kind_exec}"
    fi

    # Verify docker is available

    # Create kind cluster
    sudo kind create cluster
    kubectl cluster-info

    # Install clusterctl
    clusterctl_exec=$(which clusterctl 2>/dev/null)
    if [[ "$clusterctl_exec" == "clusterctl not found" ]]
    then
        echo "Downloading and installing clusterctl..."
        curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.12/clusterctl-linux-amd64 -o clusterctl
        chmod +x ./clusterctl
        sudo mv ./clusterctl /usr/local/bin/clusterctl
    else
        echo "kind executable found in ${clusterctl_exec}"
    fi
    clusterctl version

    # Variables
    export CAPI_PROVIDER="azure"
    export AZURE_ENVIRONMENT="AzurePublicCloud"
    export AZURE_LOCATION=$location
    KUBERNETES_VERSION="1.18.17" # For example: "1.18.17"
    CONTROL_PLANE_MACHINE_COUNT=1
    WORKER_MACHINE_COUNT=2
    CAPI_WORKLOAD_CLUSTER_NAME="capi" # Name of the CAPI workload cluster. Must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')
    subscription_id=$(az account show --query id -o tsv)
    tenant_id=$(az account show --query tenantId -o tsv)

    # Azure cloud settings - Do not change!
    export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$subscription_id" | base64 | tr -d '\n')"
    export AZURE_TENANT_ID_B64="$(echo -n "$tenant_id" | base64 | tr -d '\n')"
    export AZURE_CLIENT_ID_B64="$(echo -n "$sp_app_id" | base64 | tr -d '\n')"
    export AZURE_CLIENT_SECRET_B64="$(echo -n "$sp_app_secret" | base64 | tr -d '\n')"

    # Transforming the kind cluster to a Cluster API management cluster
    echo "Transforming the Kubernetes cluster to a management cluster with the Cluster API Azure Provider (CAPZ)..."
    clusterctl init --infrastructure azure
    echo "Making sure cluster is ready..."
    echo ""
    kubectl wait --for=condition=Available --timeout=60s --all deployments -A >/dev/null
    echo ""

    # Deploy CAPI Workload cluster
    echo "Deploying Kubernetes workload cluster"
    echo ""
    clusterctl config cluster $CAPI_WORKLOAD_CLUSTER_NAME \
      --kubernetes-version v$KUBERNETES_VERSION \
      --control-plane-machine-count=$CONTROL_PLANE_MACHINE_COUNT \
      --worker-machine-count=$WORKER_MACHINE_COUNT \
      --infrastructure=$CAPI_PROVIDER \
      --kubernetes-version=$KUBERNETES_VERSION \
      > "/tmp/${CAPI_WORKLOAD_CLUSTER_NAME}.yaml"

    curl https://raw.githubusercontent.com/Azure/Azure-Security-Center/master/Pricing%20%26%20Settings/Defender%20for%20Kubernetes/audit-policy.yaml -o /tmp/audit.yaml

    cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
  name: audit
type: Opaque
data:
  audit.yaml: $(cat "audit.yaml" | base64 -w0)
  username: $(echo -n "jumpstart" | base64 -w0)
EOF

    line=$(expr $(grep -n -B 1 "extraArgs" $CAPI_WORKLOAD_CLUSTER_NAME.yaml | grep "apiServer" | cut -f1 -d-) + 5)
    sed -i -e "$line"' i\          readOnly: true' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          name: audit-policy' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          mountPath: /etc/kubernetes/audit.yaml' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\        - hostPath: /etc/kubernetes/audit.yaml' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          name: kubeaudit' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          mountPath: /var/log/kube-apiserver' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\        - hostPath: /var/log/kube-apiserver' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    line=$(expr $(grep -n -B 1 "extraArgs" $CAPI_WORKLOAD_CLUSTER_NAME.yaml | grep "apiServer" | cut -f1 -d-) + 2)
    sed -i -e "$line"' i\          audit-policy-file: /etc/kubernetes/audit.yaml' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          audit-log-path: /var/log/kube-apiserver/audit.log' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          audit-log-maxsize: "100"' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          audit-log-maxbackup: "10"' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          audit-log-maxage: "30"' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    line=$(expr $(grep -n -A 3 files: $CAPI_WORKLOAD_CLUSTER_NAME.yaml | grep "control-plane" | cut -f1 -d-) + 5)
    sed -i -e "$line"' i\      permissions: "0644"' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\      path: /etc/kubernetes/audit.yaml' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\      owner: root:root' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          name: audit' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\          key: audit.yaml' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\        secret:' $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    sed -i -e "$line"' i\    - contentFrom:' $CAPI_WORKLOAD_CLUSTER_NAME.yaml


    kubectl apply -f $CAPI_WORKLOAD_CLUSTER_NAME.yaml
    echo ""

    until kubectl get cluster --all-namespaces | grep -q "Provisioned"; do echo "Waiting for Kubernetes control plane to be in Provisioned phase..." && sleep 20 ; done
    echo ""
    kubectl get cluster --all-namespaces
    echo ""

    until kubectl get kubeadmcontrolplane --all-namespaces | grep -q "true"; do echo "Waiting for control plane to initialize. This may take a few minutes..." && sleep 20 ; done
    echo ""
    kubectl get kubeadmcontrolplane --all-namespaces
    clusterctl get kubeconfig $CAPI_WORKLOAD_CLUSTER_NAME > $CAPI_WORKLOAD_CLUSTER_NAME.kubeconfig
    echo ""
    kubectl --kubeconfig=./$CAPI_WORKLOAD_CLUSTER_NAME.kubeconfig apply -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/master/templates/addons/calico.yaml
    echo ""

    CLUSTER_TOTAL_MACHINE_COUNT=`expr $CONTROL_PLANE_MACHINE_COUNT + $WORKER_MACHINE_COUNT`
    export CLUSTER_TOTAL_MACHINE_COUNT="$(echo $CLUSTER_TOTAL_MACHINE_COUNT)"
    until [[ $(kubectl --kubeconfig=./$CAPI_WORKLOAD_CLUSTER_NAME.kubeconfig get nodes | grep -c -w "Ready") == $CLUSTER_TOTAL_MACHINE_COUNT ]]; do echo "Waiting all nodes to be in Ready state. This may take a few minutes..." && sleep 30 ; done 2> /dev/null
    echo ""
    kubectl --kubeconfig=./$CAPI_WORKLOAD_CLUSTER_NAME.kubeconfig label node -l '!node-role.kubernetes.io/master' node-role.kubernetes.io/worker=worker
    echo ""
    kubectl --kubeconfig=./$CAPI_WORKLOAD_CLUSTER_NAME.kubeconfig get nodes
    echo ""

    echo "Onboarding the cluster as an Azure Arc enabled Kubernetes cluster"
    az login --service-principal --username $AZURE_CLIENT_ID --password $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID
    echo ""

    rm -rf ~/.azure/AzureArcCharts

    echo "Checking if you have up-to-date Azure Arc AZ CLI 'connectedk8s' extension..."
    az extension show --name "connectedk8s" &> extension_output
    if cat extension_output | grep -q "not installed"; then
    az extension add --name "connectedk8s"
    rm extension_output
    else
    az extension update --name "connectedk8s"
    rm extension_output
    fi
    echo ""

    echo "Checking if you have up-to-date Azure Arc AZ CLI 'k8s-configuration' extension..."
    az extension show --name "k8s-configuration" &> extension_output
    if cat extension_output | grep -q "not installed"; then
    az extension add --name "k8s-configuration"
    rm extension_output
    else
    az extension update --name "k8s-configuration"
    rm extension_output
    fi
    echo ""

    az connectedk8s connect --name $CAPI_WORKLOAD_CLUSTER_NAME --resource-group $CAPI_WORKLOAD_CLUSTER_NAME --location $AZURE_LOCATION --kube-config $CAPI_WORKLOAD_CLUSTER_NAME.kubeconfig
fi

############################
# Enabling cluster for Arc #
############################

# Docs: https://docs.microsoft.com/en-us/azure/azure-arc/kubernetes/quickstart-connect-cluster

# Az CLI extension connectedk8s
extension_name=connectedk8s
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
if [[ -z "$extension_version" ]]
then
    echo "Azure CLI extension $extension_name not found, installing now..."
    az extension add -n $extension_name
else
    echo "Azure CLI extension $extension_name found with version $extension_version, trying to upgrade..."
    az extension update -n $extension_name
fi
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
echo "Azure CLI extension $extension_name installed with version $extension_version"

# Registering providers
for provider in "Kubernetes" "KubernetesConfiguration" "ExtendedLocation" "Web"
do
    registration_state=$(az provider show -n "Microsoft.${provider}" --query registrationState -o tsv)
    if [[ "$registration_state" == "Registered" ]]
    then
        echo "Resource Provider Microsoft.${provider} is successfully registered with status ${registration_state}"
    else
        echo "It seems that provider Microsoft.${provider} is not registered, registering now..."
        az provider register --namespace "Microsoft.${provider}"
        wait_time=30
        registration_state=$(az provider show -n "Microsoft.${provider}" --query registrationState -o tsv)
        while [[ "$registration_state" != "Registered" ]]
        do
            echo "Registration state for RP Microsoft.${provider} is still $registration_state..."
            sleep $wait_time
            registration_state=$(az provider show -n "Microsoft.${provider}" --query registrationState -o tsv)
        done
        echo "Registration state for RP Microsoft.${provider} is ${registration_state}"
    fi
done
echo "All resource providers successfully registered"

# Create the ARC resource
az group create -n $arc_rg -l $location
az connectedk8s connect --name $arc_name -g $arc_rg -l $location

# Diagnostics
az connectedk8s list -g $arc_rg -o table
kubectl -n azure-arc get deployments,pods

####################################
# Gitops in an arc-enabled cluster #
####################################

# Docs: https://docs.microsoft.com/azure/azure-arc/kubernetes/tutorial-use-gitops-connected-cluster

# Az CLI extension k8s-configuration
extension_name=k8s-configuration
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
if [[ -z "$extension_version" ]]
then
    echo "Azure CLI extension $extension_name not found, installing now..."
    az extension add -n $extension_name
else
    echo "Azure CLI extension $extension_name found with version $extension_version, trying to upgrade..."
    az extension update -n $extension_name
fi
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
echo "Azure CLI extension $extension_name installed with version $extension_version"

# Create a cluster-level operator
repo_url="https://github.com/erjosito/arc-k8s-test/" # Feel free to use your own repo here
cfg_name=gitops-config
namespace=$cfg_name
az k8s-configuration create \
    --name $cfg_name \
    --cluster-name $arc_name --resource-group $arc_rg \
    --operator-instance-name $cfg_name \
    --operator-namespace $namespace \
    --repository-url $repo_url \
    --scope cluster \
    --cluster-type connectedClusters
# cluster-type can be either "connectedClusters" (for ARC clusters) or "managedClusters" (for AKS)

# Diagnostics (you need to wait some seconds for the namespace and resources to be created)
az k8s-configuration show -n $cfg_name -c $arc_name -g $arc_rg --cluster-type connectedClusters
kubectl -n $namespace get deploy -o wide

# Optional: update operator to enable helm or change the repo URL
# az k8s-configuration update -n $cfg_name -c $arc_name -g $arc_rg --cluster-type connectedClusters --enable-helm-operator
# az k8s-configuration update -n $cfg_name -c $arc_name -g $arc_rg --cluster-type connectedClusters -u $repo_url

# Optional: delete configuration
# az k8s-configuration delete -n $cfg_name -c $arc_name -g $arc_rg --cluster-type connectedClusters

##########################################
# Azure Monitor for Arc-enabled clusters #
##########################################

# Docs: https://docs.microsoft.com/en-us/azure/azure-monitor/containers/container-insights-enable-arc-enabled-clusters?toc=/azure/azure-arc/kubernetes/toc.json

# Az CLI extension log-analytics
extension_name=log-analytics
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
if [[ -z "$extension_version" ]]
then
    echo "Azure CLI extension $extension_name not found, installing now..."
    az extension add -n $extension_name
else
    echo "Azure CLI extension $extension_name found with version $extension_version, trying to upgrade..."
    az extension update -n $extension_name
fi
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
echo "Azure CLI extension $extension_name installed with version $extension_version"

# Create Log Analytics workspace
logws_name=$(az monitor log-analytics workspace list -g $arc_rg --query '[].name' -o tsv 2>/dev/null)  # Retrieve the WS name if it already existed
if [[ -z "$logws_name" ]]
then
    logws_name=log$RANDOM
    az monitor log-analytics workspace create -n $logws_name -g $arc_rg
fi
logws_id=$(az resource list -g $arc_rg -n $logws_name --query '[].id' -o tsv)
logws_customerid=$(az monitor log-analytics workspace show -n $logws_name -g $arc_rg --query customerId -o tsv)

# Enabling monitoring with enable-monitoring.sh (deprecated)
# enable_script=/tmp/enable-monitoring.sh
# curl -o $enable_script -L https://aka.ms/enable-monitoring-bash-script
# arc_id=$(az connectedk8s show -n $arc_name -g $arc_rg -o tsv --query id) && echo $arc_id
# bash $enable_script --resource-id $arc_id --workspace-id $logws_id

# Uninstall
# disable_script=/tmp/disable-monitoring.sh
# curl -o $disable_script -L https://aka.ms/disable-monitoring-bash-script
# bash $disable_script --resource-id $arc_id --workspace-id $logws_id

# See https://github.com/Azure/azure-arc-kubernetes-preview/blob/master/docs/k8s-extensions.md

# Enable extensions
# az k8s-extension create --cluster-name $arc_name --resource-group $arc_rg --cluster-type connectedClusters \
#     --extension-type Microsoft.AzureMonitor.Containers --name azuremonitor-containers --configuration-settings logAnalyticsWorkspaceResourceID=$logws_id

# Enabling monitoring using k8s extensions
az k8s-extension create --name azuremonitor-containers --cluster-name $arc_name --resource-group $arc_rg --cluster-type connectedClusters \
    --extension-type Microsoft.AzureMonitor.Containers --configuration-settings "logAnalyticsWorkspaceResourceID=${logws_id}"

# Diagnostics
az k8s-extension list -c $arc_name -g $arc_rg --cluster-type ConnectedClusters -o table

# Getting logs (sample query)
query='ContainerLog
| where TimeGenerated > ago(5m)
| project TimeGenerated, LogEntry, ContainerID
| take 20'
az monitor log-analytics query -w $logws_customerid --analytics-query $query -o tsv

##################
# Azure Defender #
##################

# Docs: https://docs.microsoft.com/en-us/azure/security-center/defender-for-kubernetes-azure-arc

# Deploy defender extension to the same WS
az k8s-extension create --name microsoft.azuredefender.kubernetes --cluster-type connectedClusters \
    --cluster-name $arc_name --resource-group $arc_rg --extension-type microsoft.azuredefender.kubernetes \
    --configuration-settings "logAnalyticsWorkspaceResourceID=${logws_id}"

# Diagnostics
az k8s-extension list -c $arc_name -g $arc_rg --cluster-type ConnectedClusters -o table

# Simulate alerts
kubectl get pods --namespace=asc-alerttest-662jfi039n

# Check alert (can take some minutes to appear)
az security alert list -g $arc_rg

#########################################
# Azure Policy for Arc-enabled clusters #
#########################################

# Docs: https://docs.microsoft.com/azure/governance/policy/concepts/policy-for-kubernetes

# Registering providers
for provider in "PolicyInsights"
do
    registration_state=$(az provider show -n "Microsoft.${provider}" --query registrationState -o tsv)
    if [[ "$registration_state" == "Registered" ]]
    then
        echo "Resource Provider Microsoft.${provider} is successfully registered with status ${registration_state}"
    else
        echo "It seems that provider Microsoft.${provider} is not registered, registering now..."
        az provider register --namespace "Microsoft.${provider}"
        wait_time=30
        registration_state=$(az provider show -n "Microsoft.${provider}" --query registrationState -o tsv)
        while [[ "$registration_state" != "Registered" ]]
        do
            echo "Registration state for RP Microsoft.${provider} is still $registration_state..."
            sleep $wait_time
            registration_state=$(az provider show -n "Microsoft.${provider}" --query registrationState -o tsv)
        done
        echo "Registration state for RP Microsoft.${provider} is ${registration_state}"
    fi
done
echo "All resource providers successfully registered"

# We will use the same SP as we did for AKS Engine, in prod you would use something else
echo "Getting Arc-enabled cluster ID..."
arc_id=$(az connectedk8s show -n $arc_name -g $arc_rg -o tsv --query id) && echo $arc_id
scope=$arc_id
role="Policy Insights Data Writer (Preview)"
assignee=$(az ad sp show --id $sp_app_id --query objectId -o tsv)
echo "Assigning role $role for OID ${assignee} on scope ${scope}..."
az role assignment create --scope $scope --role $role --assignee $assignee
echo "Getting IDs for tenant and subscription..."
tenant_id=$(az account show --query tenantId -o tsv) && echo $tenant_id
subscription_id=$(az account show --query id -o tsv) && echo $subscription_id

# Deploy Helm chart
helm repo add azure-policy https://raw.githubusercontent.com/Azure/azure-policy/master/extensions/policy-addon-kubernetes/helm-charts
helm repo update
helm install azure-policy-addon azure-policy/azure-policy-addon-arc-clusters \
    --set azurepolicy.env.resourceid=$arc_id \
    --set azurepolicy.env.clientid=$sp_app_id \
    --set azurepolicy.env.clientsecret=$sp_app_secret \
    --set azurepolicy.env.tenantid=$tenant_id

# Diagnostics
kubectl get pods -n kube-system
kubectl get pods -n gatekeeper-system

# Sample policy 1: no public ALB
policy_name=$(az policy definition list --subscription $subscription_id --query "[?contains(displayName,'Kubernetes clusters should use internal load balancers')].name" -o tsv)
if [[ -n "$policy_name" ]]
then
    echo "Successfully retrieved policy name to enforce internal load balancers: ${policy_name}. Creating policy assignment..."
    az policy assignment create -n noPublicLBresource --policy $policy_name --scope $arc_id
fi

# Sample policy 2: no privileged containers
policy_name=$(az policy definition list --subscription $subscription_id --query "[?contains(displayName,'Kubernetes cluster should not allow privileged containers')].name" -o tsv)
if [[ -n "$policy_name" ]]
then
    echo "Successfully retrieved policy name to disallow privileged containers: ${policy_name}. Creating policy assignment..."
    az policy assignment create -n noPrivilegedContainers --policy $policy_name --scope $arc_id
    # az policy assignment create -n noPrivilegedContainers --policy $policy_name --scope $arc_id --params '{ "excludedNamespaces": {"value": [ "kube-system", "gatekeeper-system", "azure-arc", "azuredefender" ] }}'
fi
yaml_file=/tmp/privileged.yml
cat <<EOF > $yaml_file
apiVersion: v1
kind: Pod
metadata:
  name: nginx-privileged
spec:
  containers:
    - name: nginx-privileged
      image: mcr.microsoft.com/oss/nginx/nginx:1.15.5-alpine
      securityContext:
        privileged: true
EOF

# Test: You should receive an error: Error from server ([denied by azurepolicy-container-no-privilege-73b124012cd393825d53]
# It could take some seconds until the policy is effective
kubectl apply -f $yaml_file

# Diagnostics
az policy assignment list --scope $arc_id -o table

# Verify policy state (WORK IN PROGRESS WITH THE CLI -> PORTAL IS PREFERRED)
assignment1_id=$(az policy assignment show -n noPublicLBresource --scope $arc_id --query id -o tsv) && echo $assignment1_id
assignment2_id=$(az policy assignment show -n noPrivilegedContainers --scope $arc_id --query id -o tsv) && echo $assignment2_id
# az policy state list --resource $arc_id -o table
# az policy state list --resource $arc_id -o table --top 5 --order-by "timestamp desc, policyAssignmentName asc" --select "timestamp, policyDefinitionId"
# az policy state list --resource $arc_id -o table --top 5 --apply "groupby((policyAssignmentId, policySetDefinitionId,policyDefinitionReferenceId, policyDefinitionId), aggregate($count as numStates))"
az policy state summarize --resource $arc_id -o table

# Diagnostics
kubectl get pods -n gatekeeper-system
kubectl get constrainttemplate
az policy assignment list --scope $arc_id -o table

# Cleanup
az policy assignment delete -n noPrivilegedContainers --scope $arc_id
az policy assignment delete -n noPublicLBresource --scope $arc_id

###################
# Cluster Connect #
###################

# You need to be in default namespace!

az connectedk8s enable-features --features cluster-connect -n $arc_name -g $arc_rg

kubectl create serviceaccount admin-user
kubectl create clusterrolebinding admin-user-binding --clusterrole cluster-admin --serviceaccount default:admin-user
secret_name=$(kubectl get serviceaccount admin-user -o jsonpath='{$.secrets[0].name}')
k8s_token=$(kubectl get secret ${secret_name} -o jsonpath='{$.data.token}' | base64 -d | sed $'s/$/\\\n/g')

az connectedk8s proxy -n $arc_name -g $arc_rg --token $k8s_token
kubectl get pods


################
# App Services #
################

# CLI extension
az extension remove --name appservice-kube
az extension add --yes --source "https://aka.ms/appsvc/appservice_kube-latest-py2.py3-none-any.whl"

# Create LA workspace
logws_name="log$RANDOM" # Name of the Log Analytics workspace
az monitor log-analytics workspace create -g $k8s_rg --workspace-name $logws_name
logws_customer_id=$(az monitor log-analytics workspace show -g $k8s_rg -n $logws_name --query customerId -o tsv)
logws_customer_id_enc=$(printf %s $logws_customer_id | base64 -w0) # Needed for the next step
logws_key=$(az monitor log-analytics workspace get-shared-keys -g $k8s_rg -n $logws_name --query primarySharedKey -o tsv)
logws_key_enc=$(printf %s $logws_key | base64 -w0) # Needed for the next step

# Create public IP
appsvc_pip_name=appsvc-pip
az network public-ip create -g $k8s_rg -n $appsvc_pip_name --sku Standard
appsvc_pip=$(az network public-ip show -g $k8s_rg --name $appsvc_pip_name --output tsv --query ipAddress) && echo $appsvc_pip

# Deploy arc extension
appsvc_extension_name="appsvc" # Name of the App Service extension
appsvc_ns="appsvc"             # Namespace in your cluster to install the extension and provision resources
kube_env_name="appsvc$RANDOM"  # Name of the App Service Kubernetes environment resource (must be unique!)
az k8s-extension create \
    --resource-group $arc_rg \
    --name $appsvc_extension_name \
    --cluster-type connectedClusters \
    --cluster-name $arc_name \
    --extension-type 'Microsoft.Web.Appservice' \
    --release-train stable \
    --auto-upgrade-minor-version true \
    --scope cluster \
    --release-namespace $appsvc_ns \
    --configuration-settings "Microsoft.CustomLocation.ServiceAccount=default" \
    --configuration-settings "appsNamespace=${namespace}" \
    --configuration-settings "clusterName=${kube_env_name}" \
    --configuration-settings "loadBalancerIp=${appsvc_pip}" \
    --configuration-settings "keda.enabled=true" \
    --configuration-settings "buildService.storageClassName=default" \
    --configuration-settings "buildService.storageAccessMode=ReadWriteOnce" \
    --configuration-settings "customConfigMap=${namespace}/kube-environment-config" \
    --configuration-settings "envoy.annotations.service.beta.kubernetes.io/azure-load-balancer-resource-group=${arc_rg}" \
    --configuration-settings "logProcessor.appLogs.destination=log-analytics" \
    --configuration-protected-settings "logProcessor.appLogs.logAnalyticsConfig.customerId=${logws_customer_id_enc}" \
    --configuration-protected-settings "logProcessor.appLogs.logAnalyticsConfig.sharedKey=${logws_key_enc}"

# If you need to update it, you can delete and recreate it
# az k8s-extension delete -n $appsvc_extension_name -g $arc_rg --cluster-name $arc_name --cluster-type connectedClusters

# Wait for extension to be provisioned
appsvc_extension_id=$(az k8s-extension show -n $appsvc_extension_name --cluster-type connectedClusters --cluster-name $arc_name -g $arc_rg --query id -o tsv) && echo $appsvc_extension_id
az resource wait --ids $appsvc_extension_id --custom "properties.installState!='Pending'" --api-version "2020-07-01-preview"

# Az CLI extension customlocation
extension_name=customlocation
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
if [[ -z "$extension_version" ]]
then
    echo "Azure CLI extension $extension_name not found, installing now..."
    az extension add -n $extension_name
else
    echo "Azure CLI extension $extension_name found with version $extension_version, trying to upgrade..."
    az extension update -n $extension_name
fi
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
echo "Azure CLI extension $extension_name installed with version $extension_version"

# Custom location (custom location is not an extension, but a resource on its own)
# It refers to the namespace where the appsvc is installed
customloc_name=mycluster
arc_cluster_id=$(az connectedk8s show -g $arc_rg -n $arc_name --query id --output tsv) && echo $arc_cluster_id
az customlocation create -g $arc_rg -n $customloc_name --host-resource-id $arc_cluster_id --namespace $appsvc_ns --cluster-extension-ids $appsvc_extension_id
az customlocation show -g $arc_rg -n $customloc_name
customloc_id=$(az customlocation show -g $arc_rg -n $customloc_name --query id -o tsv) && echo $customloc_id

# App Svc environment
az appservice kube create -n $kube_env_name -g $arc_rg  -l $location --custom-location $customloc_id    # Static IP not required for Arc clusters
#az appservice kube create -n $kube_env_name -g $arc_rg --custom-location $customloc_id --static-ip $appsvc_pip
az appservice kube show -n $kube_env_name -g $arc_rg
# If no IP is specified, it will look for a svc in the namespace specified in the customlocation. If no svc is found, it will give this error:
# "Service with specified selector k8se.microsoft.com/component=envoy not found in namespace customlocation"

# App Svc Plan
appsvc_plan_name=mysvcplan
az appservice plan create -g $arc_rg -n $appsvc_plan_name --custom-location $customloc_id --per-site-scaling --is-linux --sku K1

# Web App (src code)
webapp_name=mywebapp$RANDOM
az webapp create --plan $appsvc_plan_name -g $arc_rg -n $webapp_name --custom-location $customloc_id --runtime 'NODE|12-lts'
current_dir=$(pwd)
cd /tmp
git clone https://github.com/Azure-Samples/nodejs-docs-hello-world
cd nodejs-docs-hello-world
zip -r package.zip .
az webapp deployment source config-zip -g $arc_rg --name $webapp_name --src package.zip
cd $current_dir

# Web App (container)
webapp2_name=sqlapi$RANDOM
az webapp create --plan $appsvc_plan_name -g $arc_rg -n $webapp2_name --custom-location $customloc_id --deployment-container-image-name erjosito/sqlapi:1.0

# Logic App: Install extension and create logic app
logicapp_name=arclogicapp$RANDOM
subscription_id=$(az account show --query id -o tsv)
storage_account_name=$(az storage account list -g $arc_rg --query '[].name' -o tsv)
if [[ -n "$storage_account_name" ]]
then
    echo "Storage account $storage_account_name found in resource gorup $arc_rg"
else
    echo "No storage account found in resource group $arc_rg, creating one..."
    storage_account_name=logicapp$RANDOM
    az storage account create -n $storage_account_name -g $arc_rg --sku Standard_LRS -l $location --kind StorageV2
fi
az extension remove --name logicapp
az extension add --yes --source "https://aka.ms/logicapp-latest-py2.py3-none-any.whl"
az logicapp create -n $logicapp_name -g $arc_rg --subscription $subscription_id --storage-account $storage_account_name \
  --custom-location $customloc_id --plan $appsvc_plan_name --os-type linux
# ERROR: "Unable to find Kube Environment associated to the Custom Location" (!!!)
# Created issue https://github.com/Azure/logicapps/issues/396

# Delete the logicapp, if you need to recreate it
# az logicapp delete -n $logicapp_name -g $arc_rg

# Troubleshooting
az connectedk8s list -g $arc_rg -o table  # There should be an ARC cluster
kubectl -n azure-arc get deployments,pods # The Arc pods should be up and running
az k8s-extension list --cluster-type connectedClusters --cluster-name $arc_name -g $arc_rg -o table  # The appsvc extension should be successful
kubectl -n $appsvc_ns get deploy,pod      # The AppSvc pods should be up and running
az customlocation list -g $arc_rg -o table
az appservice kube list -g $arc_rg -o table
k get sc  # There should be a storage class for app svc build called something like "appsvc-k8se-build-service"
k describe sc appsvc-k8se-build-service
k get svc -A -l k8se.microsoft.com/component=envoy  # Service that hte appsvc kube environment will look for, to get the public IP
az appservice plan list -g $arc_rg -o table
az storage account list -g $arc_rg -o table
az webapp list -g $arc_rg -o table
az logicapp show -n $logicapp_name -g $arc_rg -o table
az rest --method GET --uri "https://management.azure.com/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/providers/Microsoft.Web/kubeEnvironments?api-version=2020-12-01"
#########
#  AAD  #
#########

# https://docs.microsoft.com/azure/azure-arc/kubernetes/azure-rbac
# Admin privilege required at the AAD tenant level

###################
# Custom location #
###################

# You need to be in default namespace!

# Az CLI extension customlocation
extension_name=customlocation
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
if [[ -z "$extension_version" ]]
then
    echo "Azure CLI extension $extension_name not found, installing now..."
    az extension add -n $extension_name
else
    echo "Azure CLI extension $extension_name found with version $extension_version, trying to upgrade..."
    az extension update -n $extension_name
fi
extension_version=$(az extension show -n $extension_name --query version -o tsv 2>/dev/null)
echo "Azure CLI extension $extension_name installed with version $extension_version"

# Custom location
az connectedk8s enable-features -n $arc_name -g $arc_rg --features cluster-connect custom-locations
extension_name=customLocation
location_name=mycluster
location_ns=customlocation
# az k8s-extension create --name $extension_name --extension-type microsoft.arcdataservices \
#     --cluster-type connectedClusters -c $arc_name -g $arc_rg --scope cluster --release-namespace arc \
#     --config Microsoft.CustomLocation.ServiceAccount=sa-bootstrapper
# extension_id=$(az k8s-extension show --name $extension_name --cluster-type connectedClusters -c $arc_name -g $arc_rg  --query id -o tsv) && echo $extension_id
# az customlocation create -n $extension_name -g $arc_rg --namespace $location_ns --host-resource-id $arc_id --cluster-extension-ids $extension_id
arc_cluster_id=$(az connectedk8s show -g $arc_rg -n $arc_name --query id --output tsv) && echo $arc_cluster_id
az customlocation create -n $location_name  -g $arc_rg --namespace $location_ns --host-resource-id $arc_cluster_id --cluster-extension-ids $appsvc_extension_id

# Diagnostics
az customlocation list -o table
az customlocation list-enabled-resource-types -n mycluster -g $arc_rg -o table

###########
# Cleanup #
###########

az group delete -y --no-wait -n $k8s_rg
az group delete -y --no-wait -n $arc_rg