diff --git a/.github/workflows/named_entity_recognition_ci_dev_workflow.yml b/.github/workflows/named_entity_recognition_ci_dev_workflow.yml index dfada3a59..c496572a7 100644 --- a/.github/workflows/named_entity_recognition_ci_dev_workflow.yml +++ b/.github/workflows/named_entity_recognition_ci_dev_workflow.yml @@ -39,4 +39,4 @@ jobs: secrets: azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }} - registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }} \ No newline at end of file + registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }} diff --git a/.jenkins/jobs/aml_real_deployment b/.jenkins/jobs/aml_real_deployment new file mode 100644 index 000000000..78c12fa82 --- /dev/null +++ b/.jenkins/jobs/aml_real_deployment @@ -0,0 +1,100 @@ +@Library('shared-library') _ +pipeline { + agent any + + parameters { + string(name: 'env_name', defaultValue: "dev", description: 'env stage e.g. dev, test, prod') + string(name: 'flow_type', defaultValue: 'named_entity_recognition', description: 'flow type to be registered and deployed') + string(name: 'model_version', description: 'flow version in registry to be deployed') + string(name: 'run_id', description: 'run id of the flow to be deployed') + } + + // Install requirements and provision AML Managed Online Endpoint + stages { + stage('Provision AML Online Endpoint') { + steps { + installRequirements('execute_job_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.provision_endpoint \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --build_id $run_id \ + --output_file "endpoint_principal.txt" \ + --env_name $env_name \ + --flow_to_execute $flow_type + """ + } + } + } + } + + // Assign Azure resource permissions to endpoint principal + stage('Assign Azure resource permissions') { + steps { + azureLogin() + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + sh ''' + endpoint_principal=$(cat endpoint_principal.txt) + echo $endpoint_principal + file_path="./$flow_type/llmops_config.json" + echo $file_path + env_type="$env_name" + + selected_object=\$(jq ".envs[] | select(.ENV_NAME == \"$env_type\")" "\$file_path") + echo $selected_object + + key_vault_name=$(echo "$selected_object" | jq -r ".KEYVAULT_NAME") + resource_group_name=$(echo "$selected_object" | jq -r ".RESOURCE_GROUP_NAME") + workspace_name=$(echo "$selected_object" | jq -r ".WORKSPACE_NAME") + + az role assignment create --assignee $endpoint_principal --role "AzureML Data Scientist" --scope "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourcegroups/$resource_group_name/providers/Microsoft.MachineLearningServices/workspaces/$workspace_name" + + auth_type=$(az keyvault show -n $key_vault_name -g $resource_group_name --query "properties.enableRbacAuthorization") + if [[ -z "$auth_type" ]]; then + echo "assigning RBAC permission" + az role assignment create --assignee $endpoint_principal --role "Key Vault Reader" --scope "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourcegroups/$resource_group_name/providers/Microsoft.KeyVault/vaults/$key_vault_name" + else + echo "assigning policy permission" + az keyvault set-policy --name $key_vault_name --resource-group $resource_group_name --object-id $endpoint_principal --secret-permissions get list + fi + ''' + } + } + } + + // Provision AML Online Deployment for given model version and run id + stage('Provision AML Online Deployment') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.provision_deployment \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --model_version $model_version \ + --build_id $run_id \ + --env_name $env_name \ + --flow_to_execute $flow_type + """ + } + } + } + } + + // Test Online AML Deployment + stage('Test AML Deployment') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.test_model_on_aml \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --env_name $env_name \ + --flow_to_execute $flow_type + """ + } + } + } + } + } +} \ No newline at end of file diff --git a/.jenkins/jobs/build_validation b/.jenkins/jobs/build_validation new file mode 100644 index 000000000..1699e4073 --- /dev/null +++ b/.jenkins/jobs/build_validation @@ -0,0 +1,62 @@ +@Library('shared-library') _ +pipeline { + agent any + + options { + // This is required if you want to clean before build + skipDefaultCheckout(true) + } + + parameters { + string(name: 'flow_type', defaultValue: 'named_entity_recognition', description: 'The flow use-case to validate') + } + + stages { + stage('Checkout') { + steps { + cleanWs() + checkout scm + } + } + + // Load dependencies, login to Azure and load subscription details + stage('Load Dependencies') { + steps { + installRequirements('build_validation_requirements') + sh 'az version' + } + } + + stage('Azure Login') { + steps { + azureLogin() + } + } + + stage('Load Azure Subscription Details') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + sh ''' + export subscriptionId=$(az account show --query id -o tsv) + echo "SUBSCRIPTION_ID=$AZURE_SUBSCRIPTION_ID" + ''' + } + } + } + + // Execute unit tests and publish results + stage('Execute Unit Tests') { + steps { + withPythonEnv('/usr/bin/python3.9') { + sh "pytest ${params.flow_type}/tests --junitxml=junit/test-results.xml --cov=. --cov-report=xml" + } + } + } + + stage('Publish Unit Test Results') { + steps { + archiveArtifacts artifacts: '**/test-*.xml', fingerprint: true + } + } + } +} \ No newline at end of file diff --git a/.jenkins/jobs/kubernetes_deployment b/.jenkins/jobs/kubernetes_deployment new file mode 100644 index 000000000..bcc8cce5c --- /dev/null +++ b/.jenkins/jobs/kubernetes_deployment @@ -0,0 +1,99 @@ +@Library('shared-library') _ +pipeline { + agent any + + parameters { + string(name: 'env_name', defaultValue: "dev", description: 'env stage e.g. dev, test, prod') + string(name: 'flow_type', defaultValue: 'named_entity_recognition', description: 'flow type to be registered and deployed') + string(name: 'model_version', description: 'flow version in registry to be deployed') + string(name: 'run_id', description: 'run id of the flow to be deployed') + } + + // Install requirements and provision Kubernetes Online Endpoint + stages { + stage('Provision Kubernetes Online Endpoint') { + steps { + installRequirements('execute_job_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.kubernetes_endpoint \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --build_id $run_id \ + --output_file "endpoint_principal.txt" \ + --env_name $env_name \ + --flow_to_execute $flow_type + """ + } + } + } + } + + // Assign Azure resource permissions to endpoint principal + stage('Assign Azure resource permissions') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + sh ''' + endpoint_principal=$(cat endpoint_principal.txt) + echo $endpoint_principal + file_path="./$flow_type/llmops_config.json" + echo $file_path + env_type="$env_name" + + selected_object=\$(jq ".envs[] | select(.ENV_NAME == \"$env_type\")" "\$file_path") + echo $selected_object + + key_vault_name=$(echo "$selected_object" | jq -r ".KEYVAULT_NAME") + resource_group_name=$(echo "$selected_object" | jq -r ".RESOURCE_GROUP_NAME") + workspace_name=$(echo "$selected_object" | jq -r ".WORKSPACE_NAME") + + az role assignment create --assignee $endpoint_principal --role "AzureML Data Scientist" --scope "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourcegroups/$resource_group_name/providers/Microsoft.MachineLearningServices/workspaces/$workspace_name" + + auth_type=$(az keyvault show -n $key_vault_name -g $resource_group_name --query "properties.enableRbacAuthorization") + if [[ -z "$auth_type" ]]; then + echo "assigning RBAC permission" + az role assignment create --assignee $endpoint_principal --role "Key Vault Reader" --scope "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourcegroups/$resource_group_name/providers/Microsoft.KeyVault/vaults/$key_vault_name" + else + echo "assigning policy permission" + az keyvault set-policy --name $key_vault_name --resource-group $resource_group_name --object-id $endpoint_principal --secret-permissions get list + fi + ''' + } + } + } + + // Provision Kubernetes Online Deployment + stage('Provision Kubernetes Online Deployment') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.kubernetes_deployment \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --model_version $model_version \ + --build_id $run_id \ + --env_name $env_name \ + --flow_to_execute $flow_type + """ + } + } + } + } + + // Test Kubernetes Deployment + stage('Test Kubernetes Deployment') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.test_model_on_kubernetes \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --env_name $env_name \ + --flow_to_execute $flow_type + """ + } + } + } + } + } +} \ No newline at end of file diff --git a/.jenkins/jobs/prepare_docker_image b/.jenkins/jobs/prepare_docker_image new file mode 100644 index 000000000..313bb1def --- /dev/null +++ b/.jenkins/jobs/prepare_docker_image @@ -0,0 +1,63 @@ +@Library('shared-library') _ +pipeline { + agent any + parameters { + string(name: 'flow_to_execute', defaultValue: 'named_entity_recognition', description: 'The flow use-case to execute') + string(name: 'deploy_environment', defaultValue: 'dev', description: 'Execution Environment') + } + + environment { + dev_connections = credentials('COMMON_DEV_CONNECTIONS') + registry_details = credentials('DOCKER_IMAGE_REGISTRY') + } + + // Install requirements for job execution and build validation and create local promptflow connections + stages { + stage('Create local promptflow connections') { + steps { + installRequirements('execute_job_requirements') + installRequirements('build_validation_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh ''' + python -m llmops.common.prompt_local_connections \\ + --flow_to_execute ${flow_to_execute} \\ + --env_name ${deploy_environment} \\ + --connection_details "$dev_connections" + ''' + } + } + } + } + + // Create Docker image and push to Azure Container Registry + stage('Create Docker Image') { + steps { + installRequirements('build_validation_requirements') + withCredentials([string(credentialsId: 'DOCKER_IMAGE_REGISTRY', variable: 'registry_details'), + string(credentialsId: 'COMMON_DEV_CONNECTIONS', variable: 'connection_details')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + # Your Docker image creation command here + echo "build no script:" + echo ${BUILD_NUMBER} + #!/bin/bash + ./llmops/common/scripts/gen_docker_image.sh --flow_to_execute $flow_to_execute --deploy_environment $deploy_environment --build_id ${BUILD_NUMBER} --REGISTRY_DETAILS '${registry_details}' --CONNECTION_DETAILS '${connection_details}' + """ + } + } + } + } + + // Deploy Docker image to Azure Webapp + stage('Create Webapp') { + steps { + withCredentials([string(credentialsId: 'COMMON_DEV_CONNECTIONS', variable: 'connection_details')]) { + sh """ + /bin/bash ./llmops/common/scripts/az_webapp_deploy.sh --flow_to_execute $flow_to_execute --deploy_environment $deploy_environment --build_id ${BUILD_NUMBER} --CONNECTION_DETAILS '${connection_details}' + """ + } + } + } + } +} diff --git a/.jenkins/jobs/webapp_deployment b/.jenkins/jobs/webapp_deployment new file mode 100644 index 000000000..032023cf5 --- /dev/null +++ b/.jenkins/jobs/webapp_deployment @@ -0,0 +1,64 @@ +@Library('shared-library') _ +pipeline { + agent any + parameters { + string(name: 'flow_to_execute', defaultValue: 'named_entity_recognition', description: 'The flow use-case to execute') + string(name: 'deploy_environment', defaultValue: 'dev', description: 'Execution Environment') + } + + environment { + dev_connections = credentials('COMMON_DEV_CONNECTIONS') + registry_details = credentials('DOCKER_IMAGE_REGISTRY') + } + + stages { + + // Create local promptflow connections + stage('Create local promptflow connections') { + steps { + installRequirements('execute_job_requirements') + installRequirements('build_validation_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh ''' + python -m llmops.common.prompt_local_connections \\ + --flow_to_execute ${flow_to_execute} \\ + --env_name ${deploy_environment} \\ + --connection_details "$dev_connections" + ''' + } + } + } + } + + // Create Docker image and push to Azure Container Registry + stage('Create Docker Image') { + steps { + installRequirements('build_validation_requirements') + withCredentials([string(credentialsId: 'DOCKER_IMAGE_REGISTRY', variable: 'registry_details'), + string(credentialsId: 'COMMON_DEV_CONNECTIONS', variable: 'connection_details')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + # Your Docker image creation command here + echo "build no script:" + echo ${BUILD_NUMBER} + #!/bin/bash + ./llmops/common/scripts/gen_docker_image.sh --flow_to_execute $flow_to_execute --deploy_environment $deploy_environment --build_id ${BUILD_NUMBER} --REGISTRY_DETAILS '${registry_details}' --CONNECTION_DETAILS '${connection_details}' + """ + } + } + } + } + + // Deploy Docker image to Azure Webapp + stage('Create Webapp') { + steps { + withCredentials([string(credentialsId: 'COMMON_DEV_CONNECTIONS', variable: 'connection_details')]) { + sh """ + /bin/bash ./llmops/common/scripts/az_webapp_deploy.sh --flow_to_execute $flow_to_execute --deploy_environment $deploy_environment --build_id ${BUILD_NUMBER} --CONNECTION_DETAILS '${connection_details}' + """ + } + } + } + } +} \ No newline at end of file diff --git a/.jenkins/pipelines/math_coding_ci_dev b/.jenkins/pipelines/math_coding_ci_dev new file mode 100644 index 000000000..16282ea4a --- /dev/null +++ b/.jenkins/pipelines/math_coding_ci_dev @@ -0,0 +1,42 @@ +pipeline { + agent any + + parameters { + choice(name: 'env_name', choices: ['dev', 'prod'], description: 'Execution Environment') + string(name: 'flow_type', description: 'The flow usecase to execute', defaultValue: 'math_coding') + choice(name: 'deployment_type', choices: ['aml', 'aks', 'webapp'], description: 'Determine type of deployment') + } + + // Trigger the pipeline on push to development branch + triggers { + GenericTrigger( + genericVariables: [ + [key: 'ref', value: '$.ref'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on ref', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$ref', + regexpFilterExpression: 'refs/heads/development' + ) + } + + // Trigger Platform CI Dev Pipeline + stages { + stage('Trigger Platform CI Dev Pipeline') { + steps { + script { + build job: 'platform_ci_dev', parameters: [ + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'deployment_type', value: "${params.deployment_type}") + ] + } + } + } + } +} diff --git a/.jenkins/pipelines/math_coding_pr_dev b/.jenkins/pipelines/math_coding_pr_dev new file mode 100644 index 000000000..0025268d8 --- /dev/null +++ b/.jenkins/pipelines/math_coding_pr_dev @@ -0,0 +1,40 @@ +pipeline { + agent any + + parameters { + string(name: 'env_name', defaultValue: 'pr', description: 'Execution Environment: pr, dev or test') + string(name: 'flow_type', defaultValue: 'math_coding', description: 'The flow usecase to execute') + } + + // Trigger the pipeline on PR opened, reopened or synchronized + triggers { + GenericTrigger( + genericVariables: [ + [key: 'action', value: '$.action'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on $action', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$action', + regexpFilterExpression: '^(opened|reopened|synchronize)$' + ) + } + + // Trigger Platform PR Dev Pipeline + stages { + stage('Trigger Platform PR Dev Pipeline') { + steps { + script { + build job: 'platform_pr_dev', parameters: [ + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'flow_type', value: "${params.flow_type}") + ] + } + } + } + } +} diff --git a/.jenkins/pipelines/named_entity_recognition_ci_dev b/.jenkins/pipelines/named_entity_recognition_ci_dev new file mode 100644 index 000000000..3735caacf --- /dev/null +++ b/.jenkins/pipelines/named_entity_recognition_ci_dev @@ -0,0 +1,42 @@ +pipeline { + agent any + + parameters { + choice(name: 'env_name', choices: ['dev', 'prod'], description: 'Execution Environment') + string(name: 'flow_type', description: 'The flow usecase to execute', defaultValue: 'named_entity_recognition') + choice(name: 'deployment_type', choices: ['aml', 'aks', 'webapp'], description: 'Determine type of deployment') + } + + // Trigger the pipeline on push to development branch + triggers { + GenericTrigger( + genericVariables: [ + [key: 'ref', value: '$.ref'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on ref', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$ref', + regexpFilterExpression: 'refs/heads/development' + ) + } + + // Trigger Platform CI Dev Pipeline + stages { + stage('Trigger Platform CI Dev Pipeline') { + steps { + script { + build job: 'platform_ci_dev', parameters: [ + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'deployment_type', value: "${params.deployment_type}") + ] + } + } + } + } +} diff --git a/.jenkins/pipelines/named_entity_recognition_pr_dev b/.jenkins/pipelines/named_entity_recognition_pr_dev new file mode 100644 index 000000000..49cd98bdf --- /dev/null +++ b/.jenkins/pipelines/named_entity_recognition_pr_dev @@ -0,0 +1,40 @@ +pipeline { + agent any + + parameters { + string(name: 'env_name', defaultValue: 'pr', description: 'Execution Environment: pr, dev or test') + string(name: 'flow_type', defaultValue: 'named_entity_recognition', description: 'The flow usecase to execute') + } + + // Trigger the pipeline on PR opened, reopened or synchronized + triggers { + GenericTrigger( + genericVariables: [ + [key: 'action', value: '$.action'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on $action', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$action', + regexpFilterExpression: '^(opened|reopened|synchronize)$' + ) + } + + // Trigger Platform PR Dev Pipeline + stages { + stage('Trigger Platform PR Dev Pipeline') { + steps { + script { + build job: 'platform_pr_dev', parameters: [ + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'flow_type', value: "${params.flow_type}") + ] + } + } + } + } +} diff --git a/.jenkins/pipelines/platform_cd_dev b/.jenkins/pipelines/platform_cd_dev new file mode 100644 index 000000000..0161efcf3 --- /dev/null +++ b/.jenkins/pipelines/platform_cd_dev @@ -0,0 +1,106 @@ +@Library('shared-library') _ +pipeline { + agent any + +parameters { + string(name: 'env_name', description: 'Execution Environment', defaultValue: 'dev') + string(name: 'flow_type', description: 'The flow use-case to execute', defaultValue: 'named_entity_recognition') + string(name: 'deployment_type', description: 'Determine type of deployment - aml, aks, webapp', defaultValue: 'aml') + string(name: 'run_id', description: 'run id of the flow to be deployed') + } + + stages { + stage('Checkout Git') { + steps { + checkout scm + } + } + + // Installs requirements and registers the flow as a model in AzureML + stage('Register flow as model in AzureML') { + steps { + installRequirements('execute_job_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.deployment.register_model \ + --subscription_id $AZURE_SUBSCRIPTION_ID \ + --flow_to_execute $flow_type \ + --output_file "model_version.txt" \ + --build_id $run_id \ + --env_name $env_name \ + """ + } + } + } + } + + // Export model_version and env_name_quoted as env var to be used in downstream jobs + stage('Export model_version and env_name_quoted as environment variable') { + steps { + script { + def env_name_quoted = "\"${params.env_name}\"" + env.env_name_quoted = env_name_quoted + def model_version = sh (returnStdout: true, script: 'cat model_version.txt').trim() + env.model_version = model_version + sh 'echo $model_version' + + } + } + } + + // Based on the deployment type one of the stages below will deploy the model + stage('Deploy to AML real-time online endpoint') { + when { + expression { + params.deployment_type == 'aml' + } + } + steps { + script { + build job: 'jobs/aml_real_deployment', parameters: [ + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'env_name', value: "${env_name_quoted}"), + string(name: 'model_version', value: "${model_version}"), + string(name: 'run_id', value: "${params.run_id}") + ] + } + } + } + + stage('Deploy to kubernetes real-time online endpoint') { + when { + expression { + params.deployment_type == 'aks' + } + } + steps { + script { + build job: 'jobs/kubernetes_deployment', parameters: [ + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'env_name', value: "${env_name_quoted}"), + string(name: 'model_version', value: "${model_version}"), + string(name: 'run_id', value: "${params.run_id}") + ] + } + } + } + + stage('Deploy to webapp') { + when { + expression { + params.deployment_type == 'webapp' + } + } + steps { + script { + build job: 'jobs/webapp_deployment', parameters: [ + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'env_name', value: "${env_name_quoted}") + ] + } + } + } + } +} + diff --git a/.jenkins/pipelines/platform_ci_dev b/.jenkins/pipelines/platform_ci_dev new file mode 100644 index 000000000..7e7475b07 --- /dev/null +++ b/.jenkins/pipelines/platform_ci_dev @@ -0,0 +1,136 @@ +@Library('shared-library') _ +pipeline { + agent any + + environment { + AZURE_CREDENTIALS = credentials('AZURE_CREDENTIALS') + } + + parameters { + string(name: 'flow_type', defaultValue: 'named_entity_recognition', description: 'The flow use-case to execute') + string(name: 'env_name', defaultValue: 'dev', description: 'Execution Environment') + string(name: 'deployment_type', defaultValue: 'aml', description: 'Determine type of deployment - aml, aks, webapp') + } + + stages { + stage('Checkout Actions') { + steps { + cleanWs() + checkout scm + } + } + + // Install requirements for job execution and register the training data asset in AzureML + stage('Register experiment data asset') { + steps { + installRequirements('execute_job_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh ''' + python -m llmops.common.register_data_asset \\ + --subscription_id $AZURE_SUBSCRIPTION_ID \\ + --data_purpose "training_data" \\ + --flow_to_execute $flow_type \\ + --env_name $env_name + ''' + } + } + + } + } + + // Install requirements for build validation and execute prompt flow bulk run + stage('Execute prompt flow bulk run') { + steps { + installRequirements('build_validation_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh ''' + python -m llmops.common.prompt_pipeline \\ + --subscription_id $AZURE_SUBSCRIPTION_ID \\ + --data_purpose "training_data" \\ + --flow_to_execute $flow_type \\ + --build_id $BUILD_NUMBER \\ + --env_name $env_name \\ + --output_file run_id.txt + ''' + } + } + } + } + + // Register test data set in AzureML + stage('Register evaluation data asset') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh ''' + python -m llmops.common.register_data_asset \\ + --subscription_id $AZURE_SUBSCRIPTION_ID \\ + --data_purpose ""test_data"" \\ + --flow_to_execute $flow_type \\ + --env_name $env_name + ''' + } + } + } + } + + // Export run_id as environment variable to be used in downstream jobs + stage('Export run_id as environment variable') { + steps { + script { + def run_id = sh (returnStdout: true, script: 'cat run_id.txt').trim() + env.run_id = run_id + sh 'echo $run_id' + + } + } + } + + // Read last prompt flow run id and execute bulk run evaluations + stage('Read prompt flow runs & Execute bulk run evaluations') { + steps { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh ''' + run_id=$(cat run_id.txt) + echo $run_id + python -m llmops.common.prompt_eval \\ + --subscription_id $AZURE_SUBSCRIPTION_ID \\ + --build_id $BUILD_NUMBER \\ + --data_purpose ""test_data"" \\ + --flow_to_execute $flow_type \\ + --env_name $env_name \\ + --run_id $run_id + ''' + } + } + } + } + + // Save evaluation results CSV reports as artifacts + stage('Archive CSV') { + steps { + script { + archiveArtifacts artifacts: 'reports/', allowEmptyArchive: true, onlyIfSuccessful: true + } + } + } + + // Execute Platform CD Pipeline to deploy flow to specified deployment type + stage('Execute Platform CD Pipeline to deploy flow') { + steps { + script { + build job: 'platform_cd_dev', parameters: [ + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'deployment_type', value: "${params.deployment_type}"), + string(name: 'run_id', value: "${run_id}") + ] + } + } + } + } +} + diff --git a/.jenkins/pipelines/platform_pr_dev b/.jenkins/pipelines/platform_pr_dev new file mode 100644 index 000000000..4c3b6b0e3 --- /dev/null +++ b/.jenkins/pipelines/platform_pr_dev @@ -0,0 +1,48 @@ +@Library('shared-library') _ +pipeline { + agent any + + parameters { + string(name: 'env_name', defaultValue: 'pr', description: 'Execution Environment') + string(name: 'flow_type', defaultValue: 'named_entity_recognition') + } + + stages { + stage('Install Dependencies') { + steps { + script { + sh 'sudo apt-get install -y azure-cli' + } + } + } + + // Run PR code validation + stage('PR Code Validation') { + steps { + script { + build job: 'jobs/build_validation', parameters: [ + string(name: 'flow_type', value: "${params.flow_type}") + ] + } + } + } + + // Register PR data asset in AzureML + stage('Register experiment data asset') { + steps { + installRequirements('execute_job_requirements') + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + python -m llmops.common.register_data_asset \\ + --subscription_id $AZURE_SUBSCRIPTION_ID \\ + --data_purpose "pr_data" \\ + --flow_to_execute $flow_type \\ + --env_name $env_name + """ + } + } + } + } + } +} \ No newline at end of file diff --git a/.jenkins/pipelines/web_classification_ci_dev b/.jenkins/pipelines/web_classification_ci_dev new file mode 100644 index 000000000..ad8063e42 --- /dev/null +++ b/.jenkins/pipelines/web_classification_ci_dev @@ -0,0 +1,42 @@ +pipeline { + agent any + + parameters { + choice(name: 'env_name', choices: ['dev', 'prod'], description: 'Execution Environment') + string(name: 'flow_type', description: 'The flow usecase to execute', defaultValue: 'web_classification') + choice(name: 'deployment_type', choices: ['aml', 'aks', 'webapp'], description: 'Determine type of deployment') + } + + // Trigger the pipeline on push to development branch + triggers { + GenericTrigger( + genericVariables: [ + [key: 'ref', value: '$.ref'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on ref', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$ref', + regexpFilterExpression: 'refs/heads/development' + ) + } + + // Trigger Platform CI Dev Pipeline + stages { + stage('Trigger Platform CI Dev Pipeline') { + steps { + script { + build job: 'platform_ci_dev', parameters: [ + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'flow_type', value: "${params.flow_type}"), + string(name: 'deployment_type', value: "${params.deployment_type}") + ] + } + } + } + } +} diff --git a/.jenkins/pipelines/web_classification_pr_dev b/.jenkins/pipelines/web_classification_pr_dev new file mode 100644 index 000000000..37568863b --- /dev/null +++ b/.jenkins/pipelines/web_classification_pr_dev @@ -0,0 +1,40 @@ +pipeline { + agent any + + parameters { + string(name: 'env_name', defaultValue: 'pr', description: 'Execution Environment: pr, dev or test') + string(name: 'flow_type', defaultValue: 'web_classification', description: 'The flow usecase to execute') + } + + // Trigger the pipeline on PR opened, reopened or synchronized + triggers { + GenericTrigger( + genericVariables: [ + [key: 'action', value: '$.action'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on $action', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$action', + regexpFilterExpression: '^(opened|reopened|synchronize)$' + ) + } + + // Trigger Platform PR Dev Pipeline + stages { + stage('Trigger Platform PR Dev Pipeline') { + steps { + script { + build job: 'platform_pr_dev', parameters: [ + string(name: 'env_name', value: "${params.env_name}"), + string(name: 'flow_type', value: "${params.flow_type}") + ] + } + } + } + } +} diff --git a/.jenkins/requirements/build_validation_requirements.txt b/.jenkins/requirements/build_validation_requirements.txt new file mode 100644 index 000000000..6958387f4 --- /dev/null +++ b/.jenkins/requirements/build_validation_requirements.txt @@ -0,0 +1,18 @@ +flake8-docstrings==1.6.0 +flake8==4.0.1 +pep8-naming==0.13.0 +pytest-cov==3.0.0 +pytest-azurepipelines==1.0.3 +pytest-mock==3.7.0 +pytest==7.1.2 +azure-ai-ml==1.13.0 +azure-identity==1.15.0 +mldesigner==0.1.0b4 +pandas==2.2.1 +promptflow==1.6.0 +promptflow-tools==1.3.0 +promptflow-sdk==0.0.1 +jinja2==3.1.3 +promptflow[azure] +openai +promptflow-sdk[builtins] diff --git a/.jenkins/requirements/execute_job_requirements.txt b/.jenkins/requirements/execute_job_requirements.txt new file mode 100644 index 000000000..ac859fe24 --- /dev/null +++ b/.jenkins/requirements/execute_job_requirements.txt @@ -0,0 +1,5 @@ +azure-ai-ml==1.13.0 +azure-identity==1.14.0 +python-dotenv>=0.10.3 +azureml-mlflow>=1.51 +keyrings.alt diff --git a/.jenkins/shared-library/vars/azureLogin.groovy b/.jenkins/shared-library/vars/azureLogin.groovy new file mode 100644 index 000000000..6375ccf25 --- /dev/null +++ b/.jenkins/shared-library/vars/azureLogin.groovy @@ -0,0 +1,6 @@ +def call() { + withCredentials([azureServicePrincipal('AZURE_CREDENTIALS')]) { + sh 'az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET -t $AZURE_TENANT_ID' + sh 'az account set -s $AZURE_SUBSCRIPTION_ID' + } +} \ No newline at end of file diff --git a/.jenkins/shared-library/vars/installRequirements.groovy b/.jenkins/shared-library/vars/installRequirements.groovy new file mode 100644 index 000000000..0fe145298 --- /dev/null +++ b/.jenkins/shared-library/vars/installRequirements.groovy @@ -0,0 +1,9 @@ +def call(String requirements_type) { + withPythonEnv('/usr/bin/python3.9') { + sh """ + pip install setuptools wheel + pip install -r .jenkins/requirements/${requirements_type}.txt + pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] + """ + } +} \ No newline at end of file diff --git a/README.md b/README.md index 08559ed0d..4c9df20df 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,9 @@ Additionally, there is a llmops_config.json file that refers to important infras - The '.github' folder contains the Github workflows for the platform as well as the use-cases. This is bit different than Azure DevOps because all Github workflows should be within this single folder for execution. -- The 'docs' folder contains documentation for step-by-step guides for both Azure DevOps and Github Workflow related configuration. +- The '.jenkins' folder contains the Jenkins declarative pipelines for the platform as well as the use-cases and individual jobs. + +- The 'docs' folder contains documentation for step-by-step guides for both Azure DevOps, Github Workflow and Jenkins related configuration. - The 'llmops' folder contains all the code related to flow execution, evaluation and deployment. @@ -75,6 +77,7 @@ Additionally, there is a llmops_config.json file that refers to important infras - Full documentation on using this repo using Azure DevOps can be found [here](./docs/Azure_devops_how_to_setup.md) - Full documentation on using this repo using Github Workflows can be found [here](./docs/github_workflows_how_to_setup.md) +- Full documentation on using this repo using Jenkins can be found [here](./docs/jenkins_how_to_setup.md) - Documentation about adding a new flow is available [here](./docs/how_to_onboard_new_flows.md) # Deployment diff --git a/docs/images/jenkins-credentials-dev-conn.png b/docs/images/jenkins-credentials-dev-conn.png new file mode 100644 index 000000000..9b7c71ac6 Binary files /dev/null and b/docs/images/jenkins-credentials-dev-conn.png differ diff --git a/docs/images/jenkins-credentials.png b/docs/images/jenkins-credentials.png new file mode 100644 index 000000000..ee9507c71 Binary files /dev/null and b/docs/images/jenkins-credentials.png differ diff --git a/docs/images/jenkins-pipeline-test.png b/docs/images/jenkins-pipeline-test.png new file mode 100644 index 000000000..e259ebc09 Binary files /dev/null and b/docs/images/jenkins-pipeline-test.png differ diff --git a/docs/images/jenkins-pr-trigger.png b/docs/images/jenkins-pr-trigger.png new file mode 100644 index 000000000..3fede2bd0 Binary files /dev/null and b/docs/images/jenkins-pr-trigger.png differ diff --git a/docs/images/jenkins-webhook-gh.png b/docs/images/jenkins-webhook-gh.png new file mode 100644 index 000000000..0092b7c89 Binary files /dev/null and b/docs/images/jenkins-webhook-gh.png differ diff --git a/docs/jenkins_how_to_setup.md b/docs/jenkins_how_to_setup.md new file mode 100644 index 000000000..1b810c78b --- /dev/null +++ b/docs/jenkins_how_to_setup.md @@ -0,0 +1,543 @@ +# How to setup the repo with Jenkins + +This template supports Azure Machine Learning (ML) as a platform for LLMOps, and Jenkins as a platform for Flow operationalization. LLMOps with Prompt flow provides automation of: + +* Experimentation by executing flows +* Evaluation of prompts along with their variants +* Registration of prompt flow 'flows' +* Deployment of prompt flow 'flows' +* Generation of Docker Image +* Deployment to Kubernetes, Azure Web Apps and Azure ML compute +* A/B deployments +* Role based access control (RBAC) permissions to deployment system managed id to key vault and Azure ML workspace +* Endpoint testing +* Report generation + +It is important to understand how [Prompt flow works](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/get-started-prompt-flow?view=azureml-api-2) before using this template. + +## Prerequisites + +- An Azure subscription. If you don't have an Azure subscription, create a free account before you begin. Try the [free or paid version of Machine Learning](https://azure.microsoft.com/free/). +- An Azure Machine Learning workspace. +- Git running on your local machine. +- GitHub as the source control repository. +- Azure OpenAI with Model deployed with name `gpt-35-turbo`. +- In case of Kubernetes based deployment, Kubernetes resources and associating it with Azure Machine Learning workspace would be required. More details about using Kubernetes as compute in AzureML is available [here](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-attach-kubernetes-anywhere?view=azureml-api-2). + +The template deploys real-time online endpoints for flows. These endpoints have Managed ID assigned to them and in many cases they need access to Azure Machine learning workspace and its associated key vault. The template by default provides access to both the key vault and Azure Machine Learning workspace based on this [document](https://learn.microsoft.com/en-ca/azure/machine-learning/prompt-flow/how-to-deploy-for-real-time-inference?view=azureml-api-2#grant-permissions-to-the-endpoint). + +## Create Azure service principal + +Create one Azure service principal for the purpose of understanding this repository. You can add more depending on how many environments, you want to work on (Dev or Prod or Both). Service principals can be created using cloud shell, bash, powershell or from Azure UI. If your subscription is part of an organization with multiple tenants, ensure that the Service Principal has access across tenants. + +1. Copy the following bash commands to your computer and update the **spname** and **subscriptionId** variables with the values for your project. This command will also grant the **owner** role to the service principal in the subscription provided. This is required for Jenkins to properly use resources in that subscription. + + ``` bash + spname="" + roleName="Owner" + subscriptionId="" + servicePrincipalName="Azure-ARM-${spname}" + + # Verify the ID of the active subscription + echo "Using subscription ID $subscriptionID" + echo "Creating SP for RBAC with name $servicePrincipalName, with role $roleName and in scopes /subscriptions/$subscriptionId" + + az ad sp create-for-rbac --name $servicePrincipalName --role $roleName --scopes /subscriptions/$subscriptionId --sdk-auth + + echo "Please ensure that the information created here is properly save for future use." + +1. Copy your edited commands into the Azure Shell and run them (**Ctrl** + **Shift** + **v**). If executing the commands on local machine, ensure Azure CLI is installed and successfully able to access after executing `az login` command. Azure CLI can be installed using information available [here](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) + +1. After running these commands, you'll be presented with information related to the service principal. Save this information to a safe location, you'll use it later in the demo to configure Jenkins. + +`NOTE: The below information should never be part of your repository and its branches. These are important secrets and should never be pushed to any branch in any repository.` + + ```json + + { + "clientId": "", + "clientSecret": "", + "subscriptionId": "", + "tenantId": "", + "activeDirectoryEndpointUrl": "https://login.microsoftonline.com", + "resourceManagerEndpointUrl": "https://management.azure.com/", + "activeDirectoryGraphResourceId": "https://graph.windows.net/", + "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/", + "galleryEndpointUrl": "https://gallery.azure.com/", + "managementEndpointUrl": "https://management.core.windows.net/" + } + ``` + +1. Copy the output, braces included. It will be used later in the demo to configure GitHub Repo. + +1. Close the Cloud Shell once the service principals are created. + +## Setup runtime for Prompt flow + +Prompt flow 'flows' require runtime associated with compute instance in Azure Machine Learning workspace. Both the compute instance and the associated runtime should be created prior to executing the flows. Both the Compute Instance and Prompt flow runtime should be created using the Service Principal. This ensures that Service Principal is the owner of these resources and Flows can be executed on them from Azure DevOps pipelines, Github workflows and Jenkins. This repo provides Azure CLI commands to create both the compute instance and the runtime using Service Principal. + +Compute Instances and Prompt flow runtimes can be created using cloud shell, local shells, or from Azure UI. If your subscription is a part of organization with multiple tenants, ensure that the Service Principal has access across tenants. The steps shown next can be executed from Cloud shell or any shell. The steps mentioned are using Cloud shell and they explicitly mentions any step that should not be executed in cloud shell. + +### Steps: + +1. Assign values to variables. Copy the following bash commands to your computer and update the variables with the values for your project. Note that there should not be any spaces between both side of "=" operator while assigning values to bash variables. + +```bash +subscriptionId= +rgname= +workspace_name= +userAssignedId= +keyvault= +compute_name= +location= +runtimeName= +sp_id= +sp_password= +tenant_id= +``` + +2. This next set of commands should not be performed from Cloud shell. It should be performed if you are using a local terminal. The commands help to interactively log in to Azure and selects a subscription. + +```bash +az login +az account set -s $subscriptionId +``` + +3. Create a user-assigned managed identity + +```bash +az identity create -g $rgname -n $userAssignedId --query "id" +``` + +4. Get id, principalId of user-assigned managed identity + +```bash +um_details=$(az identity show -g $rgname -n $userAssignedId --query "[id, clientId, principalId]") +``` + +5. Get id of user-assigned managed identity + +```bash +user_managed_id="$(echo $um_details | jq -r '.[0]')" +``` + +6. Get principal Id of user-assigned managed identity + +```bash +principalId="$(echo $um_details | jq -r '.[2]')" +``` + +7. Grant the user managed identity permission to access the workspace (AzureML Data Scientist) + +```bash +az role assignment create --assignee $principalId --role "AzureML Data Scientist" --scope "/subscriptions/$subscriptionId/resourcegroups/$rgname/providers/Microsoft.MachineLearningServices/workspaces/$workspace_name" +``` + +8. Grant the user managed identity permission to access the workspace keyvault (get and list) + +```bash +az keyvault set-policy --name $keyvault --resource-group $rgname --object-id $principalId --secret-permissions get list +``` + +9. login with Service Principal + +```bash +az login --service-principal -u $sp_id -p $sp_password --tenant $tenant_id +az account set -s $subscriptionId +``` + +10. Create compute instance and assign user managed identity to it + +```bash +az ml compute create --name $compute_name --size Standard_E4s_v3 --identity-type UserAssigned --type ComputeInstance --resource-group $rgname --workspace-name $workspace_name --user-assigned-identities $user_managed_id +``` + +11. Get Service Principal Azure Entra token for REST API + +```bash +access_token=$(az account get-access-token | jq -r ".accessToken") +``` + +12. Construct POST url for runtime + +```bash +runtime_url_post=$(echo "https://ml.azure.com/api/$location/flow/api/subscriptions/$subscriptionId/resourceGroups/$rgname/providers/Microsoft.MachineLearningServices/workspaces/$workspace_name/FlowRuntimes/$runtimeName?asyncCall=true") +``` + +13. Construct GET url for runtime + +```bash +runtime_url_get=$(echo "https://ml.azure.com/api/$location/flow/api/subscriptions/$subscriptionId/resourceGroups/$rgname/providers/Microsoft.MachineLearningServices/workspaces/$workspace_name/FlowRuntimes/$runtimeName") +``` + +14. Create runtime using REST API + +```bash +curl --request POST \ + --url "$runtime_url_post" \ + --header "Authorization: Bearer $access_token" \ + --header 'Content-Type: application/json' \ + --data "{ + \"runtimeType\": \"ComputeInstance\", + \"computeInstanceName\": \"$compute_name\", +}" +``` + +15. Get runtime creation status using REST API. Execute this step multiple times unless either you get output that shows createdOn with a valid date and time value or failure. In case of failure, troubleshoot the issue before moving forward. + +```bash +curl --request GET \ + --url "$runtime_url_get" \ + --header "Authorization: Bearer $access_token" +``` + +The template also provides support for 'automatic runtime' where flows are executed within a runtime provisioned automatically during execution. This feature is in preview. The first execution might need additional time for provisioning of the runtime. + +The template supports using dedicated compute instances and runtimes by default and 'automatic runtime' can be enabled easily with minimal change in code. (Search for COMPUTE_RUNTIME in code for such changes. The comments in code provides additional information and context for required changes.) and also remove any value in `llmops_config.json` for each use-case example for `RUNTIME_NAME`. + +## Set up Github Repo + +Fork this repository [LLMOps Prompt flow Template Repo](https://github.com/microsoft/llmops-promptflow-template) in your GitHub organization. This repo has reusable LLMOps code that can be used across multiple projects. + +![fork the repository](images/fork.png) + +Create a *development* branch from *main* branch and also make it as default one to make sure that all PRs should go towards it. This template assumes that the team works at a *development* branch as a primary source for coding and improving the prompt quality. Later, you can implement Jenkins pipelines that move code from the *development* branch into qa/main or execute a release process right away. Release management is not in part of this template. + +![create a new development branch](images/new-branch.png) + +At this time, `main` is the default branch. + +![main as default branch](images/main-default.png) + +From the settings page, switch the default branch from `main` to `development` + +![change development as default branch](images/change-default-branch.png) + +It might ask for confirmation. + +![default branch confirmation](images/confirmation.png) + +Eventually, the default branch in github repo should show `development` as the default branch. + +![make development branch as default branch](images/default-branch.png) + +The template comes with a few declarative Jenkins pipelines related to Prompt flow flows for providing a jumpstart (named_entity_recognition, web_classification and math_coding). Each scenario has 2 primary workflows and 1 optional workflow. The first one is executed during pull request(PR) e.g. [named_entity_recognition_pr_dev](../.jenkins/pipelines/named_entity_recognition_pr_dev), and it helps to maintain code quality for all PRs. Usually, this pipeline uses a smaller dataset to make sure that the Prompt flow job can be completed fast enough. + +The second Jenkins pipelines [named_entity_recognition_ci_dev](../.jenkins/pipelines/named_entity_recognition_ci_dev) is executed automatically before a PR is merged into the *development* or *main* branch. The main idea of this pipeline is to execute bulk run and evaluation on the full dataset for all prompt variants. The workflow can be modified and extended based on the project's requirements. + +## Set up Jenkins Credentials for Azure + +From your Jenkins dashboard, select **Manage Jenkins** -> **Credentials**, **(global) Domain** and **+ Add Credentials**. From the **Kind** dropdown box, select the **Azure Service Principal** option. Provide the service principal secret values you saved earlier: `Subscription ID`, `Client ID`, `Client Secret` and `Tenant ID`. Enter `AZURE_CREDENTIALS` for the `ID` field - this value will be used as a name/reference for the credentials inside each pipeline. + +![Screenshot of Jenkins Credentials.](images/jenkins-credentials.png) + +## Setup connections for Prompt flow + +Prompt flow Connections helps securely store and manage secret keys or other sensitive credentials required for interacting with LLM and other external tools, for example Azure OpenAI. + +This repository has 3 examples, and all the examples use a connection named `aoai` inside, we need to set up a connection with this name if we haven’t created it before. + +This repository has all the examples use Azure OpenAI model `gpt-35-turbo` deployed with the same name `gpt-35-turbo`, we need to set up this deployment if we haven’t created it before. + +Please go to Azure Machine Learning workspace portal, click `Prompt flow` -> `Connections` -> `Create` -> `Azure OpenAI`, then follow the instructions to create your own connections called `aoai`. Learn more on [connections](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/concept-connections?view=azureml-api-2). The samples use a connection named "aoai" connecting to a gpt-35-turbo model deployed with the same name in Azure OpenAI. This connection should be created before executing the out-of-box flows provided with the template. + +![aoai connection in Prompt flow](images/connection.png) + +The configuration for connection used while authoring the repo: + +![connection details](images/connection-details.png) + +## Set up Jenkins Credentials for Prompt Flow and ACR + +### Prompt flow Connection + +Create Jenkins Credentials of the type **Secret Text** named `COMMON_DEV_CONNECTIONS` with information related to Prompt flow connections. The value for this secret is a json string with given structure as shown next. Create one json object for each connection as shown in the example. As of now, Azure Open AI Connections are supported and more will get added soon. Information for each connection can be obtained from respective Azure OpenAI resource. + +![Jenkins Secret Text Credential for common dev connections](images/jenkins-credentials-dev-conn.png) + +It is important to note that there can be any number of variables and each storing Prompt flow connection details as shown next and they can be named anything permissible based on your preference. You should use the same variable in your use-case related CI pipelines. The template by default uses `COMMON_DEV_CONNECTIONS` for this purpose. + +```json +[ +{ + "name": "aoai", + "type": "azure_open_ai", + "api_base": "https://xxxxxxxxxxx/", + "api_key": "xxxxxxxxxxxx", + "api_type": "azure", + "api_version": "2023-03-15-preview" +}, +{ + "name": "another_connection", + "type": "azure_open_ai", + "api_base": "https://xxxxxxxxxxxx/", + "api_key": "xxxxxxxxxxxx", + "api_type": "azure", + "api_version": "2023-03-15-preview" +} +] + +``` + +### Azure Container Registry + +Create Jenkins Credentials of the type **Secret Text** named `DOCKER_IMAGE_REGISTRY` with information related to Docker Image Registry. The value for this secret is also a json string with given structure. Create one json object for each registry. As of now, Azure Container Registry are supported and more will get added soon. Information for each registry can be obtained from Azure Container Registry resource. + +It is important to note that there can be any number of variables and each storing Azure Container Registry details as shown next and they can be named anything permissible based on your preference. You should use the same variable in your use-case related CI pipelines. The template by default uses `DOCKER_IMAGE_REGISTRY` for this purpose. + +```json +[ + { + "registry_name" : "xxxxxxxx", + "registry_server" : "xxxx.azurecr.io", + "registry_username" : "xxxxxxxxx", + "registry_password": "xxxxxxxxxxxxxx" + } +] + +``` + +## Cloning the repo + + Now, we can clone the forked repo on your local machine using command shown here. Replace the repo url with your url. + +``` bash +git clone https://github.com/ritesh-modi/llmops-promptflow-template.git + +cd llmops-promptflow-template + +git branch + +``` + +Create a new feature branch using the command shown here. Replace the branch name with your preferred name. + +``` bash + +git checkout -b featurebranch + +``` + +Update the `llmops_config.json` file for any one of the examples (e.g. `named_entity_recognization`). Update configuration so that we can create a pull request for any one of the example scenarios (e.g. named_entity_recognition). Navigate to scenario folder and update the `llmops_config.json` file. Update the KEYVAULT_NAME, RESOURCE_GROUP_NAME, RUNTIME_NAME and WORKSPACE_NAME. Update the `ENDPOINT_NAME` and `CURRENT_DEPLOYMENT_NAME` in `configs/deployment_config.json` file for deployment to Azure Machine Learning compute. Update the `CONNECTION_NAMES`, `REGISTRY_NAME`, `REGISTRY_RG_NAME`, `APP_PLAN_NAME`, `WEB_APP_NAME`, `WEB_APP_RG_NAME`, `WEB_APP_SKU`, and `USER_MANAGED_ID` in `configs/deployment_config.json` file for deployment to Azure Web App. + +### Update llmops_config.json + +Modify the configuration values in the `llmops_config.json` file available for each example based on description. + +- `ENV_NAME`: This represents the environment type. (The template supports *pr* and *dev* environments.) +- `RUNTIME_NAME`: This is the name of a Prompt flow runtime environment, used for executing the prompt flows. Add values to this field only when you are using dedicated runtime and compute. The template uses automatic runtime by default. +- `KEYVAULT_NAME`: This points to an Azure Key Vault related to the Azure ML service, a service for securely storing and managing secrets, keys, and certificates. +- `RESOURCE_GROUP_NAME`: Name of the Azure resource group related to Azure ML workspace. +- `WORKSPACE_NAME`: This is name of Azure ML workspace. +- `STANDARD_FLOW_PATH`: This is the relative folder path to files related to a standard flow. e.g. e.g. "flows/standard_flow.yml" +- `EVALUATION_FLOW_PATH`: This is a string value referring to relative evaluation flow paths. It can have multiple comma separated values- one for each evaluation flow. e.g. "flows/eval_flow_1.yml,flows/eval_flow_2.yml" + +For the optional post production evaluation workflow, the above configuration will be same only `ENV_NAME` will be *postprodeval* and the respective flow path need to be mentioned in `STANDARD_FLOW_PATH` configuration. + +### Update deployment_config.json in config folder + +Modify the configuration values in the `deployment_config.json` file for each environment. These are required for deploying Prompt flows in Azure ML. Ensure the values for `ENDPOINT_NAME` and `CURRENT_DEPLOYMENT_NAME` are changed before pushing the changes to remote repository. + +- `ENV_NAME`: This indicates the environment name, referring to the "development" or "production" or any other environment where the prompt will be deployed and used in real-world scenarios. +- `TEST_FILE_PATH`: The value represents the file path containing sample input used for testing the deployed model. +- `ENDPOINT_NAME`: The value represents the name or identifier of the deployed endpoint for the prompt flow. +- `ENDPOINT_DESC`: It provides a description of the endpoint. It describes the purpose of the endpoint, which is to serve a prompt flow online. +- `DEPLOYMENT_DESC`: It provides a description of the deployment itself. +- `PRIOR_DEPLOYMENT_NAME`: The name of prior deployment. Used during A/B deployment. The value is "" if there is only a single deployment. Refer to CURRENT_DEPLOYMENT_NAME property for the first deployment. +- `PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION`: The traffic allocation of prior deployment. Used during A/B deployment. The value is "" if there is only a single deployment. Refer to CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION property for the first deployment. +- `CURRENT_DEPLOYMENT_NAME`: The name of current deployment. +- `CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION`: The traffic allocation of current deployment. A value of 100 indicates that all traffic is directed to this deployment. +- `DEPLOYMENT_VM_SIZE`: This parameter specifies the size or configuration of the virtual machine instances used for the deployment. +- `DEPLOYMENT_INSTANCE_COUNT`:This parameter specifies the number of instances (virtual machines) that should be deployed for this particular configuration. +- `ENVIRONMENT_VARIABLES`: This parameter represents a set of environment variables that can be passed to the deployment. + +Kubernetes deployments have additional properties - `COMPUTE_NAME`, `DEPLOYMENT_VM_SIZE`, `CPU_ALLOCATION` and `MEMORY_ALLOCATION` related to infrastructure and resource requirements. These should also be updates with your values before executing Kubernetes based deployments. + +Azure Web App deployments do not have similar properties as that of Kubernetes and Azure ML compute. For Azure Web App, following properties should be updated. + +- `ENV_NAME`: This indicates the environment name, referring to the "development" or "production" or any other environment where the prompt will be deployed and used in real-world scenarios. +- `TEST_FILE_PATH`: The value represents the file path containing sample input used for testing the deployed model. +- `CONNECTION_NAMES`: The name of the connections used in standard flow in json aray format. e.g. ["aoai", "another_connection"], +- `REGISTRY_NAME`: This is the name of the Container Registry that is available in the `DOCKER_IMAGE_REGISTRY` secret. Based on this name, appropriate registry details will be used for `Docker` image management. +- `REGISTRY_RG_NAME`: This is the name of the resource group related to the Container Registry. It is used for downloading the Docker Image. +- `APP_PLAN_NAME`: Name of the App Services plan. It will be provisioned by the pipeline. +- `WEB_APP_NAME`: Name of the Web App. It will be provisioned by the pipeline. +- `WEB_APP_RG_NAME`: Name of the resource group related to App Service plan and Web App. It will be provisioned by the pipeline. +- `WEB_APP_SKU`: This is the `SKU` (size) of the Web App. e.g. "B3" +- `USER_MANAGED_ID`: This is the name of the user defined managed id created during deployment associated with the Web App. + +`NOTE: For Docker based deployments, ensure to add promptflow, promptflow[azure], promptflow-tools packages in each flow's requirements.txt file`. Check-out existing use-cases (named_entity_recognition, math_coding, web_classification) for examples.` + +Now, push the new feature branch to the newly forked repo. + +``` bash + +git add . +git commit -m "changed code" +git push -u origin featurebranch + +``` + +## Configure pipelines in Jenkins + +In this example, all pipelines will be created for the **Named Entity Recognition Flow**, feel free to use instead one of the other supported flows (Math Coding or Web Classification) for your setup. + +In your Jenkins environment create a new **Pipeline** with the name `named_entity_recognition_pr_dev` and select the option to define the **Pipeline Script from SCM**. Select **Git** as the Source Control option, and provide the **Repository URL** for your forked repository. In the **Branch Specifier** field, provide `*/`. For the **Script Path** field, provide the path to the Jenkinsfile for the pipeline you're creating, which is `.jenkins/pipelines/named_entity_recognition_pr_dev`, and press **Save**. + +These steps shoud be repeated for all pipelines which are required to run the PR and CI/CD pipelines for the NER flow. In the table below you can find the list of pipeline names, branches and script paths required for each pipeline. + +| Pipeline Name | Branch Specifier | Script Path | +| ------------- | ----------- | ----------- | +| named_entity_recognition_pr_dev | `*/` | `.jenkins/pipelines/named_entity_recognition_pr_dev` | +| platform_pr_dev | `*/` | `.jenkins/pipelines/platform_pr_dev` | +| named_entity_recognition_ci_dev | `*/development` | `.jenkins/pipelines/named_entity_recognition_ci_dev` | +| platform_ci_dev | `*/development` |`.jenkins/pipelines/platform_ci_dev` | +| platform_cd_dev | `*/development` |`.jenkins/pipelines/platform_cd_dev` | + +## Configure actions in Jenkins + +Many of the Jenkins pipelines trigger specific actions which are stored in the [.jenkins/jobs/](../.jenkins/jobs/) directory. Therefore it is necessary to configure Jenkins pipelines for each of the individual actions. + +First, create a **+ New Item** of the type **Folder** and call it `jobs`. Inside the `jobs` folder, create pipelines for each action file in the same way you did for the pipelines, using the **Pipeline Script from SCM** option. The details for which branch and script path to use are in the table below: + +| Pipeline Name | Branch Specifier | Script Path | +| ------------- | ----------- | ----------- | +| aml_real_deployment | `*/development` | `.jenkins/jobs/aml_real_deployment` | +| build_validation | `*/` | `.jenkins/jobs/build_validation` | +| kubernetes_deployment | `*/development` | `.jenkins/jobs/kubernetes_deployment` | +| prepare_docker_image | `*/development` | `.jenkins/jobs/prepare_docker_image` | +| webapp_deployment | `*/development` | `.jenkins/jobs/webapp_deployment` | + +## Set up Trigger Webhook + +Both the PR and CI pipelines for each use case will be triggered by different events. The PR pipeline will be triggered whenever a PR is opened on a `feature branch`, and the CI pipeline will be triggered once a merge is done on the `development` branch. To send these events from the GitHub repository to Jenkins it's necessary to define a `Webhook`. This can be done in the **Settings** tab of your GitHub repository, then going to the **Webhooks** tab and press **Add webhook**. + +In the **payload** section, provide the following url: +`http://:/generic-webhook-trigger/invoke?token=`. +In the **secret** section, provide a random string with high entropy to be used as a token by Jenkins. +Select **Send me everything** from the options on which events to send through this webhook. Then press **Add Webhook**. + +![Create a GitHub webhook](images/jenkins-webhook-gh.png) + +## Create Jenkins Credentials for Webhook Token Secret + +Create another set of **Jenkins Credentials** of type **Secret Text** with the name `WEBHOOK-TOKEN-SECRET`, to store the value of the GitHub Webhook secret, such that it can be used inside the pipeline. + +Inside the Jenkins pipelines, the `GenericTrigger` block is already set up to define the regular expressions which ensure that the pipeline is triggered only when certain events are part of the webhook request, such as the PR being `opened`, `reopened` or `synchronized` as can be seen in this example: + +```groovy + triggers { + GenericTrigger( + genericVariables: [ + [key: 'action', value: '$.action'] + ], + genericHeaderVariables: [ + ], + causeString: 'Triggered on $action', + tokenCredentialId: 'WEBHOOK-TOKEN-SECRET', + printContributedVariables: true, + printPostContent: false, + silentResponse: false, + regexpFilterText: '$action', + regexpFilterExpression: '^(opened|reopened|synchronize)$' + ) + } +``` + +## Raise PR to trigger pipeline + +Raise a new PR to merge code from `feature branch` to the `development` branch. Ensure that the PR from feature branch to development branch happens within your repository and organization. + +![raise a new PR](images/pr.png) + +This should start the process of executing the Named Entity Recognition PR pipeline. + +![PR pipeline execution](images/jenkins-pr-trigger.png) + +After the execution is complete, the code can be merged to the `development` branch. Upon merging into `development` the CI Pipeline will be triggered, which in its turn triggers the CD Pipeline. This CI/CD pipeline will target the `dev` environment for deployment. +Now a new PR can be opened from `development` branch to the `main` branch. This should execute both the PR as well as the CI/CD pipeline. + +## Update configurations for Prompt flow and Jenkins + +There are multiple configuration files for enabling Prompt flow run and evaluation in Azure ML and Jenkins + +### Update mapping_config.json in config folder + +Modify the configuration values in the `mapping_config.json` file based on both the standard and evaluation flows for an example. These are used in both experiment and evaluation flow execution. + +- `experiment`: This section defines inputs for standard flow. The values comes from corresponding experiment dataset. +- `evaluation`: This section defines the inputs for the related evaluation flows. The values generally comes from two sources - dataset and output from bulk run. Evaluation involves comparing predictions made during bulk run execution of a standard flow with corresponding expected ground truth values and eventually used to assess the performance of prompt variants. + +### Update data_config.json in config folder + +Modify the configuration values in the `data_config.json` file based on the environment. These are required in creating data assets in Azure ML and also consume them in pipelines. + +- `ENV_NAME`: This indicates the environment name, referring to the "development" or "production" or any other environment where the prompt will be deployed and used in real-world scenarios. +- `DATA_PURPOSE`: This denotes the purpose of the data usage. This includes data for pull-request(pr_data), experimentation(training_data) or evaluation(test_data). These 3 types are supported by the template. +- `DATA_PATH`: This points to the file path e.g. "flows/web_classification/data/data.jsonl". +- `DATASET_NAME`: This is the name used for the created Data Asset on Azure ML. Special characters are not allowed for naming the dataset. +- `RELATED_EXP_DATASET`: This element is used to relate data used for bulk run with the data used for evaluation. The value is the name of the dataset used for standard flow. +- `DATASET_DESC`: This provides a description for the dataset. + +### Update data folder with data files + +Add your data into the `data` folder under the use case folder. It supports `jsonl` files and the examples already contains data files for both running and evaluating Prompt flows. + +### Update Standard and evaluation flows + +The `flows` folder contains one folder for each standard and evaluation flow. Each example in the repository already has these flows. + +### Update Environment related dependencies + +The `environment folder` contains dockerfile file for webapp and function app deployments. Any additional dependencies needed by the flow should be added to it. This file is used during deployment process. + +### Update test data + +The `sample-request.json` file contains a single test data used for testing the online endpoint after deployment in the pipeline. Each example has its own `sample-request.json` file and for custom flows, it should be updated to reflect test data needed for testing. + +## Example Prompt Run, Evaluation and Deployment Scenario + +There are three examples in this template. While `named_entity_recognition` and `math_coding` have the same functionality, `web_classification` has multiple evaluation flows and datasets for the dev environment. This is a flow in general across all examples. + +This Jenkins CI workflow contains the following steps: + +**Run Prompts in Flow** +- Upload bulk run dataset +- Bulk run prompt flow based on dataset +- Bulk run each prompt variant + +**Evaluate Results** +- Upload ground test dataset +- Execution of multiple evaluation flows for a single bulk run (only for web_classification) +- Evaluation of the bulk run result using single evaluation flow (for others) + +**Register Prompt flow LLM App** +- Register Prompt flow as a Model in Azure Machine Learning Model Registry + +**Deploy and Test LLM App** +- Deploy the Flow as a model to the development environment either as Kubernetes or Azure ML Compute endpoint +- Assign RBAC permissions to the newly deployed endpoint to Key Vault and Azure ML workspace +- Test the model/promptflow realtime endpoint. + +**Run post production deployment evaluation** +- Upload the sampled production log dataset +- Execute the evaluation flow on the production log dataset +- Generate the evaluation report + +### Online Endpoint + +1. After the CI pipeline for an example scenario has run successfully, depending on the configuration it will either deploy to + + ![Managed online endpoint](./images/online-endpoint.png) or to a Kubernetes compute type + + ![Managed online endpoint](./images/kubernetes.png) + +2. Once pipeline execution completes, it would have successfully completed the test using data from `sample-request.json` file as well. + + ![online endpoint test in pipeline](./images/jenkins-pipeline-test.png) + +## Moving to production + +The example scenario can be run and deployed both for Dev environments. When you are satisfied with the performance of the prompt evaluation pipeline, Prompt flow model, and deployment in development, additional pipelines similar to `dev` pipelines can be replicated and deployed in the Production environment. + +The sample Prompt flow run & evaluation and Jenkins pipelines can be used as a starting point to adapt your own prompt engineering code and data. diff --git a/llmops/common/scripts/az_webapp_deploy.sh b/llmops/common/scripts/az_webapp_deploy.sh index c743f0888..1d59744af 100755 --- a/llmops/common/scripts/az_webapp_deploy.sh +++ b/llmops/common/scripts/az_webapp_deploy.sh @@ -10,6 +10,32 @@ # private networks and/or use different means of provisioning # using Terraform, Bicep or any other way. +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --flow_to_execute) + flow_to_execute="$2" + shift 2 + ;; + --deploy_environment) + deploy_environment="$2" + shift 2 + ;; + --build_id) + build_id="$2" + shift 2 + ;; + --CONNECTION_DETAILS) + CONNECTION_DETAILS="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + set -e # fail on error # read values from deployment_config.json related to `webapp_endpoint` @@ -46,14 +72,14 @@ az appservice plan create --name $appserviceplan --resource-group $rgname --is-l # create/update Web App az webapp create --resource-group $rgname --plan $appserviceplan --name $appserviceweb --deployment-container-image-name \ - $REGISTRY_NAME.azurecr.io/"$flow_to_execute"_"$deploy_environment":$build_id + $REGISTRY_NAME.azurecr.io/"$flow_to_execute"_"$deploy_environment":"$build_id" # create/update Web App config settings az webapp config appsettings set --resource-group $rgname --name $appserviceweb \ --settings WEBSITES_PORT=8080 for name in "${connection_names[@]}"; do - api_key=$(echo $CONNECTION_DETAILS | jq -r --arg name "$name" '.[] | select(.name == $name) | .api_key') + api_key=$(echo ${CONNECTION_DETAILS} | jq -r --arg name "$name" '.[] | select(.name == $name) | .api_key') uppercase_name=$(echo "$name" | tr '[:lower:]' '[:upper:]') modified_name="${uppercase_name}_API_KEY" diff --git a/llmops/common/scripts/gen_docker_image.sh b/llmops/common/scripts/gen_docker_image.sh index 6d4430703..5e822e8db 100755 --- a/llmops/common/scripts/gen_docker_image.sh +++ b/llmops/common/scripts/gen_docker_image.sh @@ -1,8 +1,42 @@ #!/bin/bash +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --flow_to_execute) + flow_to_execute="$2" + shift 2 + ;; + --deploy_environment) + deploy_environment="$2" + shift 2 + ;; + --build_id) + build_id="$2" + shift 2 + ;; + --REGISTRY_DETAILS) + REGISTRY_DETAILS="$2" + shift 2 + ;; + --CONNECTION_DETAILS) + CONNECTION_DETAILS="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Use the assigned variables as needed +echo "Flow to execute: $flow_to_execute" +echo "Deploy environment: $deploy_environment" +echo "Build ID: $build_id" + # Description: # This script generates docker image for Prompt flow deployment - set -e # fail on error # read values from llmops_config.json related to given environment @@ -26,11 +60,10 @@ if [[ -n "$selected_object" ]]; then con_object=$(jq ".webapp_endpoint[] | select(.ENV_NAME == \"$env_name\")" "$deploy_config") read -r -a connection_names <<< "$(echo "$con_object" | jq -r '.CONNECTION_NAMES | join(" ")')" - echo $connection_names result_string="" for name in "${connection_names[@]}"; do - api_key=$(echo $CONNECTION_DETAILS | jq -r --arg name "$name" '.[] | select(.name == $name) | .api_key') + api_key=$(echo ${CONNECTION_DETAILS} | jq -r --arg name "$name" '.[] | select(.name == $name) | .api_key') uppercase_name=$(echo "$name" | tr '[:lower:]' '[:upper:]') modified_name="${uppercase_name}_API_KEY" result_string+=" -e $modified_name=$api_key" @@ -43,27 +76,24 @@ if [[ -n "$selected_object" ]]; then sleep 15 docker ps -a - - chmod +x "./$flow_to_execute/sample-request.json" - + + chmod +x "./$flow_to_execute/sample-request.json" + file_contents=$(<./$flow_to_execute/sample-request.json) echo "$file_contents" - + python -m llmops.common.deployment.test_local_flow \ --flow_to_execute $flow_to_execute - REGISTRY_NAME=$(echo "$con_object" | jq -r '.REGISTRY_NAME') + registry_name=$(echo "${REGISTRY_DETAILS}" | jq -r '.[0].registry_name') + registry_server=$(echo "${REGISTRY_DETAILS}" | jq -r '.[0].registry_server') + registry_username=$(echo "${REGISTRY_DETAILS}" | jq -r '.[0].registry_username') + registry_password=$(echo "${REGISTRY_DETAILS}" | jq -r '.[0].registry_password') - registry_object=$(echo $REGISTRY_DETAILS | jq -r --arg name "$REGISTRY_NAME" '.[] | select(.registry_name == $name)') - registry_server=$(echo "$registry_object" | jq -r '.registry_server') - registry_username=$(echo "$registry_object" | jq -r '.registry_username') - registry_password=$(echo "$registry_object" | jq -r '.registry_password') + docker login "$registry_server" -u "$registry_username" --password-stdin <<< "$registry_password" + docker tag localpf "$registry_server"/"$flow_to_execute"_"$deploy_environment":"$build_id" + docker push "$registry_server"/"$flow_to_execute"_"$deploy_environment":"$build_id" - - docker login "$registry_server" -u "$registry_username" --password-stdin <<< "$registry_password" - docker tag localpf "$registry_server"/"$flow_to_execute"_"$deploy_environment":$build_id - docker push "$registry_server"/"$flow_to_execute"_"$deploy_environment":$build_id - else echo "Object in config file not found" - fi + fi \ No newline at end of file diff --git a/named_entity_recognition/flows/standard/requirements.txt b/named_entity_recognition/flows/standard/requirements.txt index 24a615daa..3cd95d693 100644 --- a/named_entity_recognition/flows/standard/requirements.txt +++ b/named_entity_recognition/flows/standard/requirements.txt @@ -1,3 +1,3 @@ promptflow promptflow-tools -promptflow-sdk[builtins] +promptflow-sdk[builtins] \ No newline at end of file