Skip to content

Commit

Permalink
Merge pull request #7 from pagopa/PAGOPA-2645-tuning-fdr-to-eventhub
Browse files Browse the repository at this point in the history
feat: PAGOPA-2645 test tuning
  • Loading branch information
alessio-acitelli authored Feb 14, 2025
2 parents a273f74 + da1fe56 commit df34653
Show file tree
Hide file tree
Showing 15 changed files with 477 additions and 121 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/deploy_with_github_runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:
cluster_name: ${{ vars.CLUSTER_NAME }}
resource_group: ${{ vars.CLUSTER_RESOURCE_GROUP }}
app_name: ${{ env.APP_NAME }}${{inputs.suffix_name}}
helm_upgrade_options: '--debug --set microservice-chart.azure.workloadIdentityClientId=${{vars.WORKLOAD_IDENTITY_ID}} ${{inputs.helm_options}}'
helm_upgrade_options: '--debug --set microservice-chart.azure.workloadIdentityClientId=${{vars.WORKLOAD_IDENTITY_ID}}'
timeout: '10m0s'

- name: Remove deployment
Expand Down
24 changes: 24 additions & 0 deletions .github/workflows/github_scripts/check_required_labels.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
module.exports = async ({github, context, core}) => {
const comments = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo
});
for (const comment of comments.data) {
if (comment.body.includes('This pull request does not contain a valid label')) {
github.rest.issues.deleteComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: comment.id
})
}
}
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: 'This pull request does not contain a valid label. Please add one of the following labels: `[major, minor, patch, patch, skip]`'
})
core.setFailed('Missing required labels')
}
99 changes: 99 additions & 0 deletions .github/workflows/github_scripts/check_size.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
module.exports = async ({github, context, core}) => {
const additions = context.payload.pull_request.additions || 0
const deletions = context.payload.pull_request.deletions || 0
let changes = additions + deletions;
console.log('additions: ' + additions + ' + deletions: ' + deletions + ' = total changes: ' + changes);

const {IGNORED_FILES, BRANCH_NAME} = process.env
const ignored_files = IGNORED_FILES.trim().split(',').filter(word => word.length > 0);
if (ignored_files.length > 0) {
var ignored = 0
const execSync = require('child_process').execSync;
for (const file of IGNORED_FILES.trim().split(',')) {

const ignored_additions_str = execSync('git --no-pager diff --numstat origin/main..origin/' + BRANCH_NAME + ' | grep ' + file + ' | cut -f 1', {encoding: 'utf-8'})
const ignored_deletions_str = execSync('git --no-pager diff --numstat origin/main..origin/' + BRANCH_NAME + ' | grep ' + file + ' | cut -f 2', {encoding: 'utf-8'})

const ignored_additions = ignored_additions_str.split('\n').map(elem => parseInt(elem || 0)).reduce(
(accumulator, currentValue) => accumulator + currentValue,
0);
const ignored_deletions = ignored_deletions_str.split('\n').map(elem => parseInt(elem || 0)).reduce(
(accumulator, currentValue) => accumulator + currentValue,
0);

ignored += ignored_additions + ignored_deletions;
}
changes -= ignored
console.log('ignored lines: ' + ignored + ' , consider changes: ' + changes);
}

var labels = await github.rest.issues.listLabelsOnIssue({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo
});

if (changes <= 400) {
if (labels.data.find(label => label.name === 'size/large')) {
github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: 'size/large'
})
}
}

if (changes >= 200) {
if (labels.data.find(label => label.name === 'size/small')) {
github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: 'size/small'
})
}
}

var comments = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo
});
for (const comment of comments.data) {
if (comment.body.includes('This PR exceeds the recommended size')) {
github.rest.issues.deleteComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: comment.id
})
}
}

if (changes < 200) {
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ['size/small']
})
}

if (changes > 400) {
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ['size/large']
})

github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: 'This PR exceeds the recommended size of 400 lines. Please make sure you are NOT addressing multiple issues with one PR. _Note this PR might be rejected due to its size._'
})

}
}
10 changes: 5 additions & 5 deletions .identity/00_data.tf
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data "azurerm_key_vault" "key_vault" {


data "azurerm_user_assigned_identity" "identity_cd" {
name = "${local.product}-${local.domain}-01-github-cd-identity"
name = "${local.product}-${local.domain}-job-01-github-cd-identity"
resource_group_name = "${local.product}-identity-rg"
}

Expand Down Expand Up @@ -58,8 +58,8 @@ data "azurerm_key_vault_secret" "key_vault_cucumber_token" {
key_vault_id = data.azurerm_key_vault.key_vault.id
}

# data "azurerm_user_assigned_identity" "workload_identity_clientid" {
# name = "ebollo-workload-identity"
# resource_group_name = "pagopa-${var.env_short}-${var.env}-aks-rg"
# }
data "azurerm_user_assigned_identity" "workload_identity_clientid" {
name = "fdr-workload-identity"
resource_group_name = "pagopa-${var.env_short}-weu-${var.env}-aks-rg"
}

2 changes: 1 addition & 1 deletion .identity/01_github_environment.tf
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ locals {
"CLUSTER_NAME" : local.aks_cluster.name,
"CLUSTER_RESOURCE_GROUP" : local.aks_cluster.resource_group_name,
"NAMESPACE" : local.domain,
# "WORKLOAD_IDENTITY_ID": data.azurerm_user_assigned_identity.workload_identity_clientid.client_id
"WORKLOAD_IDENTITY_ID": data.azurerm_user_assigned_identity.workload_identity_clientid.client_id
}
repo_secrets = {
"SONAR_TOKEN" : data.azurerm_key_vault_secret.key_vault_sonar.value,
Expand Down
6 changes: 0 additions & 6 deletions helm/Chart.lock

This file was deleted.

10 changes: 5 additions & 5 deletions helm/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
apiVersion: v2
name: pagopa-functions-template
description: Microservice description
name: pagopa-fdr-to-event-hub
description: Microservice fdr to event hub
type: application
version: 0.27.0
appVersion: 0.0.2
version: 0.43.0
appVersion: 0.0.2-16-PAGOPA-2645-tuning-fdr-to-eventhub
dependencies:
- name: microservice-chart
version: 1.21.0
version: 7.1.1
repository: "https://pagopa.github.io/aks-microservice-chart-blueprint"
91 changes: 69 additions & 22 deletions helm/values-dev.yaml
Original file line number Diff line number Diff line change
@@ -1,40 +1,65 @@
microservice-chart:
namespace: "fdr"
nameOverride: ""
fullnameOverride: ""
nameOverride: "pagopa-fdr-2-event"
fullnameOverride: "pagopa-fdr-2-event-hub"
image:
repository: ghcr.io/pagopa/pagopa-fdr-2-event-hub
tag: "0.0.2"
tag: "0.0.2-16-PAGOPA-2645-tuning-fdr-to-eventhub"
pullPolicy: Always
# https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script.WebHost/Controllers/HostController.cs
# livenessProbe:
# httpGet:
# path: /info
# port: 80
# initialDelaySeconds: 60
# failureThreshold: 6
# periodSeconds: 10
# readinessProbe:
# httpGet:
# path: /info
# port: 80
# initialDelaySeconds: 60
# failureThreshold: 6
# periodSeconds: 10
livenessProbe:
httpGet:
path: /info
port: 80
initialDelaySeconds: 60
failureThreshold: 6
periodSeconds: 10
handlerType: tcpSocket
tcpSocket:
port: 8080
initialDelaySeconds: 30
periodSeconds: 30
failureThreshold: 10
readinessProbe:
httpGet:
path: /info
port: 80
initialDelaySeconds: 60
failureThreshold: 6
periodSeconds: 10
handlerType: tcpSocket
tcpSocket:
port: 8080
initialDelaySeconds: 30
periodSeconds: 30
failureThreshold: 10
deployment:
create: true
service:
serviceMonitor:
create: true
endpoints:
- interval: 10s #jmx-exporter
targetPort: 12345
path: /metrics
ports:
- 12345 #jmx-exporter
- 8080
service:
type: ClusterIP
port: 80
ports:
- 8080
- 12345 #jmx-exporter
ingress:
create: true
host: "weudev.fdr.internal.dev.platform.pagopa.it"
path: /pagopa-fdr-to-event-hub-service/(.*)
servicePort: 80
serviceAccount:
create: false
annotations: {}
name: ""
name: "fdr-workload-identity"
azure:
workloadIdentityClientId: <workload-identity-client-id-set-automatically-by-gha>
podAnnotations: {}
podSecurityContext:
seccompProfile:
Expand Down Expand Up @@ -68,6 +93,16 @@ microservice-chart:
BLOB_STORAGE_FDR3_CONTAINER: "fdr3-flows"
EVENT_HUB_FLOWTX_NAME: "fdr-qi-flows"
EVENT_HUB_REPORTEDIUV_NAME: "fdr-qi-reported-iuv"
ASPNETCORE_URLS: "http://*:8080"
# AzureFunctionsJobHost__logging__logLevel__default: "Debug"
# AzureFunctionsJobHost__logging__logLevel__Host__Results: "Debug"
# AzureFunctionsJobHost__logging__logLevel__Host__Aggregator: "Debug"
# AzureFunctionsJobHost__logging__logLevel__Function__ProcessFDR1BlobFiles: "Debug"
# AzureFunctionsJobHost__logging__logLevel__Function__ProcessFDR3BlobFiles: "Debug"
FUNCTIONS_SECRETS_PATH: "/tmp/secrets"
envFieldRef:
APP_NAME: "metadata.labels['app.kubernetes.io/instance']"
APP_VERSION: "metadata.labels['app.kubernetes.io/version']"
envSecret:
APPLICATIONINSIGHTS_CONNECTION_STRING: "ai-connection-string"
FDR_SA_CONNECTION_STRING: "fdr-sa-connection-string"
Expand All @@ -78,7 +113,19 @@ microservice-chart:
name: "pagopa-d-fdr-kv"
tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d"
nodeSelector: {}
tolerations: []
affinity: {}
tolerations:
- key: dedicated
operator: Equal
value: "nodo"
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: nodo
operator: In
values:
- "true"
canaryDelivery:
create: false
Loading

0 comments on commit df34653

Please sign in to comment.