From e1f82833be83e61f2959fd50487c326a56dc0162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 20 Dec 2024 16:50:18 +0100 Subject: [PATCH 1/8] Rename all_all_sh_components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- src/org/mbed/tls/jenkins/BranchInfo.groovy | 4 ++-- vars/common.groovy | 8 ++++---- vars/gen_jobs.groovy | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/org/mbed/tls/jenkins/BranchInfo.groovy b/src/org/mbed/tls/jenkins/BranchInfo.groovy index 7384a852b..2712a79ea 100755 --- a/src/org/mbed/tls/jenkins/BranchInfo.groovy +++ b/src/org/mbed/tls/jenkins/BranchInfo.groovy @@ -6,7 +6,7 @@ class BranchInfo { /** Map from component name to chosen platform to run it, or to null * if no platform has been chosen yet. */ - public Map all_all_sh_components + public Map mbed_tls_all_sh_components /** Whether scripts/min_requirements.py is available. Older branches don't * have it, so they only get what's hard-coded in the docker files on Linux, @@ -35,7 +35,7 @@ class BranchInfo { String coverage_details BranchInfo() { - this.all_all_sh_components = [:] + this.mbed_tls_all_sh_components = [:] this.has_min_requirements = false this.python_requirements_override_content = '' this.python_requirements_override_file = '' diff --git a/vars/common.groovy b/vars/common.groovy index 1fca4a9c0..548bb46be 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -326,14 +326,14 @@ List get_branch_information(Collection branches) { infos.each { BranchInfo info -> String prefix = infos.size() > 1 ? "$info.branch-" : '' - info.all_all_sh_components = results[prefix + 'all-platforms'] + info.mbed_tls_all_sh_components = results[prefix + 'all-platforms'] linux_platforms.reverseEach { platform -> - info.all_all_sh_components << results[prefix + platform] + info.mbed_tls_all_sh_components << results[prefix + platform] } if (env.JOB_TYPE == 'PR') { // Do not run release components in PR jobs - info.all_all_sh_components = info.all_all_sh_components.findAll { + info.mbed_tls_all_sh_components = info.mbed_tls_all_sh_components.findAll { component, platform -> !component.startsWith('release') } } @@ -343,7 +343,7 @@ List get_branch_information(Collection branches) { void check_every_all_sh_component_will_be_run(Collection infos) { Map> untested_all_sh_components = infos.collectEntries { info -> - def components = info.all_all_sh_components.findResults { + def components = info.mbed_tls_all_sh_components.findResults { name, platform -> platform ? null : name } return components ? [(info.branch): components] : [:] diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 2d712a330..396896487 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -654,7 +654,7 @@ def gen_release_jobs(BranchInfo info, String label_prefix='', boolean run_exampl } if (env.RUN_ALL_SH == "true") { - info.all_all_sh_components.each({component, platform -> + info.mbed_tls_all_sh_components.each({ component, platform -> jobs = jobs + gen_all_sh_jobs(info, platform, component, label_prefix) }) } From 31dd5b6a4218b79fa6b199b53f40d613629535f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 20 Dec 2024 17:26:01 +0100 Subject: [PATCH 2/8] Collect information about tf-psa-crypto components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- src/org/mbed/tls/jenkins/BranchInfo.groovy | 5 + vars/common.groovy | 122 +++++++++++++-------- 2 files changed, 81 insertions(+), 46 deletions(-) diff --git a/src/org/mbed/tls/jenkins/BranchInfo.groovy b/src/org/mbed/tls/jenkins/BranchInfo.groovy index 2712a79ea..b839b347b 100755 --- a/src/org/mbed/tls/jenkins/BranchInfo.groovy +++ b/src/org/mbed/tls/jenkins/BranchInfo.groovy @@ -8,6 +8,10 @@ class BranchInfo { * if no platform has been chosen yet. */ public Map mbed_tls_all_sh_components + /** Map from component name to chosen platform to run it, or to null + * if no platform has been chosen yet. Empty if we are not running any tests.*/ + public Map tf_psa_crypto_all_sh_components + /** Whether scripts/min_requirements.py is available. Older branches don't * have it, so they only get what's hard-coded in the docker files on Linux, * and bare python on other platforms. */ @@ -36,6 +40,7 @@ class BranchInfo { BranchInfo() { this.mbed_tls_all_sh_components = [:] + this.tf_psa_crypto_all_sh_components = [:] this.has_min_requirements = false this.python_requirements_override_content = '' this.python_requirements_override_file = '' diff --git a/vars/common.groovy b/vars/common.groovy index 548bb46be..4b0d5300d 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -228,6 +228,12 @@ List get_branch_information(Collection branches) { List infos = [] Map jobs = [:] + def repo_roots = ['tls': '.'] + + if (env.RUN_TF_PSA_CRYPTO_ALL_SH == 'true') { + repo_roots['tf-psa-crypto'] = 'tf-psa-crypto' + } + branches.each { String branch -> BranchInfo info = new BranchInfo() info.branch = branch @@ -256,26 +262,33 @@ List get_branch_information(Collection branches) { String platform = linux_platforms[0] get_docker_image(platform) - def all_sh_help = sh( - script: docker_script( - platform, "./tests/scripts/all.sh", "--help" - ), - returnStdout: true - ) - if (all_sh_help.contains('list-components')) { - def all = sh( - script: docker_script( - platform, "./tests/scripts/all.sh", - "--list-all-components" - ), - returnStdout: true - ).trim().split('\n') - echo "All all.sh components: ${all.join(" ")}" - return all.collectEntries { element -> - return [(element): null] + return repo_roots.collectEntries { repo, root -> + dir(root) { + if (!fileExists('./tests/scripts/all.sh')) { + return [(repo): [:]] + } + def all_sh_help = sh( + script: docker_script( + platform, "./tests/scripts/all.sh", "--help" + ), + returnStdout: true + ) + if (all_sh_help.contains('list-components')) { + def all = sh( + script: docker_script( + platform, "./tests/scripts/all.sh", + "--list-all-components" + ), + returnStdout: true + ).trim().split('\n') + echo "All all.sh components: ${all.join(" ")}" + return [(repo): all.collectEntries { element -> + return [(element): null] + }] + } else { + error('Pre Test Checks failed: Base branch out of date. Please rebase') + } } - } else { - error('Pre Test Checks failed: Base branch out of date. Please rebase') } } finally { deleteDir() @@ -292,25 +305,36 @@ List get_branch_information(Collection branches) { checkout_repo.checkout_tls_repo(branch) } get_docker_image(platform) - def all_sh_help = sh( - script: docker_script( - platform, "./tests/scripts/all.sh", "--help" - ), - returnStdout: true - ) - if (all_sh_help.contains('list-components')) { - def available = sh( - script: docker_script( - platform, "./tests/scripts/all.sh", "--list-components" - ), - returnStdout: true - ).trim().split('\n') - echo "Available all.sh components on ${platform}: ${available.join(" ")}" - return available.collectEntries { element -> - return [(element): platform] + return repo_roots.collectEntries { repo, root -> + // Only run tf-psa-crypto tests on the first branch + if (repo == 'tf-psa-crypto' && branch != branches[0]) { + return [(repo): [:]] + } + dir(root) { + if (!fileExists('./tests/scripts/all.sh')) { + return [(repo): [:]] + } + def all_sh_help = sh( + script: docker_script( + platform, "./tests/scripts/all.sh", "--help" + ), + returnStdout: true + ) + if (all_sh_help.contains('list-components')) { + def available = sh( + script: docker_script( + platform, "./tests/scripts/all.sh", "--list-components" + ), + returnStdout: true + ).trim().split('\n') + echo "Available all.sh components on ${platform}: ${available.join(" ")}" + return [(repo): available.collectEntries { element -> + return [(element): platform] + }] + } else { + error('Pre Test Checks failed: Base branch out of date. Please rebase') + } } - } else { - error('Pre Test Checks failed: Base branch out of date. Please rebase') } } finally { deleteDir() @@ -321,22 +345,28 @@ List get_branch_information(Collection branches) { } jobs.failFast = true - def results = (Map>) parallel(jobs) + def results = (Map>>) parallel(jobs) infos.each { BranchInfo info -> String prefix = infos.size() > 1 ? "$info.branch-" : '' - info.mbed_tls_all_sh_components = results[prefix + 'all-platforms'] - linux_platforms.reverseEach { platform -> - info.mbed_tls_all_sh_components << results[prefix + platform] - } + Map> repo_components = repo_roots.keySet().collectEntries {repo -> + def components = results[prefix + 'all-platforms'][repo] + linux_platforms.reverseEach { platform -> + components << results[prefix + platform][repo] + } - if (env.JOB_TYPE == 'PR') { - // Do not run release components in PR jobs - info.mbed_tls_all_sh_components = info.mbed_tls_all_sh_components.findAll { - component, platform -> !component.startsWith('release') + if (env.JOB_TYPE == 'PR') { + // Do not run release components in PR jobs + components = components.findAll { + component, platform -> !component.startsWith('release') + } } + return [(repo): components] } + + info.mbed_tls_all_sh_components = repo_components['tls'] + info.tf_psa_crypto_all_sh_components = repo_components['tf-psa-crypto'] ?: [:] } return infos } From 84f843595e5b52d54729aaeb1599decdbc8aad3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 20 Dec 2024 17:35:46 +0100 Subject: [PATCH 3/8] Break out submodule checkout into separate methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- vars/checkout_repo.groovy | 49 +++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/vars/checkout_repo.groovy b/vars/checkout_repo.groovy index 9d2be2782..819a81720 100644 --- a/vars/checkout_repo.groovy +++ b/vars/checkout_repo.groovy @@ -60,6 +60,30 @@ Map checkout_report_errors(scm_config) { } } +void checkout_framework_repo() { + if (env.TARGET_REPO == 'framework' && env.CHECKOUT_METHOD == 'scm') { + checkout_report_errors(scm) + } else if (env.FRAMEWORK_REPO && env.FRAMEWORK_BRANCH) { + checkout_report_errors(parametrized_repo(env.FRAMEWORK_REPO, env.FRAMEWORK_BRANCH)) + } else { + echo 'Using default framework version' + } +} + +void checkout_tf_psa_crypto_repo() { + if (env.TARGET_REPO == 'tf-psa-crypto' && env.CHECKOUT_METHOD == 'scm') { + checkout_report_errors(scm) + } else if (env.TF_PSA_CRYPTO_REPO && env.TF_PSA_CRYPTO_BRANCH) { + checkout_report_errors(parametrized_repo(env.TF_PSA_CRYPTO_REPO, env.TF_PSA_CRYPTO_BRANCH)) + } else { + echo 'Using default tf-psa-crypto version' + } + + dir('framework') { + checkout_framework_repo() + } +} + Map checkout_tls_repo(String branch) { def scm_config if (env.TARGET_REPO == 'tls' && env.CHECKOUT_METHOD == 'scm') { @@ -76,30 +100,11 @@ Map checkout_tls_repo(String branch) { def result = checkout_report_errors(scm_config) dir('tf-psa-crypto') { - if (env.TARGET_REPO == 'tf-psa-crypto' && env.CHECKOUT_METHOD == 'scm') { - checkout_report_errors(scm) - } else if (env.TF_PSA_CRYPTO_REPO && env.TF_PSA_CRYPTO_BRANCH) { - checkout_report_errors(parametrized_repo(env.TF_PSA_CRYPTO_REPO, env.TF_PSA_CRYPTO_BRANCH)) - } else { - echo 'Using default tf-psa-crypto version' - } + checkout_tf_psa_crypto_repo() } - def framework_dirs = ['framework', 'tf-psa-crypto/framework'] - if (env.TARGET_REPO == 'framework' && env.CHECKOUT_METHOD == 'scm') { - framework_dirs.each { framework_dir -> - dir(framework_dir) { - checkout_report_errors(scm) - } - } - } else if (env.FRAMEWORK_REPO && env.FRAMEWORK_BRANCH) { - framework_dirs.each { framework_dir -> - dir(framework_dir) { - checkout_report_errors(parametrized_repo(env.FRAMEWORK_REPO, env.FRAMEWORK_BRANCH)) - } - } - } else { - echo 'Using default framework version' + dir('framework') { + checkout_framework_repo() } // After the clone, replicate it in the local config, so it is effective when running inside docker From 5de7135c22d88420f343d77046292b25f1ca6055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 20 Dec 2024 17:47:11 +0100 Subject: [PATCH 4/8] Test tf-psa-crypto components without Mbed TLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- vars/gen_jobs.groovy | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 396896487..0a9ed884f 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -183,7 +183,7 @@ fi } } -def gen_all_sh_jobs(BranchInfo info, platform, component, label_prefix='') { +def gen_all_sh_jobs(BranchInfo info, String platform, String component, String label_prefix='', String repo='tls') { def shorthands = [ "arm-compilers-amd64": "armcc", "ubuntu-16.04-amd64": "u16", @@ -198,7 +198,7 @@ def gen_all_sh_jobs(BranchInfo info, platform, component, label_prefix='') { ] /* Default to the full platform hame is a shorthand is not found */ def shortplat = shorthands.getOrDefault(platform, platform) - def job_name = "${label_prefix}all_${shortplat}-${component}" + def job_name = "${label_prefix}${repo == 'tls' ? '' : "${repo}_"}all_${shortplat}-${component}" def outcome_file = "${job_name.replace((char) '/', (char) '_')}-outcome.csv" def use_docker = platform_has_docker(platform) def extra_setup_code = '' @@ -245,7 +245,7 @@ echo >&2 'Note: "clang" will run /usr/bin/clang -Wno-error=c11-extensions' ''' } - if (info.has_min_requirements) { + if (repo == 'tls' && info.has_min_requirements) { extra_setup_code += """ scripts/min_requirements.py --user ${info.python_requirements_override_file} """ @@ -258,6 +258,16 @@ scripts/min_requirements.py --user ${info.python_requirements_override_file} common.get_docker_image(platform) } dir('src') { + switch(repo) { + case 'tls': + checkout_repo.checkout_tls_repo(info) + break + case 'tf-psa-crypto': + checkout_repo.checkout_tf_psa_crypto_repo() + break + default: + error("Invalid repo: $repo") + } checkout_repo.checkout_tls_repo(info) writeFile file: 'steps.sh', text: """\ #!/bin/sh @@ -668,6 +678,12 @@ def gen_release_jobs(BranchInfo info, String label_prefix='', boolean run_exampl } } + if (env.RUN_TF_PSA_CRYPTO_ALL_SH == "true") { + info.tf_psa_crypto_all_sh_components.each({ component, platform -> + jobs = jobs + gen_all_sh_jobs(info, platform, component, label_prefix) + }) + } + if (env.RUN_WINDOWS_TEST == "true") { jobs = jobs + gen_windows_jobs(info, label_prefix) } From 4c2a1ac9622657be79029f794d35008421eeb2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 13 Jan 2025 02:54:53 +0100 Subject: [PATCH 5/8] Run tf-psa-crypto all.sh in PR jobs for that repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- vars/environ.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vars/environ.groovy b/vars/environ.groovy index 7deb5a4f4..446dc8359 100644 --- a/vars/environ.groovy +++ b/vars/environ.groovy @@ -34,8 +34,12 @@ void set_pr_environment(String target_repo, boolean is_production) { env.JOB_TYPE = 'PR' env.TARGET_REPO = target_repo if (is_production) { - if (target_repo in ['framework', 'tf-psa-crypto']) { - env.MBED_TLS_REPO = 'git@github.com:Mbed-TLS/mbedtls.git' + switch (target_repo) { + case 'tf-psa-crypto': + env.RUN_TF_PSA_CRYPTO_ALL_SH = 'true' + // fallthrough + case 'framework': + env.MBED_TLS_REPO = 'git@github.com:Mbed-TLS/mbedtls.git' } set_common_pr_production_environment() } else { From 5054935459053cd263691d2e2981e3ad3d8465a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 13 Jan 2025 02:56:34 +0100 Subject: [PATCH 6/8] Update wording of log message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- vars/common.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/common.groovy b/vars/common.groovy index 4b0d5300d..35165cca1 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -281,7 +281,7 @@ List get_branch_information(Collection branches) { ), returnStdout: true ).trim().split('\n') - echo "All all.sh components: ${all.join(" ")}" + echo "all.sh components: ${all.join(" ")}" return [(repo): all.collectEntries { element -> return [(element): null] }] From cb1450f12b39f585fad8a7a3f2497c1e2bc4c90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 13 Jan 2025 02:58:35 +0100 Subject: [PATCH 7/8] Remove left-over checkout of TLS repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- vars/gen_jobs.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 0a9ed884f..b672f2c94 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -268,7 +268,6 @@ scripts/min_requirements.py --user ${info.python_requirements_override_file} default: error("Invalid repo: $repo") } - checkout_repo.checkout_tls_repo(info) writeFile file: 'steps.sh', text: """\ #!/bin/sh set -eux From fcce535ef8c585fa7a42fa7ef08ef214992df6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 13 Jan 2025 03:01:41 +0100 Subject: [PATCH 8/8] Pass correct repo value for tf-psa-crypto tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- vars/gen_jobs.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index b672f2c94..130d442f6 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -679,7 +679,7 @@ def gen_release_jobs(BranchInfo info, String label_prefix='', boolean run_exampl if (env.RUN_TF_PSA_CRYPTO_ALL_SH == "true") { info.tf_psa_crypto_all_sh_components.each({ component, platform -> - jobs = jobs + gen_all_sh_jobs(info, platform, component, label_prefix) + jobs = jobs + gen_all_sh_jobs(info, platform, component, label_prefix, 'tf-psa-crypto') }) }