From c6624965a673cd2506dc41e06acc735b87388ca2 Mon Sep 17 00:00:00 2001
From: Dave Rodgman <dave.rodgman@arm.com>
Date: Sun, 24 Dec 2023 13:18:29 +0000
Subject: [PATCH 1/5] speed up outcome file compression

Signed-off-by: Dave Rodgman <dave.rodgman@arm.com>
---
 vars/analysis.groovy | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vars/analysis.groovy b/vars/analysis.groovy
index 83c94fac8..2e43fabfa 100644
--- a/vars/analysis.groovy
+++ b/vars/analysis.groovy
@@ -237,16 +237,16 @@ def process_outcomes() {
         deleteDir()
     }
 
-    // The complete outcome file is ~14MB compressed as I write.
+    // The complete outcome file is 2.1GB uncompressed / 56MB compressed as I write.
     // Often we just want the failures, so make an artifact with just those.
     // Only produce a failure file if there was a failing job (otherwise
     // we'd just waste time creating an empty file).
     if (gen_jobs.failed_builds) {
         sh '''\
-awk -F';' '$5 == "FAIL"' outcomes.csv >"failures.csv"
+grep ';FAIL;' outcomes.csv >"failures.csv"
 # Compress the failure list if it is large (for some value of large)
 if [ "$(wc -c <failures.csv)" -gt 99999 ]; then
-    xz failures.csv
+    xz -0 -T8 failures.csv
 fi
 '''
     }
@@ -258,7 +258,7 @@ fi
             }
         }
     } finally {
-        sh 'xz outcomes.csv'
+        sh 'xz -0 -T8 outcomes.csv'
         archiveArtifacts(artifacts: 'outcomes.csv.xz, failures.csv*',
                          fingerprint: true,
                          allowEmptyArchive: true)

From 1fcbc384d3e7e010ebc55c44b72705fce4ea0906 Mon Sep 17 00:00:00 2001
From: Dave Rodgman <dave.rodgman@arm.com>
Date: Tue, 2 Jan 2024 12:53:05 +0000
Subject: [PATCH 2/5] Add comment about grep false positives

Signed-off-by: Dave Rodgman <dave.rodgman@arm.com>
---
 vars/analysis.groovy | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vars/analysis.groovy b/vars/analysis.groovy
index 2e43fabfa..9435c90bf 100644
--- a/vars/analysis.groovy
+++ b/vars/analysis.groovy
@@ -241,6 +241,9 @@ def process_outcomes() {
     // Often we just want the failures, so make an artifact with just those.
     // Only produce a failure file if there was a failing job (otherwise
     // we'd just waste time creating an empty file).
+    //
+    // Note that grep ';FAIL;' could pick up false positives, if another field such
+    // as test description or test suite was "FAIL".
     if (gen_jobs.failed_builds) {
         sh '''\
 grep ';FAIL;' outcomes.csv >"failures.csv"

From 828cacae89ec9546058a3c9dd746d03dcbaf7787 Mon Sep 17 00:00:00 2001
From: Dave Rodgman <dave.rodgman@arm.com>
Date: Tue, 2 Jan 2024 12:53:39 +0000
Subject: [PATCH 3/5] set locale when running grep for robustness

Signed-off-by: Dave Rodgman <dave.rodgman@arm.com>
---
 vars/analysis.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vars/analysis.groovy b/vars/analysis.groovy
index 9435c90bf..bcd348a40 100644
--- a/vars/analysis.groovy
+++ b/vars/analysis.groovy
@@ -249,7 +249,7 @@ def process_outcomes() {
 grep ';FAIL;' outcomes.csv >"failures.csv"
 # Compress the failure list if it is large (for some value of large)
 if [ "$(wc -c <failures.csv)" -gt 99999 ]; then
-    xz -0 -T8 failures.csv
+    LC_ALL=C xz -0 -T8 failures.csv
 fi
 '''
     }

From 86fb584b9f187882f7ff61da95ad76ae3aa28b5a Mon Sep 17 00:00:00 2001
From: Dave Rodgman <dave.rodgman@arm.com>
Date: Tue, 2 Jan 2024 15:59:40 +0000
Subject: [PATCH 4/5] Fix LC_ALL in wrong place

Signed-off-by: Dave Rodgman <dave.rodgman@arm.com>
---
 vars/analysis.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vars/analysis.groovy b/vars/analysis.groovy
index bcd348a40..9df8b0ad5 100644
--- a/vars/analysis.groovy
+++ b/vars/analysis.groovy
@@ -246,10 +246,10 @@ def process_outcomes() {
     // as test description or test suite was "FAIL".
     if (gen_jobs.failed_builds) {
         sh '''\
-grep ';FAIL;' outcomes.csv >"failures.csv"
+LC_ALL=C grep ';FAIL;' outcomes.csv >"failures.csv"
 # Compress the failure list if it is large (for some value of large)
 if [ "$(wc -c <failures.csv)" -gt 99999 ]; then
-    LC_ALL=C xz -0 -T8 failures.csv
+    xz -0 -T8 failures.csv
 fi
 '''
     }

From ba120af51c4d2429e8f6a3b1d1c72b84d6b4b8e9 Mon Sep 17 00:00:00 2001
From: Dave Rodgman <dave.rodgman@arm.com>
Date: Wed, 3 Jan 2024 11:02:26 +0000
Subject: [PATCH 5/5] allow xz to pick num threads

Signed-off-by: Dave Rodgman <dave.rodgman@arm.com>
---
 vars/analysis.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vars/analysis.groovy b/vars/analysis.groovy
index 9df8b0ad5..986d5f2d4 100644
--- a/vars/analysis.groovy
+++ b/vars/analysis.groovy
@@ -249,7 +249,7 @@ def process_outcomes() {
 LC_ALL=C grep ';FAIL;' outcomes.csv >"failures.csv"
 # Compress the failure list if it is large (for some value of large)
 if [ "$(wc -c <failures.csv)" -gt 99999 ]; then
-    xz -0 -T8 failures.csv
+    xz -0 -T0 failures.csv
 fi
 '''
     }
@@ -261,7 +261,7 @@ fi
             }
         }
     } finally {
-        sh 'xz -0 -T8 outcomes.csv'
+        sh 'xz -0 -T0 outcomes.csv'
         archiveArtifacts(artifacts: 'outcomes.csv.xz, failures.csv*',
                          fingerprint: true,
                          allowEmptyArchive: true)