Merge branch 'develop' into 2021-instrument-x-ray-sdk

gchq · Apr 25, 2024 · 8291331 · 8291331
2 parents e26d632 + 945236f
commit 8291331
Show file tree

Hide file tree

Showing 250 changed files with 2,952 additions and 878 deletions.
diff --git a/.github/config/chunks.yaml b/.github/config/chunks.yaml
@@ -18,7 +18,7 @@ chunks:
   common:
     name: Common
     workflow: chunk-common.yaml
-    modules: [ core, configuration, sketches, parquet, common-job, common-task, build, dynamodb-tools ]
+    modules: [ core, configuration, sketches, parquet, common/common-job, common/common-task, build, common/dynamodb-tools ]
   compaction:
     name: Compaction
     workflow: chunk-compaction.yaml

diff --git a/.github/workflows/chunk-athena.yaml b/.github/workflows/chunk-athena.yaml
@@ -13,14 +13,14 @@ on:
       - 'java/query/query-core/**'
       - 'java/splitter/splitter-core/**'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/.github/workflows/chunk-bulk-import.yaml b/.github/workflows/chunk-bulk-import.yaml
@@ -13,13 +13,13 @@ on:
       - 'java/bulk-import/bulk-import-starter/**'
       - 'java/bulk-import/bulk-import-runner/**'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/configuration/**'
       - 'java/core/**'
 

diff --git a/.github/workflows/chunk-cdk.yaml b/.github/workflows/chunk-cdk.yaml
@@ -21,7 +21,7 @@ on:
       - 'java/bulk-import/bulk-import-starter/**'
       - 'java/clients/**'
       - 'java/ingest/ingest-batcher-submitter/**'
-      - 'java/common-task/**'
+      - 'java/common/common-task/**'
       - 'java/ingest/ingest-batcher-store/**'
       - 'java/bulk-import/bulk-import-common/**'
       - 'java/splitter/splitter-core/**'
@@ -34,15 +34,15 @@ on:
       - 'java/ingest/ingest-runner/**'
       - 'java/compaction/compaction-status-store/**'
       - 'java/compaction/compaction-job-creation/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
       - 'java/compaction/compaction-core/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/.github/workflows/chunk-clients.yaml b/.github/workflows/chunk-clients.yaml
@@ -13,20 +13,20 @@ on:
       - 'java/compaction/compaction-status-store/**'
       - 'java/splitter/splitter-core/**'
       - 'java/ingest/ingest-batcher-store/**'
-      - 'java/common-task/**'
+      - 'java/common/common-task/**'
       - 'java/ingest/ingest-batcher-core/**'
       - 'java/compaction/compaction-core/**'
       - 'java/query/query-runner/**'
       - 'java/query/query-core/**'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/.github/workflows/chunk-common.yaml b/.github/workflows/chunk-common.yaml
@@ -8,14 +8,15 @@ on:
       - 'code-style/checkstyle*.xml'
       - 'code-style/spotbugs*.xml'
       - 'java/pom.xml'
+      - 'java/common/pom.xml'
       - 'java/core/**'
       - 'java/configuration/**'
       - 'java/sketches/**'
       - 'java/parquet/**'
-      - 'java/common-job/**'
-      - 'java/common-task/**'
+      - 'java/common/common-job/**'
+      - 'java/common/common-task/**'
       - 'java/build/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
 
 jobs:
   chunk-workflow:

diff --git a/.github/workflows/chunk-compaction.yaml b/.github/workflows/chunk-compaction.yaml
@@ -18,16 +18,16 @@ on:
       - 'java/compaction/compaction-core/**'
       - 'java/splitter/splitter-core/**'
       - 'java/splitter/splitter-lambda/**'
-      - 'java/common-task/**'
+      - 'java/common/common-task/**'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/.github/workflows/chunk-data.yaml b/.github/workflows/chunk-data.yaml
@@ -11,7 +11,7 @@ on:
       - 'java/metrics/**'
       - 'java/garbage-collector/**'
       - 'java/statestore/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
       - 'java/core/**'

diff --git a/.github/workflows/chunk-ingest-common.yaml b/.github/workflows/chunk-ingest-common.yaml
@@ -16,11 +16,11 @@ on:
       - 'java/ingest/ingest-batcher-store/**'
       - 'java/ingest/ingest-batcher-submitter/**'
       - 'java/ingest/ingest-batcher-job-creator/**'
-      - 'java/common-task/**'
+      - 'java/common/common-task/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/.github/workflows/chunk-ingest-runner.yaml b/.github/workflows/chunk-ingest-runner.yaml
@@ -10,13 +10,13 @@ on:
       - 'java/pom.xml'
       - 'java/ingest/pom.xml'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/configuration/**'
       - 'java/core/**'
 

diff --git a/.github/workflows/chunk-query.yaml b/.github/workflows/chunk-query.yaml
@@ -13,14 +13,14 @@ on:
       - 'java/query/query-runner/**'
       - 'java/query/query-lambda/**'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/.github/workflows/chunk-statestore.yaml b/.github/workflows/chunk-statestore.yaml
@@ -9,7 +9,7 @@ on:
       - 'code-style/spotbugs*.xml'
       - 'java/pom.xml'
       - 'java/statestore/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
       - 'java/core/**'

diff --git a/.github/workflows/chunk-trino.yaml b/.github/workflows/chunk-trino.yaml
@@ -12,14 +12,14 @@ on:
       - 'java/query/query-runner/**'
       - 'java/query/query-core/**'
       - 'java/ingest/ingest-runner/**'
-      - 'java/common-job/**'
+      - 'java/common/common-job/**'
       - 'java/ingest/ingest-status-store/**'
       - 'java/sketches/**'
       - 'java/ingest/ingest-core/**'
       - 'java/statestore/**'
       - 'java/parquet/**'
       - 'java/configuration/**'
-      - 'java/dynamodb-tools/**'
+      - 'java/common/dynamodb-tools/**'
       - 'java/core/**'
 
 jobs:

diff --git a/code-style/checkstyle-idea/checkstyle-suppressions.xml b/code-style/checkstyle-idea/checkstyle-suppressions.xml
@@ -25,10 +25,12 @@
     <suppress files="[\\/]src[\\/]test[\\/].*" id="missingSwitchDefault"/>
     <suppress files="[\\/]src[\\/]test[\\/].*" id="methodLengthMax"/>
     <suppress files="[\\/]src[\\/]test[\\/].*" id="parameterNumber"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*Test$" id="missingJavadocMethod"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*IT$" id="missingJavadocMethod"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*Test$" id="missingJavadocType"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*IT$" id="missingJavadocType"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*TestBase.java$" id="missingJavadocMethod"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*Test.java$" id="missingJavadocMethod"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*IT.java$" id="missingJavadocMethod"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*TestBase.java$" id="missingJavadocType"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*Test.java$" id="missingJavadocType"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*IT.java$" id="missingJavadocType"/>
     <suppress files=".*Application.java" id="hideUtilityClassConstructor"/>
     <suppress files="[\\/]athena[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]athena[\\/].*" id="missingJavadocType"/>
@@ -38,20 +40,14 @@
     <suppress files="[\\/]cdk[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]system-test[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]system-test[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]core[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]core[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]configuration[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]configuration[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]parquet[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]parquet[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]sketches[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]sketches[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]dynamodb-tools[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]dynamodb-tools[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]common-job[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]common-job[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]common-task[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]common-task[\\/].*" id="missingJavadocType"/>
+    <suppress files="[\\/]common[\\/].*" id="missingJavadocMethod"/>
+    <suppress files="[\\/]common[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]clients[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]clients[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]compaction[\\/].*" id="missingJavadocMethod"/>

diff --git a/code-style/checkstyle-idea/checkstyle.xml b/code-style/checkstyle-idea/checkstyle.xml
@@ -86,8 +86,10 @@
         <module name="MissingJavadocMethod">
             <property name="id" value="missingJavadocMethod"/>
             <property name="allowMissingPropertyJavadoc" value="true"/>
-            <property name="allowedAnnotations" value="Override, Test"/>
+            <property name="allowedAnnotations" value="Override"/>
+            <property name="ignoreMethodNamesRegex" value="^builder$|^build$|^toBuilder$"/>
             <property name="scope" value="public"/>
+            <property name="tokens" value="METHOD_DEF,ANNOTATION_FIELD_DEF"/>
         </module>
         <module name="MissingJavadocType">
             <property name="id" value="missingJavadocType"/>

diff --git a/code-style/checkstyle-suppressions.xml b/code-style/checkstyle-suppressions.xml
@@ -25,10 +25,12 @@
     <suppress files="[\\/]src[\\/]test[\\/].*" id="missingSwitchDefault"/>
     <suppress files="[\\/]src[\\/]test[\\/].*" id="methodLengthMax"/>
     <suppress files="[\\/]src[\\/]test[\\/].*" id="parameterNumber"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*Test$" id="missingJavadocMethod"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*IT$" id="missingJavadocMethod"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*Test$" id="missingJavadocType"/>
-    <suppress files="[\\/]src[\\/]test[\\/].*IT$" id="missingJavadocType"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*TestBase.java$" id="missingJavadocMethod"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*Test.java$" id="missingJavadocMethod"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*IT.java$" id="missingJavadocMethod"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*TestBase.java$" id="missingJavadocType"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*Test.java$" id="missingJavadocType"/>
+    <suppress files="[\\/]src[\\/]test[\\/].*IT.java$" id="missingJavadocType"/>
     <suppress files=".*Application.java" id="hideUtilityClassConstructor"/>
     <suppress files="[\\/]athena[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]athena[\\/].*" id="missingJavadocType"/>
@@ -38,20 +40,14 @@
     <suppress files="[\\/]cdk[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]system-test[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]system-test[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]core[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]core[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]configuration[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]configuration[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]parquet[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]parquet[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]sketches[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]sketches[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]dynamodb-tools[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]dynamodb-tools[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]common-job[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]common-job[\\/].*" id="missingJavadocType"/>
-    <suppress files="[\\/]common-task[\\/].*" id="missingJavadocMethod"/>
-    <suppress files="[\\/]common-task[\\/].*" id="missingJavadocType"/>
+    <suppress files="[\\/]common[\\/].*" id="missingJavadocMethod"/>
+    <suppress files="[\\/]common[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]clients[\\/].*" id="missingJavadocMethod"/>
     <suppress files="[\\/]clients[\\/].*" id="missingJavadocType"/>
     <suppress files="[\\/]compaction[\\/].*" id="missingJavadocMethod"/>

diff --git a/code-style/checkstyle.xml b/code-style/checkstyle.xml
@@ -84,8 +84,10 @@
         <module name="MissingJavadocMethod">
             <property name="id" value="missingJavadocMethod"/>
             <property name="allowMissingPropertyJavadoc" value="true"/>
-            <property name="allowedAnnotations" value="Override, Test"/>
+            <property name="allowedAnnotations" value="Override"/>
+            <property name="ignoreMethodNamesRegex" value="^builder$|^build$|^toBuilder$"/>
             <property name="scope" value="public"/>
+            <property name="tokens" value="METHOD_DEF,ANNOTATION_FIELD_DEF"/>
         </module>
         <module name="MissingJavadocType">
             <property name="id" value="missingJavadocType"/>

diff --git a/example/full/instance.properties b/example/full/instance.properties
@@ -84,9 +84,9 @@ sleeper.task.runner.timeout.seconds=900
 # The namespaces for the metrics used in the metrics stack.
 sleeper.metrics.namespace=Sleeper
 
-# The number of tables to calculate metrics for in a single invocation. A separate invocation of the
-# lambda will be made for each batch when there are more tables than the batch size.
-sleeper.metrics.batch.size=5
+# The number of tables to calculate metrics for in a single invocation. This will be the batch size
+# for a lambda as an SQS FIFO event source. This can be a maximum of 10.
+sleeper.metrics.batch.size=1
 
 # Whether to calculate table metrics for offline tables.
 sleeper.metrics.offline.enabled=false
@@ -695,9 +695,9 @@ sleeper.partition.splitting.period.minutes=30
 # controls the maximum number of files that are read in.
 sleeper.partition.splitting.files.maximum=50
 
-# The number of tables to find partitions to split for in a single invocation. A separate invocation
-# of the lambda will be made for each batch when there are more tables than the batch size.
-sleeper.partition.splitting.table.batch.size=5
+# The number of tables to find partitions to split for in a single invocation. This will be the batch
+# size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
+sleeper.partition.splitting.finder.batch.size=1
 
 # The amount of memory in MB for the lambda function used to identify partitions that need to be
 # split.
@@ -725,9 +725,9 @@ sleeper.gc.period.minutes=15
 # The memory in MB for the lambda function used to perform garbage collection.
 sleeper.gc.memory=1024
 
-# The number of tables to perform garbage collection for in a single invocation. A separate invocation
-# of the lambda will be made for each batch when there are more tables than the batch size.
-sleeper.gc.table.batch.size=5
+# The number of tables to perform garbage collection for in a single invocation. This will be the
+# batch size for a lambda as an SQS FIFO event source. This can be a maximum of 10.
+sleeper.gc.table.batch.size=1
 
 # Whether to perform garbage collection for offline tables.
 sleeper.gc.offline.enabled=false

diff --git a/...bulk-import-runner/src/test/java/sleeper/bulkimport/job/runner/BulkImportJobDriverIT.java b/...bulk-import-runner/src/test/java/sleeper/bulkimport/job/runner/BulkImportJobDriverIT.java
@@ -93,7 +93,7 @@
 import static sleeper.configuration.properties.table.TableProperty.TABLE_ID;
 import static sleeper.configuration.properties.table.TableProperty.TABLE_NAME;
 import static sleeper.configuration.testutils.LocalStackAwsV1ClientHelper.buildAwsV1Client;
-import static sleeper.core.record.process.RecordsProcessedSummaryTestData.summary;
+import static sleeper.core.record.process.RecordsProcessedSummaryTestHelper.summary;
 import static sleeper.ingest.job.status.IngestJobStatusTestData.finishedIngestJobWithValidation;
 import static sleeper.ingest.job.status.IngestJobValidatedEvent.ingestJobAccepted;
 import static sleeper.io.parquet.utils.HadoopConfigurationLocalStackUtils.getHadoopConfiguration;

diff --git a/...lk-import-runner/src/test/java/sleeper/bulkimport/job/runner/BulkImportJobDriverTest.java b/...lk-import-runner/src/test/java/sleeper/bulkimport/job/runner/BulkImportJobDriverTest.java
@@ -45,7 +45,7 @@
 import static sleeper.configuration.properties.table.TablePropertiesTestHelper.createTestTableProperties;
 import static sleeper.configuration.properties.table.TableProperty.TABLE_ID;
 import static sleeper.configuration.properties.table.TableProperty.TABLE_NAME;
-import static sleeper.core.record.process.RecordsProcessedSummaryTestData.summary;
+import static sleeper.core.record.process.RecordsProcessedSummaryTestHelper.summary;
 import static sleeper.core.schema.SchemaTestHelper.schemaWithKey;
 import static sleeper.core.statestore.FileReferenceTestData.defaultFileOnRootPartitionWithRecords;
 import static sleeper.ingest.job.status.IngestJobStatusTestData.finishedIngestJobWithValidation;

diff --git a/java/cdk/src/main/java/sleeper/cdk/Utils.java b/java/cdk/src/main/java/sleeper/cdk/Utils.java
@@ -298,9 +298,9 @@ public static String normaliseSize(String size) {
         }
     }
 
-    public static void createAlarmForDlq(Construct scope, String name, String description, Queue dlq, Topic topic) {
+    public static void createAlarmForDlq(Construct scope, String id, String description, Queue dlq, Topic topic) {
         Alarm alarm = Alarm.Builder
-                .create(scope, name)
+                .create(scope, id)
                 .alarmDescription(description)
                 .metric(dlq.metricApproximateNumberOfMessagesVisible()
                         .with(MetricOptions.builder().statistic("Sum").period(Duration.seconds(60)).build()))