From c3a2bcddb5194065d620f49b655db1fd3ace61b2 Mon Sep 17 00:00:00 2001 From: Paul Ebermann Date: Tue, 12 Sep 2023 19:11:24 +0200 Subject: [PATCH] Make lock size required This is one option from #182. --- README.md | 7 ++++--- .../nakadiproducer/NakadiProducerAutoConfiguration.java | 2 +- .../nakadiproducer/BaseMockedExternalCommunicationIT.java | 6 +++++- .../java/org/zalando/nakadiproducer/LockTimeoutIT.java | 3 ++- .../nakadiproducer/NakadiClientContentEncodingIT.java | 1 + .../nakadiproducer/NonNakadiProducerFlywayCallbackIT.java | 7 ++++++- .../org/zalando/nakadiproducer/SubmissionDisabledIT.java | 2 +- .../snapshots/SnapshotEventGenerationWebEndpointIT.java | 1 + 8 files changed, 21 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 39b4533..e3a2802 100644 --- a/README.md +++ b/README.md @@ -422,9 +422,10 @@ There are three properties for customizing this event locking on transmission: considered for transmission. The default is currently 60 seconds but might change in future releases. * **lock-size** (events): Defines the maximum amount of events which are loaded into memory and published in one run - (in one submission per event type). By default, *all* events are loaded into memory. In some future release, this - property will become mandatory. This should be set to a value which is not too high so out-of-memory situations - are avoided. + (in one submission per event type). **This property is mandatory** (even if you disable sending events completely). + This should be set to a value which is low enough so out-of-memory situations + are avoided (depending on your typical event sizes), and high enough to allow the needed throughput. + Using the value 0 will disable the limit completely, but this is not recommended. Example: ```yaml diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java index 65793cf..aa7afb0 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java @@ -150,7 +150,7 @@ public EventLogWriter eventLogWriter(EventLogRepository eventLogRepository, Obje @Bean public EventLogRepository eventLogRepository(NamedParameterJdbcTemplate namedParameterJdbcTemplate, - @Value("${nakadi-producer.lock-size:0}") int lockSize) { + @Value("${nakadi-producer.lock-size}") int lockSize) { return new EventLogRepositoryImpl(namedParameterJdbcTemplate, lockSize); } diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/BaseMockedExternalCommunicationIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/BaseMockedExternalCommunicationIT.java index 82bb15d..855896a 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/BaseMockedExternalCommunicationIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/BaseMockedExternalCommunicationIT.java @@ -8,7 +8,11 @@ @ActiveProfiles("test") @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.MOCK, - properties = { "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false" }, + properties = { + "zalando.team.id:alpha-local-testing", + "nakadi-producer.scheduled-transmission-enabled:false", + "nakadi-producer.lock-size:100" + }, classes = { TestApplication.class, EmbeddedDataSourceConfig.class } ) public abstract class BaseMockedExternalCommunicationIT { diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/LockTimeoutIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/LockTimeoutIT.java index 4e1a668..8cb7958 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/LockTimeoutIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/LockTimeoutIT.java @@ -27,7 +27,8 @@ @SpringBootTest(properties = { "nakadi-producer.scheduled-transmission-enabled:false", "nakadi-producer.lock-duration:300", - "nakadi-producer.lock-duration-buffer:30"}) + "nakadi-producer.lock-duration-buffer:30", + "nakadi-producer.lock-size:100"}) public class LockTimeoutIT extends BaseMockedExternalCommunicationIT { private static final String MY_EVENT_TYPE = "myEventType"; diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiClientContentEncodingIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiClientContentEncodingIT.java index 6f035fc..cb050aa 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiClientContentEncodingIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiClientContentEncodingIT.java @@ -18,6 +18,7 @@ webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = { "nakadi-producer.scheduled-transmission-enabled:false", + "nakadi-producer.lock-size:100", // as we are not defining a mock nakadi client, we need to provide these properties: "nakadi-producer.encoding:ZSTD", "nakadi-producer.nakadi-base-uri:http://nakadi.example.com/", diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java index 1d21665..58ea044 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java @@ -15,7 +15,12 @@ @ActiveProfiles("test") @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.MOCK, - properties = {"zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false", "spring.flyway.enabled:false"}, + properties = { + "zalando.team.id:alpha-local-testing", + "nakadi-producer.scheduled-transmission-enabled:false", + "spring.flyway.enabled:false", + "nakadi-producer.lock-size:100" + }, classes = {TestApplication.class, EmbeddedDataSourceConfig.class} ) public class NonNakadiProducerFlywayCallbackIT { diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/SubmissionDisabledIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/SubmissionDisabledIT.java index 2b150f7..2498193 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/SubmissionDisabledIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/SubmissionDisabledIT.java @@ -21,7 +21,7 @@ // no "test" profile, as this would include the mock client. @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.MOCK, - properties = {"nakadi-producer.submission-enabled:false"}, + properties = {"nakadi-producer.submission-enabled:false", "nakadi-producer.lock-size:100"}, classes = { TestApplication.class, EmbeddedDataSourceConfig.class } ) public class SubmissionDisabledIT { diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java index f8bb8f4..bf4e127 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java @@ -24,6 +24,7 @@ "management.security.enabled=false", "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false", + "nakadi-producer.lock-size:100", "management.endpoints.web.exposure.include:snapshot-event-creation" }, classes = {TestApplication.class, EmbeddedDataSourceConfig.class, SnapshotEventGenerationWebEndpointIT.Config.class}