From 72c86045c05b5caaba95c6a6cf5a65b9ef6c1785 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 18:37:24 +0200 Subject: [PATCH 1/9] [ci] Switch from Temurin to Liberica as it still supports macOS JDK 8 Closes https://github.com/eXist-db/exist/pull/5304 --- .github/workflows/ci-deploy.yml | 2 +- .github/workflows/ci-test.yml | 12 +++--------- .github/workflows/ci-xqts.yml | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-deploy.yml b/.github/workflows/ci-deploy.yml index 713f690259f..7b01ad93dfa 100644 --- a/.github/workflows/ci-deploy.yml +++ b/.github/workflows/ci-deploy.yml @@ -13,7 +13,7 @@ jobs: - name: Set up JDK 8 uses: actions/setup-java@v4 with: - distribution: temurin + distribution: liberica java-version: '8' - name: Make buildkit default uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 5c79eca5d33..9c366c941a8 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - distribution: temurin + distribution: liberica java-version: ${{ env.DEV_JDK }} cache: 'maven' - run: mvn -V -B license:check @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - distribution: temurin + distribution: liberica java-version: ${{ env.DEV_JDK }} cache: 'maven' - name: OWASP dependency check @@ -40,19 +40,13 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] jvm: ['8'] - exclude: - - os: macOS-latest - jvm: '8' - include: - - os: macOS-latest - jvm: '11' runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: temurin + distribution: liberica java-version: ${{ matrix.jvm }} cache: 'maven' - name: Install Maven Daemon diff --git a/.github/workflows/ci-xqts.yml b/.github/workflows/ci-xqts.yml index 5ac15c77617..f7a42da20f5 100644 --- a/.github/workflows/ci-xqts.yml +++ b/.github/workflows/ci-xqts.yml @@ -12,7 +12,7 @@ jobs: - name: Set up JDK 8 uses: actions/setup-java@v4 with: - distribution: temurin + distribution: liberica java-version: '8' - name: Cache Maven packages uses: actions/cache@v4 From e23d7f8fc349f3b19e17f113041538454290bfa1 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 20:03:32 +0200 Subject: [PATCH 2/9] [bugfix] Make sure Journal Recovery progress bars reach 100% --- .../storage/recovery/RecoveryManager.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java index 99b074d685a..bcd9e797ea4 100644 --- a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java +++ b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java @@ -120,10 +120,10 @@ public boolean recover() throws LogException { Lsn lastLsn = Lsn.LSN_INVALID; Loggable next; try { - final ProgressBar progress = new ProgressBar("Scanning journal ", FileUtils.sizeQuietly(last)); + final long lastSize = FileUtils.sizeQuietly(last); + final ProgressBar scanProgressBar = new ProgressBar("Scanning journal ", lastSize); while ((next = reader.nextEntry()) != null) { // LOG.debug(next.dump()); - progress.set(next.getLsn().getOffset()); if (next.getLogType() == LogEntryTypes.TXN_START) { // new transaction starts: add it to the transactions table txnsStarted.put(next.getTransactionId(), next); @@ -135,7 +135,10 @@ public boolean recover() throws LogException { lastCheckpoint = (Checkpoint) next; } lastLsn = next.getLsn(); + + scanProgressBar.set(next.getLsn().getOffset()); } + scanProgressBar.set(lastSize); // 100% } catch (final LogException e) { if (LOG.isDebugEnabled()) { LOG.debug("Caught exception while reading log", e); @@ -250,10 +253,11 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader if (LOG.isInfoEnabled()) { LOG.info("First pass: redoing {} transactions...", txnCount);} - final ProgressBar progress = new ProgressBar("Redo ", FileUtils.sizeQuietly(last)); Loggable next = null; int redoCnt = 0; try { + final long lastSize = FileUtils.sizeQuietly(last); + final ProgressBar redoProgressBar = new ProgressBar("Redo ", lastSize); while ((next = reader.nextEntry()) != null) { SanityCheck.ASSERT(next.getLogType() != LogEntryTypes.CHECKPOINT, "Found a checkpoint during recovery run! This should not ever happen."); @@ -271,10 +275,13 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader // LOG.debug("Redo: " + next.dump()); // redo the log entry next.redo(); - progress.set(next.getLsn().getOffset()); - if (next.getLsn().equals(lastLsn)) - {break;} // last readable entry reached. Stop here. + redoProgressBar.set(next.getLsn().getOffset()); + if (next.getLsn().equals(lastLsn)) { + // last readable entry reached. Stop here. + break; + } } + redoProgressBar.set(lastSize); // 100% done } catch (final Exception e) { LOG.error("Exception caught while redoing transactions. Aborting recovery to avoid possible damage. " + "Before starting again, make sure to run a check via the emergency export tool.", e); From a6610f4cf2dca1fd1d52906400d3e806f46b6ee2 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 20:04:19 +0200 Subject: [PATCH 3/9] [bugfix] Add missing Undo progress bar for Journal Recovery --- .../java/org/exist/storage/recovery/RecoveryManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java index bcd9e797ea4..c5f82a30512 100644 --- a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java +++ b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java @@ -301,6 +301,8 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader if (runningTxns.size() > 0) { // do a reverse scan of the log, undoing all uncommitted transactions try { + final long lastSize = FileUtils.sizeQuietly(last); + final ProgressBar undoProgressBar = new ProgressBar("Undo ", lastSize); while((next = reader.previousEntry()) != null) { if (next.getLogType() == LogEntryTypes.TXN_START) { if (runningTxns.get(next.getTransactionId()) != null) { @@ -321,7 +323,10 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader // LOG.debug("Undo: " + next.dump()); next.undo(); } + + undoProgressBar.set(lastSize - next.getLsn().getOffset()); } + undoProgressBar.set(lastSize); // 100% done } catch (final Exception e) { LOG.warn("Exception caught while undoing dirty transactions. Remaining transactions to be undone: {}. Aborting recovery to avoid possible damage. Before starting again, make sure to run a check via the emergency export tool.", runningTxns.size(), e); if (next != null) From e8f7135ae2e3d3baf1b07e762e099b0545ff7293 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 20:04:47 +0200 Subject: [PATCH 4/9] [ignore] Small code cleanup --- .../java/org/exist/storage/recovery/RecoveryManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java index c5f82a30512..6d44a3a8dfc 100644 --- a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java +++ b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java @@ -149,7 +149,7 @@ public boolean recover() throws LogException { // if the last checkpoint record is not the last record in the file // we need a recovery. if ((lastCheckpoint == null || !lastCheckpoint.getLsn().equals(lastLsn)) && - txnsStarted.size() > 0) { + !txnsStarted.isEmpty()) { LOG.info("Dirty transactions: {}", txnsStarted.size()); // starting recovery: reposition the log reader to the last checkpoint if (lastCheckpoint == null) { @@ -298,7 +298,7 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader { LOG.info("Second pass: undoing dirty transactions. Uncommitted transactions: {}", runningTxns.size());} // see if there are uncommitted transactions pending - if (runningTxns.size() > 0) { + if (!runningTxns.isEmpty()) { // do a reverse scan of the log, undoing all uncommitted transactions try { final long lastSize = FileUtils.sizeQuietly(last); @@ -307,9 +307,10 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader if (next.getLogType() == LogEntryTypes.TXN_START) { if (runningTxns.get(next.getTransactionId()) != null) { runningTxns.remove(next.getTransactionId()); - if (runningTxns.size() == 0) + if (runningTxns.isEmpty()) { // all dirty transactions undone - {break;} + break; + } } } else if (next.getLogType() == LogEntryTypes.TXN_COMMIT) { // ignore already committed transaction From 9356ee28fe7c36819f467b3a20ab1f3c3357444d Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 20:12:47 +0200 Subject: [PATCH 5/9] [feature] Add the System Property 'exist.recovery.progressbar.hide' to hide the Journal Recovery progress bars --- .../storage/recovery/RecoveryManager.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java index 6d44a3a8dfc..1822c9e0439 100644 --- a/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java +++ b/exist-core/src/main/java/org/exist/storage/recovery/RecoveryManager.java @@ -45,6 +45,8 @@ import com.evolvedbinary.j8fu.function.SupplierE; import org.exist.util.sanity.SanityCheck; +import javax.annotation.Nullable; + /** * Database recovery. This class is used once during startup to check * if the database is in a consistent state. If not, the class attempts to recover @@ -59,11 +61,13 @@ public class RecoveryManager { private final DBBroker broker; private final JournalRecoveryAccessor journalRecovery; private final boolean restartOnError; + private final boolean hideProgressBar; public RecoveryManager(final DBBroker broker, final JournalManager journalManager, final boolean restartOnError) { this.broker = broker; this.journalRecovery = journalManager.getRecoveryAccessor(this); this.restartOnError = restartOnError; + this.hideProgressBar = Boolean.getBoolean("exist.recovery.progressbar.hide"); } /** @@ -121,7 +125,7 @@ public boolean recover() throws LogException { Loggable next; try { final long lastSize = FileUtils.sizeQuietly(last); - final ProgressBar scanProgressBar = new ProgressBar("Scanning journal ", lastSize); + @Nullable final ProgressBar scanProgressBar = hideProgressBar ? null : new ProgressBar("Scanning journal ", lastSize); while ((next = reader.nextEntry()) != null) { // LOG.debug(next.dump()); if (next.getLogType() == LogEntryTypes.TXN_START) { @@ -136,9 +140,14 @@ public boolean recover() throws LogException { } lastLsn = next.getLsn(); - scanProgressBar.set(next.getLsn().getOffset()); + if (scanProgressBar != null) { + scanProgressBar.set(next.getLsn().getOffset()); + } + } + + if (scanProgressBar != null) { + scanProgressBar.set(lastSize); // 100% } - scanProgressBar.set(lastSize); // 100% } catch (final LogException e) { if (LOG.isDebugEnabled()) { LOG.debug("Caught exception while reading log", e); @@ -257,7 +266,7 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader int redoCnt = 0; try { final long lastSize = FileUtils.sizeQuietly(last); - final ProgressBar redoProgressBar = new ProgressBar("Redo ", lastSize); + @Nullable final ProgressBar redoProgressBar = hideProgressBar ? null : new ProgressBar("Redo ", lastSize); while ((next = reader.nextEntry()) != null) { SanityCheck.ASSERT(next.getLogType() != LogEntryTypes.CHECKPOINT, "Found a checkpoint during recovery run! This should not ever happen."); @@ -275,13 +284,20 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader // LOG.debug("Redo: " + next.dump()); // redo the log entry next.redo(); - redoProgressBar.set(next.getLsn().getOffset()); + + if (redoProgressBar != null) { + redoProgressBar.set(next.getLsn().getOffset()); + } + if (next.getLsn().equals(lastLsn)) { // last readable entry reached. Stop here. break; } } - redoProgressBar.set(lastSize); // 100% done + + if (redoProgressBar != null) { + redoProgressBar.set(lastSize); // 100% done + } } catch (final Exception e) { LOG.error("Exception caught while redoing transactions. Aborting recovery to avoid possible damage. " + "Before starting again, make sure to run a check via the emergency export tool.", e); @@ -302,8 +318,8 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader // do a reverse scan of the log, undoing all uncommitted transactions try { final long lastSize = FileUtils.sizeQuietly(last); - final ProgressBar undoProgressBar = new ProgressBar("Undo ", lastSize); - while((next = reader.previousEntry()) != null) { + final ProgressBar undoProgressBar = hideProgressBar ? null : new ProgressBar("Undo ", lastSize); + while ((next = reader.previousEntry()) != null) { if (next.getLogType() == LogEntryTypes.TXN_START) { if (runningTxns.get(next.getTransactionId()) != null) { runningTxns.remove(next.getTransactionId()); @@ -325,9 +341,14 @@ private void doRecovery(final int txnCount, final Path last, final JournalReader next.undo(); } - undoProgressBar.set(lastSize - next.getLsn().getOffset()); + if (undoProgressBar != null) { + undoProgressBar.set(lastSize - next.getLsn().getOffset()); + } + } + + if (undoProgressBar != null) { + undoProgressBar.set(lastSize); // 100% done } - undoProgressBar.set(lastSize); // 100% done } catch (final Exception e) { LOG.warn("Exception caught while undoing dirty transactions. Remaining transactions to be undone: {}. Aborting recovery to avoid possible damage. Before starting again, make sure to run a check via the emergency export tool.", runningTxns.size(), e); if (next != null) From 1da0b7c437c2b4ec0c950c76e184d58ab713fdf6 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 20:14:27 +0200 Subject: [PATCH 6/9] [refactor] Hide the Journal Recovery progress bars when running the test suite so as not to pollute the output --- exist-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exist-core/pom.xml b/exist-core/pom.xml index 2664c859842..a8266cbb4f1 100644 --- a/exist-core/pom.xml +++ b/exist-core/pom.xml @@ -1097,7 +1097,7 @@ The BaseX Team. The original license statement is also included below.]]> - @{jacocoArgLine} -Dfile.encoding=${project.build.sourceEncoding} + @{jacocoArgLine} -Dfile.encoding=${project.build.sourceEncoding} -Dexist.recovery.progressbar.hide=true ${project.basedir}/../exist-jetty-config/target/classes/org/exist/jetty ${project.build.testOutputDirectory}/conf.xml From 4b6fbceeec0d1c076566246fbc82d55af965d46c Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 21:07:06 +0200 Subject: [PATCH 7/9] [ci] Use latest version of mvnd --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 9c366c941a8..d92c2a71b5e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -53,7 +53,7 @@ jobs: id: install-mvnd uses: ./.github/actions/install-mvnd with: - version: '1.0.1' + version: '1.0.2' file-version-suffix: '' cache: 'true' - name: Maven Build From e013cd01664df3e46b61b7a201177ece269aa83a Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 2 Oct 2024 21:15:15 +0200 Subject: [PATCH 8/9] [ci] Update GitHub Actions plugins --- .github/actions/install-mvnd/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-mvnd/action.yml b/.github/actions/install-mvnd/action.yml index 9769860c218..a0dc3bdf1ff 100644 --- a/.github/actions/install-mvnd/action.yml +++ b/.github/actions/install-mvnd/action.yml @@ -48,7 +48,7 @@ runs: - name: Cache mvnd if: inputs.cache == 'true' id: cache-mvnd - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip From 320cf57e6e4333f91fdfe2ae1e005619b6d880be Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 3 Oct 2024 12:16:44 +0200 Subject: [PATCH 9/9] [bugfix] As these are plugins log a Warning and not an Error if the plugin is unavailable --- .../src/main/java/org/exist/plugin/PluginsManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exist-core/src/main/java/org/exist/plugin/PluginsManagerImpl.java b/exist-core/src/main/java/org/exist/plugin/PluginsManagerImpl.java index 94d57e4a079..3dbbbbd3f58 100644 --- a/exist-core/src/main/java/org/exist/plugin/PluginsManagerImpl.java +++ b/exist-core/src/main/java/org/exist/plugin/PluginsManagerImpl.java @@ -241,7 +241,7 @@ public void addPlugin(final String className) { // NOTE: must set interrupted flag Thread.currentThread().interrupt(); } - LOG.error(e); + LOG.warn(e); } }