Merge remote-tracking branch 'origin/master' into release

.github/workflows/maven-matrix.yml

@@ -34,44 +34,24 @@ jobs:
         with:
           distribution: zulu
           java-version: ${{ matrix.java }}
-#      - name: Cache local Maven repository
-#        uses: actions/cache@v2
-#        with:
-#          path: |
-#            ~/.m2/repository
-#            !~/.m2/repository/com/senacor
-#            ~/.m2/wrapper
-#          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-#          restore-keys: |
-#            ${{ runner.os }}-maven-
       - name: Build and test with Maven
         run: $MVN_CMD install
 
   build-and-test-with-es-version:
     strategy:
       matrix:
-        elasticsearchVersion: [ "8.6.0", "8.5.3", "8.4.3", "8.3.3", "8.2.3", "8.1.3", "8.0.1", "7.17.8", "7.16.3", "7.15.2", "7.14.2",
+        elasticsearchVersion: [ "8.6.1", "8.5.3", "8.4.3", "8.3.3", "8.2.3", "8.1.3", "8.0.1", "7.17.9", "7.16.3", "7.15.2", "7.14.2",
                                 "7.13.4", "7.12.1", "7.11.2", "7.10.2", "7.9.3", "7.8.1", "7.7.1", "7.6.2", "7.5.2" ]
       fail-fast: false
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
           distribution: zulu
-          java-version: 11
-#      - name: Cache local Maven repository
-#        uses: actions/cache@v3
-#        with:
-#          path: |
-#            ~/.m2/repository
-#            !~/.m2/repository/com/senacor
-#            ~/.m2/wrapper
-#          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-#          restore-keys: |
-#            ${{ runner.os }}-maven-
+          java-version: 17
       - name: Build and test with Maven
         run: $MVN_CMD install -DskipTests
       - name: elasticsearch test version ${{ matrix.elasticsearchVersion }}
@@ -98,16 +78,6 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
-#      - name: Cache local Maven repository
-#        uses: actions/cache@v3
-#        with:
-#          path: |
-#            ~/.m2/repository
-#            !~/.m2/repository/com/senacor
-#            ~/.m2/wrapper
-#          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-#          restore-keys: |
-#            ${{ runner.os }}-maven-
       - name: Release to maven central
         env:
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
@@ -139,16 +109,6 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
-#      - name: Cache local Maven repository
-#        uses: actions/cache@v3
-#        with:
-#          path: |
-#            ~/.m2/repository
-#            !~/.m2/repository/com/senacor
-#            ~/.m2/wrapper
-#          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-#          restore-keys: |
-#            ${{ runner.os }}-maven-
       - name: Release to maven central
         env:
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

.github/workflows/quality.yml

@@ -30,16 +30,6 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
-#      - name: Cache local Maven repository
-#        uses: actions/cache@v3
-#        with:
-#          path: |
-#            ~/.m2/repository
-#            !~/.m2/repository/com/senacor
-#            ~/.m2/wrapper
-#          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-#          restore-keys: |
-#            ${{ runner.os }}-maven-
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.mvn/wrapper/maven-wrapper.properties

@@ -14,5 +14,5 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.5/apache-maven-3.8.5-bin.zip
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.0/apache-maven-3.9.0-bin.zip
 wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2021 Andreas Keefer
+   Copyright 2023 Andreas Keefer
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -202,7 +202,7 @@
 
 ----------
 
-Copyright (c) 2021 Andreas Keefer
+Copyright (c) 2023 Andreas Keefer
 
 "Anti 996" License Version 1.0 (Draft)
 

README.md

@@ -22,7 +22,7 @@ Successful executed migration scripts will not be executed again!
 
 - tested on Java 8, 11, 17, 18 and 19
 - runs on Spring-Boot 2.1, 2.2, 2.3, 2.4, 2.5, 2.6 and 2.7 (and of course without Spring-Boot)
-- runs on Elasticsearch version 7.5.x - 8.4.x
+- runs on Elasticsearch version 7.5.x - 8.6.x
 - runs on Opensearch version 1.x and 2.x
 - highly configurable (e.g. location(s) of your migration files, migration files format pattern)
 - placeholder substitution in migration scripts
@@ -31,11 +31,11 @@ Successful executed migration scripts will not be executed again!
 - ready to use default configuration
 - line comments in migration files
 
-| Compatibility                    | Spring Boot                  | Elasticsearch        | Opensearch |
-|----------------------------------|------------------------------|----------------------|------------|
-| elasticsearch-evolution >= 0.4.0 | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6 | 7.5.x - 8.4.x        | 1.x - 2.x  |
-| elasticsearch-evolution 0.3.x    | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6 | 7.5.x - 7.17.x       |            |
-| elasticsearch-evolution 0.2.x    | 1.5, 2.0, 2.1, 2.2, 2.3, 2.4 | 7.0.x - 7.4.x, 6.8.x |            |
+| Compatibility                    | Spring Boot                       | Elasticsearch        | Opensearch |
+|----------------------------------|-----------------------------------|----------------------|------------|
+| elasticsearch-evolution >= 0.4.0 | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7 | 7.5.x - 8.6.x        | 1.x - 2.x  |
+| elasticsearch-evolution 0.3.x    | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7 | 7.5.x - 7.17.x       |            |
+| elasticsearch-evolution 0.2.x    | 1.5, 2.0, 2.1, 2.2, 2.3, 2.4      | 7.0.x - 7.4.x, 6.8.x |            |
 
 NOTE: When you run on Java 11 and using spring-boot 2.2 or 2.3 and you hit [this issue](https://github.com/ronmamo/reflections/issues/279), you have 2 options: 
 
@@ -52,7 +52,7 @@ First add the latest version of Elasticsearch-Evolution spring boot starter as a
 <dependency>
     <groupId>com.senacor.elasticsearch.evolution</groupId>
     <artifactId>spring-boot-starter-elasticsearch-evolution</artifactId>
-    <version>0.4.0</version>
+    <version>0.4.1</version>
 </dependency>
 ```
 
@@ -76,7 +76,7 @@ First add the latest version of Elasticsearch-Evolution core as a dependency:
 <dependency>
     <groupId>com.senacor.elasticsearch.evolution</groupId>
     <artifactId>elasticsearch-evolution-core</artifactId>
-    <version>0.4.0</version>
+    <version>0.4.1</version>
 </dependency>
 ```
 
@@ -194,6 +194,7 @@ Elasticsearch-Evolution can be configured to your needs:
 -   **historyMaxQuerySize** (default=1000): The maximum query size while validating already executed scripts. This query size have to be higher than the total count of your migration scripts.
 -   **validateOnMigrate** (default=true): Whether to fail when a previously applied migration script has been modified after it was applied.
 -   **baselineVersion** (default=1.0): Version to use as a baseline. versions lower than it will not be applied.
+-   **lineSeparator** (default=\n): Line separator, used only temporary between reading raw migration file line-by-line and parsing it later. Only needed for backward compatibility / checksum stability! Should be one of `\n`, `\r` or `\r\n`
 
 ### 5.1 Spring Boot
 
@@ -288,15 +289,22 @@ ElasticsearchEvolution.configure()
 
 ## 6 changelog
 
-### v0.4.1-SNAPSHOT
+### v0.4.2-SNAPSHOT
+
+- bugfix ([#182](https://github.com/senacor/elasticsearch-evolution/issues/182)): checksum calculation was based on system dependent line separators which lead to different checksums on different operating systems (e.g. windows vs linux). The default is now `\n`. For backward compatibility you can set other line separator via `lineSeparator` config property.
+- version updates (spring-boot 2.7.8)
+- spring boot 3 compatibility + tests
+- added Opensearch 2.5 compatibility tests
+
+### v0.4.1
 
 - Optimization: Don't acquire lock if no scripts need to be executed ([#172](https://github.com/senacor/elasticsearch-evolution/issues/172))
 - Previously applied migration scripts are now checked for modifications and rejected if they've been modified after they were applied. The old behaviour can be restored by setting the new configuration parameter `validateOnMigrate` to false (default: true) ([#155](https://github.com/senacor/elasticsearch-evolution/issues/155))
 - version updates (spring-boot 2.7.7)
 - added java 19 compatibility tests
 - added spring boot 2.7 compatibility tests
 - added Elasticsearch 8.6, 8.5, 8.4, 8.3, and 8,2 compatibility test
-- added Opensearch 2.3, 2.2, 2.1 and 2.0 compatibility tests
+- added Opensearch 2.4, 2.3, 2.2, 2.1 and 2.0 compatibility tests
 - It is now possible to set a `baselineVersion` to skip migrations with versions lower than the defined `baselineVersion` ([#164](https://github.com/senacor/elasticsearch-evolution/issues/164))
 
 ### v0.4.0

elasticsearch-evolution-core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.senacor.elasticsearch.evolution</groupId>
         <artifactId>elasticsearch-evolution-parent</artifactId>
-        <version>0.4.1</version>
+        <version>0.4.2</version>
         <relativePath>../</relativePath>
     </parent>
     <artifactId>elasticsearch-evolution-core</artifactId>

elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/ElasticsearchEvolution.java

@@ -149,7 +149,8 @@ protected MigrationScriptReader createMigrationScriptReader() {
                 getConfig().getLocations(),
                 getConfig().getEncoding(),
                 getConfig().getEsMigrationPrefix(),
-                getConfig().getEsMigrationSuffixes());
+                getConfig().getEsMigrationSuffixes(),
+                getConfig().getLineSeparator());
     }
 
     protected HistoryRepository createHistoryRepository() {

elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/api/config/ElasticsearchEvolutionConfig.java

@@ -36,6 +36,17 @@ public class ElasticsearchEvolutionConfig {
      */
     private Charset encoding = StandardCharsets.UTF_8;
 
+    /**
+     * Line separator, used only temporary between reading raw migration file line-by-line and parsing it later.
+     * Only needed for backward compatibility / checksum stability!
+     * <p>
+     * Should be one of
+     * - '\n' (LF - Linux/Unix/OS X)
+     * - '\r' (CR - Classic MAC OS)
+     * - '\r\n' (CRLF - Windows)
+     */
+    private String lineSeparator = "\n";
+
     /**
      * This content type will be used as default if no contentType header is specified in the header section of a migration script.
      * If no charset is defined, the {@link #encoding} charset is used.
@@ -172,6 +183,15 @@ public ElasticsearchEvolutionConfig setEncoding(Charset encoding) {
         return this;
     }
 
+    public String getLineSeparator() {
+        return lineSeparator;
+    }
+
+    public ElasticsearchEvolutionConfig setLineSeparator(String lineSeparator) {
+        this.lineSeparator = lineSeparator;
+        return this;
+    }
+
     public String getDefaultContentType() {
         return defaultContentType;
     }
@@ -277,6 +297,7 @@ public String toString() {
                 "enabled=" + enabled +
                 ", locations=" + locations +
                 ", encoding=" + encoding +
+                ", lineSeparator='" + lineSeparator.replace("\n", "\\n").replace("\r", "\\r") + '\'' +
                 ", defaultContentType='" + defaultContentType + '\'' +
                 ", esMigrationPrefix='" + esMigrationPrefix + '\'' +
                 ", esMigrationSuffixes=" + esMigrationSuffixes +

elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/internal/migration/input/MigrationScriptReaderImpl.java

@@ -45,22 +45,26 @@ public class MigrationScriptReaderImpl implements MigrationScriptReader {
     private final Charset encoding;
     private final String esMigrationPrefix;
     private final List<String> esMigrationSuffixes;
+    private final String lineSeparator;
 
     /**
      * @param locations               Locations of migrations scripts, e.g classpath:es/migration or file:/home/migration
      * @param encoding                migrations scripts encoding
      * @param esMigrationFilePrefix   File name prefix for ES migrations.
      * @param esMigrationFileSuffixes File name suffix for ES migrations.
+     * @param lineSeparator           Line separator. should be '\n' per default and only something else for backward compatibility / hash stability
      */
 
     public MigrationScriptReaderImpl(List<String> locations,
                                      Charset encoding,
                                      String esMigrationFilePrefix,
-                                     List<String> esMigrationFileSuffixes) {
+                                     List<String> esMigrationFileSuffixes,
+                                     String lineSeparator) {
         this.locations = locations;
         this.encoding = encoding;
         this.esMigrationPrefix = esMigrationFilePrefix;
         this.esMigrationSuffixes = esMigrationFileSuffixes;
+        this.lineSeparator = lineSeparator;
     }
 
     /**
@@ -162,7 +166,8 @@ private Stream<RawMigrationScript> readScriptsFromClassPath(String location) {
 
     private Stream<RawMigrationScript> read(BufferedReader reader, String filename) {
         String content = reader.lines()
-                .collect(Collectors.joining(System.lineSeparator()));
+                // use static line separator ('\n' per default) to get predictable and system independent checksum later
+                .collect(Collectors.joining(lineSeparator));
         if (content.isEmpty()) {
             return Stream.empty();
         }