read properties files as UTF-8 against specification

CubBossa · Dec 5, 2023 · 6e2ff18 · 6e2ff18
1 parent 742a3e9
commit 6e2ff18
Show file tree

Hide file tree

Showing 7 changed files with 118 additions and 6 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>de.cubbossa</groupId>
     <artifactId>Translations</artifactId>
-    <version>3.2.0</version>
+    <version>3.2.1</version>
 
     <name>Translations</name>
 

diff --git a/src/main/java/de/cubbossa/translations/MessageStyleImpl.java b/src/main/java/de/cubbossa/translations/MessageStyleImpl.java
@@ -3,6 +3,8 @@
 import net.kyori.adventure.text.minimessage.tag.resolver.TagResolver;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.Objects;
+
 public class MessageStyleImpl implements MessageStyle {
 
     private final String key;
@@ -19,6 +21,19 @@ public MessageStyleImpl(String key, TagResolver resolver, @Nullable String backu
         this.backup = backup;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        MessageStyleImpl that = (MessageStyleImpl) o;
+        return Objects.equals(key, that.key);
+    }
+
+    @Override
+    public int hashCode() {
+        return key.hashCode();
+    }
+
     @Override
     public String getKey() {
         return key;

diff --git a/src/main/java/de/cubbossa/translations/persistent/FileStorage.java b/src/main/java/de/cubbossa/translations/persistent/FileStorage.java
@@ -2,8 +2,14 @@
 
 import org.jetbrains.annotations.Nullable;
 
+import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
 import java.util.Locale;
 
 public abstract class FileStorage {
@@ -56,4 +62,49 @@ File localeFile(Locale locale) {
         }
         return file;
     }
+
+    public Charset detectCharset(File f, Charset[] charsets) {
+        for (Charset charset : charsets) {
+            Charset result = detectCharset(f, charset);
+            if (result != null) {
+                return result;
+            }
+        }
+        return null;
+    }
+
+    private Charset detectCharset(File f, Charset charset) {
+        try {
+            BufferedInputStream input = new BufferedInputStream(new FileInputStream(f));
+
+            CharsetDecoder decoder = charset.newDecoder();
+            decoder.reset();
+
+            byte[] buffer = new byte[512];
+            boolean identified = false;
+            while ((input.read(buffer) != -1) && (!identified)) {
+                identified = identify(buffer, decoder);
+            }
+
+            input.close();
+
+            if (identified) {
+                return charset;
+            } else {
+                return null;
+            }
+
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    private boolean identify(byte[] bytes, CharsetDecoder decoder) {
+        try {
+            decoder.decode(ByteBuffer.wrap(bytes));
+        } catch (CharacterCodingException e) {
+            return false;
+        }
+        return true;
+    }
 }
diff --git a/src/main/java/de/cubbossa/translations/persistent/PropertiesMessageStorage.java b/src/main/java/de/cubbossa/translations/persistent/PropertiesMessageStorage.java
@@ -4,6 +4,8 @@
 import de.cubbossa.translations.MessageCore;
 
 import java.io.*;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -12,6 +14,7 @@
 
 public class PropertiesMessageStorage extends FileStorage implements MessageStorage {
 
+    private static final Charset[] CHARSETS = { StandardCharsets.UTF_8, StandardCharsets.ISO_8859_1 };
 
     public PropertiesMessageStorage(File directory) {
         super(directory, ".properties");
@@ -55,7 +58,7 @@ public Collection<Message> writeMessages(Collection<Message> messages, Locale lo
     private void writeFile(File file, List<Entry> entries) {
         BufferedWriter writer;
         try {
-            writer = new BufferedWriter(new FileWriter(file));
+            writer = new BufferedWriter(new FileWriter(file, detectCharset(file, CHARSETS)));
             for (Entry entry : entries) {
                 if (entry.comment() != null) {
                     if (!(entry.comment().isEmpty() || entry.comment().isBlank())) {
@@ -86,7 +89,7 @@ private Map<String, Entry> readFile(File file) {
         List<String> comments = new ArrayList<>();
 
         int lineIndex = 0;
-        try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+        try (BufferedReader br = new BufferedReader(new FileReader(file, detectCharset(file, CHARSETS)))) {
             String line;
             while ((line = br.readLine()) != null) {
                 lineIndex++;

diff --git a/src/test/java/de/cubbossa/translations/AppTranslationsTest.java b/src/test/java/de/cubbossa/translations/AppTranslationsTest.java
@@ -4,6 +4,8 @@
 import de.cubbossa.translations.util.ComponentSplit;
 import net.kyori.adventure.text.Component;
 import net.kyori.adventure.text.format.NamedTextColor;
+import net.kyori.adventure.text.format.Style;
+import net.kyori.adventure.text.format.TextDecoration;
 import net.kyori.adventure.text.minimessage.tag.resolver.Formatter;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -81,6 +83,23 @@ void globalMsgOverride() {
         assertEquals(empty().append(text("Prefix", NamedTextColor.GREEN)).append(text(" a")), translations.process(a).compact());
     }
 
+    @Test
+    void globalStyle() {
+        Translations global = TranslationsFramework.global();
+        global.getStyleSet().put("negative", Style.style(NamedTextColor.RED));
+        Message m = translations.messageBuilder("a").withDefault("<negative>abc").build();
+        assertEquals(text("abc", NamedTextColor.RED), translations.process(m));
+    }
+
+    @Test
+    void globalStyleOverride() {
+        Translations global = TranslationsFramework.global();
+        global.getStyleSet().put("negative", Style.style(NamedTextColor.RED));
+        translations.getStyleSet().put("negative", Style.style(TextDecoration.UNDERLINED));
+        Message m = translations.messageBuilder("a").withDefault("<negative>abc").build();
+        assertEquals(text("abc").decorate(TextDecoration.UNDERLINED), translations.process(m));
+    }
+
     @Test
     void translate() {
         translations.addMessages(TranslationsFramework.messageFieldsFromClass(this.getClass()));
@@ -91,8 +110,7 @@ void translate() {
         assertEquals(text("Hallo welt - Deutsch"), translations.process(SIMPLE, Locale.forLanguageTag("de-AT")));
 
         assertEquals(
-                text("Embedded: ")
-                        .append(text("Hello worlda", NamedTextColor.RED)),
+                text("Embedded: ").append(text("Hello worlda", NamedTextColor.RED)),
                 translations.process(EMBED, Locale.ENGLISH)
         );
     }

diff --git a/src/test/java/de/cubbossa/translations/persistent/PropertiesMessageStorageTest.java b/src/test/java/de/cubbossa/translations/persistent/PropertiesMessageStorageTest.java
@@ -1,14 +1,21 @@
 package de.cubbossa.translations.persistent;
 
+import de.cubbossa.translations.Message;
 import lombok.SneakyThrows;
+import net.kyori.adventure.text.Component;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
 import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Locale;
 
+import static net.kyori.adventure.text.Component.*;
+import static org.junit.jupiter.api.Assertions.*;
+
 class PropertiesMessageStorageTest extends MessageStorageTest {
 
     public MessageStorage getMessageStorage(File dir) {
@@ -40,6 +47,22 @@ void testRead() {
             t.printStackTrace();
         }
         translations.loadLocales();
-        Assertions.assertNotNull(translations.getMessage("no_perm"));
+        assertNotNull(translations.getMessage("no_perm"));
+    }
+
+    @Test
+    @SneakyThrows
+    void characters() {
+        File dir = new File(gDir, "/TestApp/lang/");
+        dir.mkdirs();
+        Files.copy(Path.of("src/test/resources/character_test_UTF-8.properties"), new File(dir, "en.properties").toPath());
+
+        translations.loadLocales();
+        Message a = translations.getMessage("char_a");
+        assertNotNull(a);
+        assertEquals(text("äöüß"), translations.process(a));
+        Message b = translations.getMessage("char_b");
+        assertNotNull(b);
+        assertEquals(text("ǮǬǱʁʀ"), translations.process(b));
     }
 }
diff --git a/src/test/resources/character_test_UTF-8.properties b/src/test/resources/character_test_UTF-8.properties
@@ -0,0 +1,2 @@
+char_a=äöüß
+char_b=ǮǬǱʁʀ