Merge pull request #253 from SeeSharpSoft/fb_auto_detect

Auto detect value separator

Author: SeeSharpSoft

CHANGELOG

@@ -1,3 +1,8 @@
+2.14.4
+Feb 14, 2021
+
+NEW: Auto detect value separator (by count)
+
 2.14.3
 Oct 10, 2020
 

README.md

@@ -132,9 +132,16 @@ The preferred editor usage can be switched between "Text Editor first", "Table E
 
 The following separators are currently supported: **,** (Comma), **;** (Semicolon), **:** (Colon), **|** (Pipe) and **↹** (Tab)
 
-_Default Value Separator_ defines which separator is used as standard for each newly opened CSV file. The separator character can be changed for each file individually in its editors context menu.
+_Default Value Separator_ defines which separator is used as standard for each newly created or opened CSV file.
+The separator character can be changed for each file individually in its editors context menu.
 
-This option has no effect on TSV/PSV files, the separator is pre-defined by their file- and language-type.
+###### Auto Detect
+
+The value separator of a newly opened CSV file can be detected automatically based on the number of predefined separator occurrences.
+
+If _Auto Detect_ is enabled (default), the _Default Value Separator_ setting is only taken into account for newly created files.
+
+**Note:** This option has no effect on TSV/PSV files, the separator is pre-defined by their file- and language-type.
 
 ##### Default Escape Character
 

build.gradle

@@ -24,11 +24,11 @@ jacocoTestReport {
 }
 
 group 'net.seesharpsoft.intellij.plugins'
-version '2.14.3'
+version '2.14.4'
 
 apply plugin: 'java'
-sourceCompatibility = javaVersion
-targetCompatibility = javaTargetVersion
+project.sourceCompatibility = JavaVersion.VERSION_11
+project.targetCompatibility = JavaVersion.VERSION_11
 tasks.withType(JavaCompile) { options.encoding = 'UTF-8' }
 repositories {
     mavenCentral()
@@ -60,8 +60,8 @@ sourceSets {
 apply plugin: 'idea'
 idea {
     project {
-        jdkName = javaVersion
-        languageLevel = javaVersion
+        jdkName = JavaVersion.VERSION_11
+        languageLevel = JavaVersion.VERSION_11
         vcs = 'Git'
     }
     module {

gradle.properties

@@ -3,5 +3,3 @@
 # https://www.jetbrains.com/intellij-repository/snapshots
 
 name='CSV Plugin'
-javaVersion=11
-javaTargetVersion=11

src/main/java/net/seesharpsoft/intellij/plugins/csv/components/CsvFileAttributes.java

@@ -5,17 +5,22 @@
 import com.intellij.openapi.components.ServiceManager;
 import com.intellij.openapi.components.State;
 import com.intellij.openapi.components.Storage;
+import com.intellij.openapi.editor.Document;
+import com.intellij.openapi.fileEditor.FileDocumentManager;
 import com.intellij.openapi.fileTypes.LanguageFileType;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiFile;
 import com.intellij.util.xmlb.XmlSerializerUtil;
 import com.intellij.util.xmlb.annotations.OptionTag;
+import net.seesharpsoft.commons.collection.Pair;
 import net.seesharpsoft.intellij.plugins.csv.*;
 import net.seesharpsoft.intellij.plugins.csv.settings.CsvEditorSettings;
+import org.apache.commons.lang.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -87,12 +92,18 @@ public boolean canChangeValueSeparator(@NotNull PsiFile psiFile) {
         return language.isKindOf(CsvLanguage.INSTANCE) && !(language instanceof CsvSeparatorHolder);
     }
 
+    private void setFileSeparator(@NotNull Project project, @NotNull VirtualFile virtualFile, @NotNull CsvValueSeparator separator) {
+        Attribute attribute = getFileAttribute(project, virtualFile, true);
+        if (attribute != null) {
+            attribute.separator = separator;
+        }
+    }
+
     public void setFileSeparator(@NotNull PsiFile psiFile, @NotNull CsvValueSeparator separator) {
         if (!canChangeValueSeparator(psiFile)) {
             return;
         }
-        Attribute attribute = getFileAttribute(psiFile.getProject(), psiFile.getOriginalFile().getVirtualFile(), true);
-        attribute.separator = separator;
+        setFileSeparator(psiFile.getProject(), psiFile.getOriginalFile().getVirtualFile(), separator);
     }
 
     public void resetValueSeparator(@NotNull PsiFile psiFile) {
@@ -105,18 +116,51 @@ public void resetValueSeparator(@NotNull PsiFile psiFile) {
         }
     }
 
+    private @NotNull
+    CsvValueSeparator autoDetectOrGetDefaultValueSeparator(Project project, VirtualFile virtualFile) {
+        return CsvEditorSettings.getInstance().isAutoDetectValueSeparator() ?
+                autoDetectSeparator(project, virtualFile) :
+                CsvEditorSettings.getInstance().getDefaultValueSeparator();
+    }
+
+    private @NotNull
+    CsvValueSeparator autoDetectSeparator(Project project, VirtualFile virtualFile) {
+        Document document = FileDocumentManager.getInstance().getDocument(virtualFile);
+        final String text = document == null ? "" : document.getText();
+        Pair<CsvValueSeparator, Integer> separatorWithCount =
+                Arrays.stream(CsvValueSeparator.values())
+                        // count
+                        .map(separator -> {
+                            String character = separator.getCharacter();
+                            return Pair.of(separator, StringUtils.countMatches(text, character));
+                        })
+                        // ignore non-matched separators
+                        .filter(p -> p.getSecond() > 0)
+                        // get the one with most hits
+                        .max((p1, p2) -> p1.getSecond() - p2.getSecond())
+                        // failsafe (e.g. empty document)
+                        .orElse(null);
+
+        CsvValueSeparator valueSeparator = separatorWithCount != null ?
+                separatorWithCount.getFirst() :
+                CsvEditorSettings.getInstance().getDefaultValueSeparator();
+
+        setFileSeparator(project, virtualFile, valueSeparator);
+        return valueSeparator;
+    }
+
     public @NotNull
     CsvValueSeparator getValueSeparator(Project project, VirtualFile virtualFile) {
         if (project == null || virtualFile == null || !(virtualFile.getFileType() instanceof LanguageFileType)) {
             return CsvEditorSettings.getInstance().getDefaultValueSeparator();
         }
         Language language = ((LanguageFileType) virtualFile.getFileType()).getLanguage();
         if (language instanceof CsvSeparatorHolder) {
-                return ((CsvSeparatorHolder) language).getSeparator();
+            return ((CsvSeparatorHolder) language).getSeparator();
         }
         Attribute attribute = getFileAttribute(project, virtualFile);
         return attribute == null || attribute.separator == null ?
-                CsvEditorSettings.getInstance().getDefaultValueSeparator() :
+                autoDetectOrGetDefaultValueSeparator(project, virtualFile) :
                 attribute.separator;
     }
 
@@ -127,7 +171,9 @@ public boolean hasValueSeparatorAttribute(@NotNull Project project, @NotNull Vir
 
     public void setEscapeCharacter(@NotNull PsiFile psiFile, @NotNull CsvEscapeCharacter escapeCharacter) {
         Attribute attribute = getFileAttribute(psiFile.getProject(), psiFile.getOriginalFile().getVirtualFile(), true);
-        attribute.escapeCharacter = escapeCharacter;
+        if (attribute != null) {
+            attribute.escapeCharacter = escapeCharacter;
+        }
     }
 
     public void resetEscapeSeparator(@NotNull PsiFile psiFile) {

src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettings.java

@@ -84,6 +84,7 @@ public static final class OptionSet {
         public boolean KEEP_TRAILING_SPACES = false;
         public String COMMENT_INDICATOR = COMMENT_INDICATOR_DEFAULT;
         public ValueColoring VALUE_COLORING = ValueColoring.RAINBOW;
+        public boolean AUTO_DETECT_VALUE_SEPARATOR = true;
 
         public OptionSet() {
             EditorSettingsExternalizable editorSettingsExternalizable = EditorSettingsExternalizable.getInstance();
@@ -312,6 +313,14 @@ public void setHeaderRowFixed(boolean headerRowFixed) {
         getState().TABLE_HEADER_ROW_FIXED = headerRowFixed;
     }
 
+    public boolean isAutoDetectValueSeparator() {
+        return getState().AUTO_DETECT_VALUE_SEPARATOR;
+    }
+
+    public void setAutoDetectValueSeparator(boolean autoDetectValueSeparator) {
+        getState().AUTO_DETECT_VALUE_SEPARATOR = autoDetectValueSeparator;
+    }
+
     public boolean checkCurrentPluginVersion(String actualVersion) {
         if (!actualVersion.equals(getState().CURRENT_PLUGIN_VERSION)) {
             getState().CURRENT_PLUGIN_VERSION = actualVersion;

src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.form

@@ -343,7 +343,7 @@
           </component>
         </children>
       </grid>
-      <grid id="29095" layout-manager="GridLayoutManager" row-count="1" column-count="3" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">
+      <grid id="29095" layout-manager="GridLayoutManager" row-count="1" column-count="4" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">
         <margin top="0" left="0" bottom="0" right="0"/>
         <constraints>
           <grid row="1" column="0" row-span="1" col-span="1" vsize-policy="3" hsize-policy="3" anchor="0" fill="3" indent="0" use-parent-layout="false"/>
@@ -363,7 +363,7 @@
           </component>
           <hspacer id="f2aa1">
             <constraints>
-              <grid row="0" column="2" row-span="1" col-span="1" vsize-policy="1" hsize-policy="6" anchor="0" fill="1" indent="0" use-parent-layout="false"/>
+              <grid row="0" column="3" row-span="1" col-span="1" vsize-policy="1" hsize-policy="6" anchor="0" fill="1" indent="0" use-parent-layout="false"/>
             </constraints>
           </hspacer>
           <component id="65f7" class="javax.swing.JComboBox" binding="comboValueSeparator" custom-create="true">
@@ -372,6 +372,14 @@
             </constraints>
             <properties/>
           </component>
+          <component id="aed54" class="javax.swing.JCheckBox" binding="cbAutoDetectSeparator">
+            <constraints>
+              <grid row="0" column="2" row-span="1" col-span="1" vsize-policy="0" hsize-policy="3" anchor="8" fill="0" indent="0" use-parent-layout="false"/>
+            </constraints>
+            <properties>
+              <text value="auto detect initially"/>
+            </properties>
+          </component>
         </children>
       </grid>
       <grid id="7eea1" layout-manager="GridLayoutManager" row-count="1" column-count="3" same-size-horizontally="false" same-size-vertically="false" hgap="-1" vgap="-1">

src/main/java/net/seesharpsoft/intellij/plugins/csv/settings/CsvEditorSettingsProvider.java

@@ -43,6 +43,7 @@ public class CsvEditorSettingsProvider implements EditorOptionsProvider {
     private JTextField tfCommentIndicator;
     private JComboBox comboValueColoring;
     private JCheckBox cbHeaderRowFixed;
+    private JCheckBox cbAutoDetectSeparator;
 
     @NotNull
     @Override
@@ -94,7 +95,8 @@ public boolean isModified() {
                 isModified(cbKeepTrailingWhitespaces, csvEditorSettings.getKeepTrailingSpaces()) ||
                 isModified(tfCommentIndicator, csvEditorSettings.getCommentIndicator()) ||
                 !Objects.equals(comboValueColoring.getSelectedItem(), csvEditorSettings.getValueColoring()) ||
-                isModified(cbHeaderRowFixed, csvEditorSettings.isHeaderRowFixed());
+                isModified(cbHeaderRowFixed, csvEditorSettings.isHeaderRowFixed()) ||
+                isModified(cbAutoDetectSeparator, csvEditorSettings.isAutoDetectValueSeparator());
     }
 
     @Override
@@ -120,6 +122,7 @@ public void reset() {
         tfCommentIndicator.setText(csvEditorSettings.getCommentIndicator());
         comboValueColoring.setSelectedItem(csvEditorSettings.getValueColoring());
         cbHeaderRowFixed.setSelected(csvEditorSettings.isHeaderRowFixed());
+        cbAutoDetectSeparator.setSelected(csvEditorSettings.isAutoDetectValueSeparator());
     }
 
     @Override
@@ -145,6 +148,7 @@ public void apply() throws ConfigurationException {
         csvEditorSettings.setCommentIndicator(tfCommentIndicator.getText());
         csvEditorSettings.setValueColoring((CsvEditorSettings.ValueColoring) comboValueColoring.getSelectedItem());
         csvEditorSettings.setHeaderRowFixed(cbHeaderRowFixed.isSelected());
+        csvEditorSettings.setAutoDetectValueSeparator(cbAutoDetectSeparator.isSelected());
     }
 
     protected void createUIComponents() {

src/main/resources/META-INF/plugin.xml

@@ -50,9 +50,7 @@
 
     <change-notes><![CDATA[
         <pre style="font-family: sans-serif">
-NEW: Added default "Header row fixed" setting
-NEW: Support "Comment with line comment" #247
-FIX: "Value coloring" change not applied to open files
+NEW: Auto detect value separator (by count)
       </pre>
     ]]>
     </change-notes>

src/test/java/net/seesharpsoft/intellij/plugins/csv/parser/CsvParsingTest.java

@@ -13,49 +13,70 @@ public CsvParsingTest() {
         super("", "csv", new CsvParserDefinition());
     }
 
+    @Override
+    protected void setUp() throws Exception {
+        super.setUp();
+        CsvEditorSettings.getInstance().setAutoDetectValueSeparator(false);
+    }
+
+    @Override
+    protected void tearDown() throws Exception {
+        CsvEditorSettings.getInstance().setAutoDetectValueSeparator(true);
+        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
+        CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT);
+        super.tearDown();
+    }
+
     public void testParsingTestData() {
         // without comment support, default lexer is used
         CsvEditorSettings.getInstance().setCommentIndicator("");
         doTest(true);
-        CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT);
     }
 
     public void testParsingTestDataWithCustomParser() {
         setName("ParsingTestData");
         CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.create(","));
         doTest(true);
-        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
     }
 
     public void testCustomMultiSymbolSeparator() {
         CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.create("~§"));
         doTest(true);
-        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
     }
 
     public void testColonSeparator() {
         // without comment support, default lexer is used
         CsvEditorSettings.getInstance().setCommentIndicator("");
         CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.COLON);
         doTest(true);
-        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
-        CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT);
     }
 
     public void testAllSeparators() {
         // without comment support, default lexer is used
         CsvEditorSettings.getInstance().setCommentIndicator("");
         CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.COMMA);
         doTest(true);
-        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
-        CsvEditorSettings.getInstance().setCommentIndicator(COMMENT_INDICATOR_DEFAULT);
     }
 
     public void testCsvWithComments() {
         // comment support by default (custom lexer is used)
         CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.COMMA);
         doTest(true);
-        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvEditorSettings.VALUE_SEPARATOR_DEFAULT);
+    }
+
+    public void testParsingTestDataWithAutoDetect() {
+        setName("ParsingTestData");
+        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.PIPE);
+        CsvEditorSettings.getInstance().setAutoDetectValueSeparator(true);
+        doTest(true);
+    }
+
+    public void testColonSeparatorWithAutoDetect() {
+        setName("ColonSeparator");
+        CsvEditorSettings.getInstance().setCommentIndicator("");
+        CsvEditorSettings.getInstance().setDefaultValueSeparator(CsvValueSeparator.PIPE);
+        CsvEditorSettings.getInstance().setAutoDetectValueSeparator(true);
+        doTest(true);
     }
 
     @Override