From 3eb9961e0a1899c3796ed7de8faca4237b9bdf53 Mon Sep 17 00:00:00 2001 From: Rene Krenn Date: Sat, 6 Apr 2024 18:32:01 +0200 Subject: [PATCH] encoding option for xls importers --- .../fileprocessors/xls/RowProcessor.java | 6 +++- .../fileprocessors/xls/XlsImporterBase.java | 4 +-- .../xls/XlsImporterContext.java | 8 ++++- .../ctsms/executable/xls/XlsImporter.java | 34 +++++++++---------- .../ctsms/test/xls/XlsImporter.java | 10 +++--- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java index c225a9136363..90ef2f790362 100644 --- a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java +++ b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java @@ -66,9 +66,13 @@ public boolean getTrimValues() { return trimValues; } - public WorkbookSettings getWorkbookSettings() { + public WorkbookSettings getWorkbookSettings(String encoding) { WorkbookSettings workbookSettings = new WorkbookSettings(); workbookSettings.setLocale(Locale.getDefault()); + if (!CommonUtil.isEmptyString(encoding)) { + jobOutput.println("using " + encoding + " encoding"); + workbookSettings.setEncoding(encoding); + } return workbookSettings; } diff --git a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java index e8748658252e..dd415d022e68 100644 --- a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java +++ b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java @@ -22,14 +22,14 @@ public abstract class XlsImporterBase { protected abstract InputStream getInputStream(String fileName, AuthenticationVO auth) throws AuthenticationException, AuthorisationException, ServiceException, FileNotFoundException; - protected long readRows(XlsImporterContext context, RowProcessor processor) throws Throwable { + protected long readRows(XlsImporterContext context, String encoding, RowProcessor processor) throws Throwable { processor.init(); long rowCount = 0l; long lineNumber = 1l; Workbook workbook = null; try { InputStream inputStream = getInputStream(context.getFileName(), context.getAuth()); - WorkbookSettings workbookSettings = processor.getWorkbookSettings(); + WorkbookSettings workbookSettings = processor.getWorkbookSettings(encoding); if (workbookSettings != null) { workbook = Workbook.getWorkbook(inputStream, workbookSettings); } else { diff --git a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java index 71f02884541f..0e3e840f1b1f 100644 --- a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java +++ b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java @@ -8,13 +8,15 @@ public class XlsImporterContext { private XlsImporterBase importer; private String fileName; + private String encoding; private AuthenticationVO auth; private Long entityId; private HashMap mandatoryMap; - public XlsImporterContext(XlsImporterBase importer, String fileName) { + public XlsImporterContext(XlsImporterBase importer, String fileName, String encoding) { this.importer = importer; this.fileName = fileName; + this.encoding = encoding; this.auth = null; this.entityId = null; mandatoryMap = new HashMap(); @@ -32,6 +34,10 @@ public String getFileName() { return fileName; } + public String getEncoding() { + return encoding; + } + public XlsImporterBase getImporter() { return importer; } diff --git a/core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java b/core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java index d982f07fa7ff..408117586c84 100644 --- a/core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java +++ b/core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java @@ -58,8 +58,8 @@ private static Search getRevisionSearch(String revision) { public XlsImporter() { } - private XlsImporterContext createContext(RowProcessor processor, String fileName, boolean mandatory) { - XlsImporterContext context = new XlsImporterContext(this, fileName); + private XlsImporterContext createContext(RowProcessor processor, String fileName, String encoding, boolean mandatory) { + XlsImporterContext context = new XlsImporterContext(this, fileName, encoding); setContext(processor, context, mandatory); return context; } @@ -72,8 +72,8 @@ public SelectionSetValueRowProcessor getSelectionSetValueRowProcessor() { return selectionSetValueRowProcessor; } - public long loadAsps(String fileName, boolean removeAllBeforeInsert, String revision) throws Throwable { - XlsImporterContext context = createContext(aspRowProcessor, fileName, true); + public long loadAsps(String fileName, String encoding, boolean removeAllBeforeInsert, String revision) throws Throwable { + XlsImporterContext context = createContext(aspRowProcessor, fileName, encoding, true); if (CommonUtil.isEmptyString(revision)) { revision = ExecUtil.removeExtension((new File(fileName)).getName()); jobOutput.println("no asp revision specified, using " + revision); @@ -93,43 +93,43 @@ public long loadAsps(String fileName, boolean removeAllBeforeInsert, String revi removeAspRecords(revision); jobOutput.println("asp revision " + revision + " cleared"); } - return readRows(context, aspRowProcessor); + return readRows(context, encoding, aspRowProcessor); } - public long loadRandomizationLists(String fileName, AuthenticationVO auth, Long trialId, boolean purge) throws Throwable { - XlsImporterContext context = createContext(randomizationListCodeRowProcessor, fileName, true); + public long loadRandomizationLists(String fileName, String encoding, AuthenticationVO auth, Long trialId, boolean purge) throws Throwable { + XlsImporterContext context = createContext(randomizationListCodeRowProcessor, fileName, encoding, true); context.setAuth(auth); context.setEntityId(trialId); randomizationListCodeRowProcessor.setPurge(purge); - return readRows(context, randomizationListCodeRowProcessor); + return readRows(context, encoding, randomizationListCodeRowProcessor); } protected long loadEcrfFields(XlsImporterContext context) throws Throwable { setContext(ecrfFieldRowProcessor, context, true); - return readRows(context, ecrfFieldRowProcessor); + return readRows(context, context.getEncoding(), ecrfFieldRowProcessor); } - public long loadEcrfs(String fileName, AuthenticationVO auth, Long trialId) throws Throwable { - XlsImporterContext context = createContext(ecrfRowProcessor, fileName, true); + public long loadEcrfs(String fileName, String encoding, AuthenticationVO auth, Long trialId) throws Throwable { + XlsImporterContext context = createContext(ecrfRowProcessor, fileName, encoding, true); context.setAuth(auth); context.setEntityId(trialId); - return readRows(context, ecrfRowProcessor); + return readRows(context, encoding, ecrfRowProcessor); } - public long loadInputFields(String fileName, AuthenticationVO auth) throws Throwable { - XlsImporterContext context = createContext(inputFieldRowProcessor, fileName, true); + public long loadInputFields(String fileName, String encoding, AuthenticationVO auth) throws Throwable { + XlsImporterContext context = createContext(inputFieldRowProcessor, fileName, encoding, true); context.setAuth(auth); - return readRows(context, inputFieldRowProcessor); + return readRows(context, encoding, inputFieldRowProcessor); } protected long loadInputFields(XlsImporterContext context) throws Throwable { setContext(inputFieldRowProcessor, context, false); - return readRows(context, inputFieldRowProcessor); + return readRows(context, context.getEncoding(), inputFieldRowProcessor); } protected long loadSelectionSetValues(XlsImporterContext context) throws Throwable { setContext(selectionSetValueRowProcessor, context, context.isMandatory(inputFieldRowProcessor)); - return readRows(context, selectionSetValueRowProcessor); + return readRows(context, context.getEncoding(), selectionSetValueRowProcessor); } protected InputStream getInputStream(String fileName, AuthenticationVO auth) throws AuthenticationException, diff --git a/core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java b/core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java index 7b221aac143b..ed2cb27cd0c6 100644 --- a/core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java +++ b/core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java @@ -24,16 +24,16 @@ public class XlsImporter extends XlsImporterBase { public XlsImporter() { } - private XlsImporterContext createContext(RowProcessor processor, String fileName, boolean mandatory) { - XlsImporterContext context = new XlsImporterContext(this, fileName); + private XlsImporterContext createContext(RowProcessor processor, String fileName, String encoding, boolean mandatory) { + XlsImporterContext context = new XlsImporterContext(this, fileName, encoding); setContext(processor, context, mandatory); return context; } - public long loadEcrfValidationVectors(String fileName, Long trialId) throws Throwable { - XlsImporterContext context = createContext(ecrfValidationRowProcessor, fileName, true); + public long loadEcrfValidationVectors(String fileName, String encoding, Long trialId) throws Throwable { + XlsImporterContext context = createContext(ecrfValidationRowProcessor, fileName, encoding, true); context.setEntityId(trialId); - return readRows(context, ecrfValidationRowProcessor); + return readRows(context, encoding, ecrfValidationRowProcessor); } public List getEcrfValidationTestVectors(String ecrfName, String ecrfRevision) {