encoding option for xls importers

phoenixctms · Apr 6, 2024 · 3eb9961 · 3eb9961
1 parent 3e5d46c
commit 3eb9961
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 26 deletions.
diff --git a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java
@@ -66,9 +66,13 @@ public boolean getTrimValues() {
 		return trimValues;
 	}
 
-	public WorkbookSettings getWorkbookSettings() {
+	public WorkbookSettings getWorkbookSettings(String encoding) {
 		WorkbookSettings workbookSettings = new WorkbookSettings();
 		workbookSettings.setLocale(Locale.getDefault());
+		if (!CommonUtil.isEmptyString(encoding)) {
+			jobOutput.println("using " + encoding + " encoding");
+			workbookSettings.setEncoding(encoding);
+		}
 		return workbookSettings;
 	}
 

diff --git a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java
@@ -22,14 +22,14 @@ public abstract class XlsImporterBase {
 	protected abstract InputStream getInputStream(String fileName, AuthenticationVO auth) throws AuthenticationException,
 			AuthorisationException, ServiceException, FileNotFoundException;
 
-	protected long readRows(XlsImporterContext context, RowProcessor processor) throws Throwable {
+	protected long readRows(XlsImporterContext context, String encoding, RowProcessor processor) throws Throwable {
 		processor.init();
 		long rowCount = 0l;
 		long lineNumber = 1l;
 		Workbook workbook = null;
 		try {
 			InputStream inputStream = getInputStream(context.getFileName(), context.getAuth());
-			WorkbookSettings workbookSettings = processor.getWorkbookSettings();
+			WorkbookSettings workbookSettings = processor.getWorkbookSettings(encoding);
 			if (workbookSettings != null) {
 				workbook = Workbook.getWorkbook(inputStream, workbookSettings);
 			} else {

diff --git a/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java b/common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java
@@ -8,13 +8,15 @@ public class XlsImporterContext {
 
 	private XlsImporterBase importer;
 	private String fileName;
+	private String encoding;
 	private AuthenticationVO auth;
 	private Long entityId;
 	private HashMap<RowProcessor, Boolean> mandatoryMap;
 
-	public XlsImporterContext(XlsImporterBase importer, String fileName) {
+	public XlsImporterContext(XlsImporterBase importer, String fileName, String encoding) {
 		this.importer = importer;
 		this.fileName = fileName;
+		this.encoding = encoding;
 		this.auth = null;
 		this.entityId = null;
 		mandatoryMap = new HashMap<RowProcessor, Boolean>();
@@ -32,6 +34,10 @@ public String getFileName() {
 		return fileName;
 	}
 
+	public String getEncoding() {
+		return encoding;
+	}
+
 	public XlsImporterBase getImporter() {
 		return importer;
 	}

diff --git a/core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java b/core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java
@@ -58,8 +58,8 @@ private static Search getRevisionSearch(String revision) {
 	public XlsImporter() {
 	}
 
-	private XlsImporterContext createContext(RowProcessor processor, String fileName, boolean mandatory) {
-		XlsImporterContext context = new XlsImporterContext(this, fileName);
+	private XlsImporterContext createContext(RowProcessor processor, String fileName, String encoding, boolean mandatory) {
+		XlsImporterContext context = new XlsImporterContext(this, fileName, encoding);
 		setContext(processor, context, mandatory);
 		return context;
 	}
@@ -72,8 +72,8 @@ public SelectionSetValueRowProcessor getSelectionSetValueRowProcessor() {
 		return selectionSetValueRowProcessor;
 	}
 
-	public long loadAsps(String fileName, boolean removeAllBeforeInsert, String revision) throws Throwable {
-		XlsImporterContext context = createContext(aspRowProcessor, fileName, true);
+	public long loadAsps(String fileName, String encoding, boolean removeAllBeforeInsert, String revision) throws Throwable {
+		XlsImporterContext context = createContext(aspRowProcessor, fileName, encoding, true);
 		if (CommonUtil.isEmptyString(revision)) {
 			revision = ExecUtil.removeExtension((new File(fileName)).getName());
 			jobOutput.println("no asp revision specified, using " + revision);
@@ -93,43 +93,43 @@ public long loadAsps(String fileName, boolean removeAllBeforeInsert, String revi
 			removeAspRecords(revision);
 			jobOutput.println("asp revision " + revision + " cleared");
 		}
-		return readRows(context, aspRowProcessor);
+		return readRows(context, encoding, aspRowProcessor);
 	}
 
-	public long loadRandomizationLists(String fileName, AuthenticationVO auth, Long trialId, boolean purge) throws Throwable {
-		XlsImporterContext context = createContext(randomizationListCodeRowProcessor, fileName, true);
+	public long loadRandomizationLists(String fileName, String encoding, AuthenticationVO auth, Long trialId, boolean purge) throws Throwable {
+		XlsImporterContext context = createContext(randomizationListCodeRowProcessor, fileName, encoding, true);
 		context.setAuth(auth);
 		context.setEntityId(trialId);
 		randomizationListCodeRowProcessor.setPurge(purge);
-		return readRows(context, randomizationListCodeRowProcessor);
+		return readRows(context, encoding, randomizationListCodeRowProcessor);
 	}
 
 	protected long loadEcrfFields(XlsImporterContext context) throws Throwable {
 		setContext(ecrfFieldRowProcessor, context, true);
-		return readRows(context, ecrfFieldRowProcessor);
+		return readRows(context, context.getEncoding(), ecrfFieldRowProcessor);
 	}
 
-	public long loadEcrfs(String fileName, AuthenticationVO auth, Long trialId) throws Throwable {
-		XlsImporterContext context = createContext(ecrfRowProcessor, fileName, true);
+	public long loadEcrfs(String fileName, String encoding, AuthenticationVO auth, Long trialId) throws Throwable {
+		XlsImporterContext context = createContext(ecrfRowProcessor, fileName, encoding, true);
 		context.setAuth(auth);
 		context.setEntityId(trialId);
-		return readRows(context, ecrfRowProcessor);
+		return readRows(context, encoding, ecrfRowProcessor);
 	}
 
-	public long loadInputFields(String fileName, AuthenticationVO auth) throws Throwable {
-		XlsImporterContext context = createContext(inputFieldRowProcessor, fileName, true);
+	public long loadInputFields(String fileName, String encoding, AuthenticationVO auth) throws Throwable {
+		XlsImporterContext context = createContext(inputFieldRowProcessor, fileName, encoding, true);
 		context.setAuth(auth);
-		return readRows(context, inputFieldRowProcessor);
+		return readRows(context, encoding, inputFieldRowProcessor);
 	}
 
 	protected long loadInputFields(XlsImporterContext context) throws Throwable {
 		setContext(inputFieldRowProcessor, context, false);
-		return readRows(context, inputFieldRowProcessor);
+		return readRows(context, context.getEncoding(), inputFieldRowProcessor);
 	}
 
 	protected long loadSelectionSetValues(XlsImporterContext context) throws Throwable {
 		setContext(selectionSetValueRowProcessor, context, context.isMandatory(inputFieldRowProcessor));
-		return readRows(context, selectionSetValueRowProcessor);
+		return readRows(context, context.getEncoding(), selectionSetValueRowProcessor);
 	}
 
 	protected InputStream getInputStream(String fileName, AuthenticationVO auth) throws AuthenticationException,

diff --git a/core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java b/core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java
@@ -24,16 +24,16 @@ public class XlsImporter extends XlsImporterBase {
 	public XlsImporter() {
 	}
 
-	private XlsImporterContext createContext(RowProcessor processor, String fileName, boolean mandatory) {
-		XlsImporterContext context = new XlsImporterContext(this, fileName);
+	private XlsImporterContext createContext(RowProcessor processor, String fileName, String encoding, boolean mandatory) {
+		XlsImporterContext context = new XlsImporterContext(this, fileName, encoding);
 		setContext(processor, context, mandatory);
 		return context;
 	}
 
-	public long loadEcrfValidationVectors(String fileName, Long trialId) throws Throwable {
-		XlsImporterContext context = createContext(ecrfValidationRowProcessor, fileName, true);
+	public long loadEcrfValidationVectors(String fileName, String encoding, Long trialId) throws Throwable {
+		XlsImporterContext context = createContext(ecrfValidationRowProcessor, fileName, encoding, true);
 		context.setEntityId(trialId);
-		return readRows(context, ecrfValidationRowProcessor);
+		return readRows(context, encoding, ecrfValidationRowProcessor);
 	}
 
 	public List<EcrfValidationTestVector> getEcrfValidationTestVectors(String ecrfName, String ecrfRevision) {