Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

encoding option for .xls importers/exporters #303

Merged
merged 5 commits into from
Apr 6, 2024
Merged

encoding option for .xls importers/exporters #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6c47a3d
adopt changes in ASP results xls format from aspregister.basg.gv.at
rkrenn Apr 6, 2024
177ab43
encoding option for xls importers
rkrenn Apr 6, 2024
3c90576
encoding option for xls exporters
rkrenn Apr 6, 2024
d78649f
wire encoding dbtool options for xls tasks
rkrenn Apr 6, 2024
a389a9f
leave encoding for testcase ecrf.xls untouched
rkrenn Apr 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,13 @@ public boolean getTrimValues() {
return trimValues;
}

public WorkbookSettings getWorkbookSettings() {
public WorkbookSettings getWorkbookSettings(String encoding) {
WorkbookSettings workbookSettings = new WorkbookSettings();
workbookSettings.setLocale(Locale.getDefault());
if (!CommonUtil.isEmptyString(encoding)) {
jobOutput.println("using " + encoding + " encoding");
workbookSettings.setEncoding(encoding);
}
return workbookSettings;
}

Expand Down
7 changes: 6 additions & 1 deletion common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/RowWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.Locale;

import org.phoenixctms.ctsms.fileprocessors.ProcessorJobOutput;
import org.phoenixctms.ctsms.util.CommonUtil;

import jxl.WorkbookSettings;

Expand Down Expand Up @@ -39,9 +40,13 @@ public int getSheetNum() {
return 0;
}

public WorkbookSettings getWorkbookSettings() {
public WorkbookSettings getWorkbookSettings(String encoding) {
WorkbookSettings workbookSettings = new WorkbookSettings();
workbookSettings.setLocale(Locale.getDefault());
if (!CommonUtil.isEmptyString(encoding)) {
jobOutput.println("using " + encoding + " encoding");
workbookSettings.setEncoding(encoding);
}
return workbookSettings;
}

Expand Down
4 changes: 2 additions & 2 deletions common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsExporterBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ public abstract class XlsExporterBase {
protected void addEmailXlsAttachment(byte[] data) {
}

protected long printRows(XlsExporterContext context, RowWriter writer) throws Throwable {
protected long printRows(XlsExporterContext context, String encoding, RowWriter writer) throws Throwable {
ByteArrayOutputStream buffer = null;
try {
WritableWorkbook workbook;
WorkbookSettings workbookSettings = writer.getWorkbookSettings();
WorkbookSettings workbookSettings = writer.getWorkbookSettings(encoding);
if (!CommonUtil.isEmptyString(context.getFileName())) {
jobOutput.println("writing to file " + context.getFileName());
if (workbookSettings != null) {
Expand Down
4 changes: 2 additions & 2 deletions common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ public abstract class XlsImporterBase {
protected abstract InputStream getInputStream(String fileName, AuthenticationVO auth) throws AuthenticationException,
AuthorisationException, ServiceException, FileNotFoundException;

protected long readRows(XlsImporterContext context, RowProcessor processor) throws Throwable {
protected long readRows(XlsImporterContext context, String encoding, RowProcessor processor) throws Throwable {
processor.init();
long rowCount = 0l;
long lineNumber = 1l;
Workbook workbook = null;
try {
InputStream inputStream = getInputStream(context.getFileName(), context.getAuth());
WorkbookSettings workbookSettings = processor.getWorkbookSettings();
WorkbookSettings workbookSettings = processor.getWorkbookSettings(encoding);
if (workbookSettings != null) {
workbook = Workbook.getWorkbook(inputStream, workbookSettings);
} else {
Expand Down
8 changes: 7 additions & 1 deletion common/src/main/java/org/phoenixctms/ctsms/fileprocessors/xls/XlsImporterContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ public class XlsImporterContext {

private XlsImporterBase importer;
private String fileName;
private String encoding;
private AuthenticationVO auth;
private Long entityId;
private HashMap<RowProcessor, Boolean> mandatoryMap;

public XlsImporterContext(XlsImporterBase importer, String fileName) {
public XlsImporterContext(XlsImporterBase importer, String fileName, String encoding) {
this.importer = importer;
this.fileName = fileName;
this.encoding = encoding;
this.auth = null;
this.entityId = null;
mandatoryMap = new HashMap<RowProcessor, Boolean>();
Expand All @@ -32,6 +34,10 @@ public String getFileName() {
return fileName;
}

public String getEncoding() {
return encoding;
}

public XlsImporterBase getImporter() {
return importer;
}
Expand Down
11 changes: 8 additions & 3 deletions core/src/exec/java/org/phoenixctms/ctsms/executable/DBTool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -827,13 +827,15 @@ public static void main(String[] args) {
job = DBToolOptions.getTaskAndLockProcess(DBToolOptions.IMPORT_INPUT_FIELDS_OPT);
dbTool.initJob(line).printPrelude(job);
if (dbTool.testForced(line, "DB will be modified - input fields and selection set values will be updated!")) {
sendEmail = dbTool.getXlsImporter().loadInputFields(line.getOptionValue(DBToolOptions.IMPORT_INPUT_FIELDS_OPT), getAuthenticationOptionValue(line)) > 0l;
sendEmail = dbTool.getXlsImporter().loadInputFields(line.getOptionValue(DBToolOptions.IMPORT_INPUT_FIELDS_OPT),
line.getOptionValue(DBToolOptions.ENCODING_OPT), getAuthenticationOptionValue(line)) > 0l;
}
} else if (line.hasOption(DBToolOptions.IMPORT_ECRFS_OPT)) {
job = DBToolOptions.getTaskAndLockProcess(DBToolOptions.IMPORT_ECRFS_OPT);
dbTool.initJob(line).printPrelude(job);
if (dbTool.testForced(line, "DB will be modified - eCRFs, eCRF fields, input fields and selection set values will be updated!")) {
sendEmail = dbTool.getXlsImporter().loadEcrfs(line.getOptionValue(DBToolOptions.IMPORT_ECRFS_OPT), getAuthenticationOptionValue(line),
sendEmail = dbTool.getXlsImporter().loadEcrfs(line.getOptionValue(DBToolOptions.IMPORT_ECRFS_OPT), line.getOptionValue(DBToolOptions.ENCODING_OPT),
getAuthenticationOptionValue(line),
getIdOptionValue(line, true)) > 0l;
}
} else if (line.hasOption(DBToolOptions.EXPORT_INPUT_FIELD_OPT)) {
Expand All @@ -843,6 +845,7 @@ public static void main(String[] args) {
|| dbTool.testOverwriteFile(line, line.getOptionValue(DBToolOptions.EXPORT_INPUT_FIELD_OPT))) {
sendEmail = dbTool.getXlsExporter().exportInputField(
line.getOptionValue(DBToolOptions.EXPORT_INPUT_FIELD_OPT),
line.getOptionValue(DBToolOptions.ENCODING_OPT),
getAuthenticationOptionValue(line), getIdOptionValue(line, true)) > 0l;
}
} else if (line.hasOption(DBToolOptions.EXPORT_ECRFS_OPT)) {
Expand All @@ -852,19 +855,21 @@ public static void main(String[] args) {
|| dbTool.testOverwriteFile(line, line.getOptionValue(DBToolOptions.EXPORT_ECRFS_OPT))) {
sendEmail = dbTool.getXlsExporter().exportEcrfs(
line.getOptionValue(DBToolOptions.EXPORT_ECRFS_OPT),
line.getOptionValue(DBToolOptions.ENCODING_OPT),
getAuthenticationOptionValue(line), getIdOptionValue(line, true)) > 0l;
}
} else if (line.hasOption(DBToolOptions.IMPORT_RANDOMIZATION_LISTS_OPT)) {
job = DBToolOptions.getTaskAndLockProcess(DBToolOptions.IMPORT_RANDOMIZATION_LISTS_OPT);
dbTool.initJob(line).printPrelude(job);
if (dbTool.testForced(line, "DB will be modified - randomization lists will be updated!")) {
sendEmail = dbTool.getXlsImporter().loadRandomizationLists(line.getOptionValue(DBToolOptions.IMPORT_RANDOMIZATION_LISTS_OPT),
line.getOptionValue(DBToolOptions.ENCODING_OPT),
getAuthenticationOptionValue(line), getIdOptionValue(line, true), true) > 0l;
}
} else if (line.hasOption(DBToolOptions.IMPORT_ASPS_OPT)) {
job = DBToolOptions.getTaskAndLockProcess(DBToolOptions.IMPORT_ASPS_OPT);
dbTool.getJobOutput().printPrelude(job);
sendEmail = dbTool.getXlsImporter().loadAsps(line.getOptionValue(DBToolOptions.IMPORT_ASPS_OPT),
sendEmail = dbTool.getXlsImporter().loadAsps(line.getOptionValue(DBToolOptions.IMPORT_ASPS_OPT), line.getOptionValue(DBToolOptions.ENCODING_OPT),
line.hasOption(DBToolOptions.FLUSH_REVISION_OPT), line.getOptionValue(DBToolOptions.ASP_REVISION_OPT)) > 0l;
} else if (line.hasOption(DBToolOptions.EXPORT_ECRF_PDFS_OPT)) {
job = DBToolOptions.getTaskAndLockProcess(DBToolOptions.EXPORT_ECRF_PDFS_OPT);
Expand Down
30 changes: 15 additions & 15 deletions core/src/exec/java/org/phoenixctms/ctsms/executable/xls/AspRowProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,19 @@ public class AspRowProcessor extends RowProcessor {
private static final int TYPE_COLUMN_INDEX = 1;
private static final int LABELING_COLUMN_INDEX = 2;
private static final int REGISTRATION_NUMBER_COLUMN_INDEX = 3;
private static final int PROPRIETOR_COLUMN_INDEX = 4;
private static final int REGISTRATION_DATE_COLUMN_INDEX = 5;
private static final int SUBSTANCES_COLUMN_INDEX = 6;
private static final int ATC_CODE_COLUMN_INDEX = 7;
private static final int NARCOTIC_COLUMN_INDEX = 8;
private static final int PSYCHOTROPIC_COLUMN_INDEX = 9;
private static final int BATCH_RELEASE_COLUMN_INDEX = 10;
private static final int BATCH_TESTING_COLUMN_INDEX = 11;
private static final int BATCH_TESTING_EXCLUSION_COLUMN_INDEX = 12;
private static final int PRESCRIPTION_COLUMN_INDEX = 13;
private static final int DISTRIBUTION_COLUMN_INDEX = 14;
private static final int HUMAN_COLUMN_INDEX = 15;
private static final int CATEGORY_COLUMN_INDEX = 16;
private static final int PROPRIETOR_COLUMN_INDEX = 5;
private static final int REGISTRATION_DATE_COLUMN_INDEX = 6;
private static final int SUBSTANCES_COLUMN_INDEX = 7;
private static final int ATC_CODE_COLUMN_INDEX = 8;
private static final int NARCOTIC_COLUMN_INDEX = 9;
private static final int PSYCHOTROPIC_COLUMN_INDEX = 10;
private static final int BATCH_RELEASE_COLUMN_INDEX = 11;
private static final int BATCH_TESTING_COLUMN_INDEX = 12;
private static final int BATCH_TESTING_EXCLUSION_COLUMN_INDEX = 13;
private static final int PRESCRIPTION_COLUMN_INDEX = 14;
private static final int DISTRIBUTION_COLUMN_INDEX = 15;
private static final int HUMAN_COLUMN_INDEX = 16;
private static final int CATEGORY_COLUMN_INDEX = 17;
private final static Pattern SUBSTANCES_SEPARATOR_REGEXP = Pattern.compile(";");
private final static Pattern ATC_CODES_SEPARATOR_REGEXP = Pattern.compile(";");

Expand All @@ -50,9 +50,9 @@ private static boolean isHuman(String value) {
}

private static boolean parseBoolean(String value) {
if ("1".equals(value)) {
if ("1".equals(value) || "ja".equals(value.toLowerCase())) {
return true;
} else if ("0".equals(value)) {
} else if ("0".equals(value) || "nein".equals(value.toLowerCase())) {
return false;
} else {
throw new IllegalArgumentException("cannot parse boolean value " + value);
Expand Down
8 changes: 4 additions & 4 deletions core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,24 @@ public class XlsExporter extends XlsExporterBase {
public XlsExporter() {
}

public long exportEcrfs(String fileName, AuthenticationVO auth, Long trialId) throws Throwable {
public long exportEcrfs(String fileName, String encoding, AuthenticationVO auth, Long trialId) throws Throwable {
XlsExporterContext context = new XlsExporterContext(this, fileName);
setContext(ecrfRowWriter, context);
setContext(ecrfFieldRowWriter, context);
setContext(inputFieldRowWriter, context);
setContext(selectionSetValueRowWriter, context);
context.setEntityId(ecrfRowWriter, trialId);
context.setAuth(auth);
return printRows(context, ecrfRowWriter);
return printRows(context, encoding, ecrfRowWriter);
}

public long exportInputField(String fileName, AuthenticationVO auth, Long inputFieldId) throws Throwable {
public long exportInputField(String fileName, String encoding, AuthenticationVO auth, Long inputFieldId) throws Throwable {
XlsExporterContext context = new XlsExporterContext(this, fileName);
setContext(inputFieldRowWriter, context);
setContext(selectionSetValueRowWriter, context);
context.setEntityId(inputFieldRowWriter, inputFieldId);
context.setAuth(auth);
return printRows(context, inputFieldRowWriter);
return printRows(context, encoding, inputFieldRowWriter);
}

public EcrfFieldRowWriter getEcrfFieldRowWriter() {
Expand Down
34 changes: 17 additions & 17 deletions core/src/exec/java/org/phoenixctms/ctsms/executable/xls/XlsImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ private static Search getRevisionSearch(String revision) {
public XlsImporter() {
}

private XlsImporterContext createContext(RowProcessor processor, String fileName, boolean mandatory) {
XlsImporterContext context = new XlsImporterContext(this, fileName);
private XlsImporterContext createContext(RowProcessor processor, String fileName, String encoding, boolean mandatory) {
XlsImporterContext context = new XlsImporterContext(this, fileName, encoding);
setContext(processor, context, mandatory);
return context;
}
Expand All @@ -72,8 +72,8 @@ public SelectionSetValueRowProcessor getSelectionSetValueRowProcessor() {
return selectionSetValueRowProcessor;
}

public long loadAsps(String fileName, boolean removeAllBeforeInsert, String revision) throws Throwable {
XlsImporterContext context = createContext(aspRowProcessor, fileName, true);
public long loadAsps(String fileName, String encoding, boolean removeAllBeforeInsert, String revision) throws Throwable {
XlsImporterContext context = createContext(aspRowProcessor, fileName, encoding, true);
if (CommonUtil.isEmptyString(revision)) {
revision = ExecUtil.removeExtension((new File(fileName)).getName());
jobOutput.println("no asp revision specified, using " + revision);
Expand All @@ -93,43 +93,43 @@ public long loadAsps(String fileName, boolean removeAllBeforeInsert, String revi
removeAspRecords(revision);
jobOutput.println("asp revision " + revision + " cleared");
}
return readRows(context, aspRowProcessor);
return readRows(context, encoding, aspRowProcessor);
}

public long loadRandomizationLists(String fileName, AuthenticationVO auth, Long trialId, boolean purge) throws Throwable {
XlsImporterContext context = createContext(randomizationListCodeRowProcessor, fileName, true);
public long loadRandomizationLists(String fileName, String encoding, AuthenticationVO auth, Long trialId, boolean purge) throws Throwable {
XlsImporterContext context = createContext(randomizationListCodeRowProcessor, fileName, encoding, true);
context.setAuth(auth);
context.setEntityId(trialId);
randomizationListCodeRowProcessor.setPurge(purge);
return readRows(context, randomizationListCodeRowProcessor);
return readRows(context, encoding, randomizationListCodeRowProcessor);
}

protected long loadEcrfFields(XlsImporterContext context) throws Throwable {
setContext(ecrfFieldRowProcessor, context, true);
return readRows(context, ecrfFieldRowProcessor);
return readRows(context, context.getEncoding(), ecrfFieldRowProcessor);
}

public long loadEcrfs(String fileName, AuthenticationVO auth, Long trialId) throws Throwable {
XlsImporterContext context = createContext(ecrfRowProcessor, fileName, true);
public long loadEcrfs(String fileName, String encoding, AuthenticationVO auth, Long trialId) throws Throwable {
XlsImporterContext context = createContext(ecrfRowProcessor, fileName, encoding, true);
context.setAuth(auth);
context.setEntityId(trialId);
return readRows(context, ecrfRowProcessor);
return readRows(context, encoding, ecrfRowProcessor);
}

public long loadInputFields(String fileName, AuthenticationVO auth) throws Throwable {
XlsImporterContext context = createContext(inputFieldRowProcessor, fileName, true);
public long loadInputFields(String fileName, String encoding, AuthenticationVO auth) throws Throwable {
XlsImporterContext context = createContext(inputFieldRowProcessor, fileName, encoding, true);
context.setAuth(auth);
return readRows(context, inputFieldRowProcessor);
return readRows(context, encoding, inputFieldRowProcessor);
}

protected long loadInputFields(XlsImporterContext context) throws Throwable {
setContext(inputFieldRowProcessor, context, false);
return readRows(context, inputFieldRowProcessor);
return readRows(context, context.getEncoding(), inputFieldRowProcessor);
}

protected long loadSelectionSetValues(XlsImporterContext context) throws Throwable {
setContext(selectionSetValueRowProcessor, context, context.isMandatory(inputFieldRowProcessor));
return readRows(context, selectionSetValueRowProcessor);
return readRows(context, context.getEncoding(), selectionSetValueRowProcessor);
}

protected InputStream getInputStream(String fileName, AuthenticationVO auth) throws AuthenticationException,
Expand Down
2 changes: 1 addition & 1 deletion core/src/exec/java/org/phoenixctms/ctsms/util/DBToolOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ public String toString() {
options.addOption(registerOptionalOption(LIMIT_OPT, "limit", "limit for number of (processed) records", 1));
options.addOption(registerOptionalOption(FLUSH_REVISION_OPT, "flush_revision", "flush alpha id, ops code, asp, asp, icd or ops systematics records prior to import",
0));
options.addOption(registerOptionalOption(ENCODING_OPT, "encoding", "encoding of csv/text file to import", 1));
options.addOption(registerOptionalOption(ENCODING_OPT, "encoding", "encoding of file to import/export", 1));
options.addOption(registerOptionalOption(USERNAME_OPT, "username", "username", 1));
options.addOption(registerOptionalOption(PASSWORD_OPT, "password", "user password", 1));
options.addOption(registerOptionalOption(AUTH_OPT, "auth", "base64 encoded username and password", 1));
Expand Down
3 changes: 2 additions & 1 deletion core/src/test/java/org/phoenixctms/ctsms/selenium/trial/EcrfDataEntryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
public class EcrfDataEntryTest extends SeleniumTestBase implements ProcessorJobOutput {

private final static String ECRF_FILE = "ecrfs.xls";
private final static String ECRF_FILE_ENCODING = null;
private String departmentPassword;
private DepartmentVO department;
private String userName;
Expand Down Expand Up @@ -102,7 +103,7 @@ public void test_03_import_ecrf_setup_job() throws Throwable {
if (waitForAddOperationSuccessful("tabView:trialjob_form")) {
if (waitForJobSuccessful("tabView:trialjob_form", 60l, 5l)) {
info("loading eCRF validation test data from " + getResourceFilePath(ECRF_FILE));
getXlsImporter().loadEcrfValidationVectors(getResourceFilePath(ECRF_FILE), trial.getId());
getXlsImporter().loadEcrfValidationVectors(getResourceFilePath(ECRF_FILE), ECRF_FILE_ENCODING, trial.getId());
return;
}
}
Expand Down
10 changes: 5 additions & 5 deletions core/src/test/java/org/phoenixctms/ctsms/test/xls/XlsImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ public class XlsImporter extends XlsImporterBase {
public XlsImporter() {
}

private XlsImporterContext createContext(RowProcessor processor, String fileName, boolean mandatory) {
XlsImporterContext context = new XlsImporterContext(this, fileName);
private XlsImporterContext createContext(RowProcessor processor, String fileName, String encoding, boolean mandatory) {
XlsImporterContext context = new XlsImporterContext(this, fileName, encoding);
setContext(processor, context, mandatory);
return context;
}

public long loadEcrfValidationVectors(String fileName, Long trialId) throws Throwable {
XlsImporterContext context = createContext(ecrfValidationRowProcessor, fileName, true);
public long loadEcrfValidationVectors(String fileName, String encoding, Long trialId) throws Throwable {
XlsImporterContext context = createContext(ecrfValidationRowProcessor, fileName, encoding, true);
context.setEntityId(trialId);
return readRows(context, ecrfValidationRowProcessor);
return readRows(context, encoding, ecrfValidationRowProcessor);
}

public List<EcrfValidationTestVector> getEcrfValidationTestVectors(String ecrfName, String ecrfRevision) {
Expand Down
Loading