From a56e025f6ecbe927194e41fe03502a3880105b0c Mon Sep 17 00:00:00 2001 From: Josh Eckels Date: Wed, 8 Jan 2025 10:32:57 -0800 Subject: [PATCH 1/4] Avoid NPE when connection is cut when rendering assay QC warnings (#6198) --- api/src/org/labkey/api/assay/AssayProtocolSchema.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/assay/AssayProtocolSchema.java b/api/src/org/labkey/api/assay/AssayProtocolSchema.java index 25918439409..c6f66891d6c 100644 --- a/api/src/org/labkey/api/assay/AssayProtocolSchema.java +++ b/api/src/org/labkey/api/assay/AssayProtocolSchema.java @@ -89,6 +89,7 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Path; import org.labkey.api.util.StringExpressionFactory; +import org.labkey.api.util.UnexpectedException; import org.labkey.api.view.ActionURL; import org.labkey.api.view.DataView; import org.labkey.api.view.NotFoundException; @@ -746,13 +747,17 @@ public void addQCWarningIndicator(QueryView baseQueryView, ViewContext context, } catch(IOException e) { - throw new RuntimeException(e); + throw UnexpectedException.wrap(e); } }); } } catch (SQLException | IOException e) { + if (errors == null) + { + throw UnexpectedException.wrap(e); + } errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); } } From 4d90e87fb591c9c18491cc777cd53e8b01b0415e Mon Sep 17 00:00:00 2001 From: Josh Eckels Date: Wed, 8 Jan 2025 13:16:27 -0800 Subject: [PATCH 2/4] Improve schema upgrade logging and version enforcement (#6197) --- .../org/labkey/api/module/ModuleLoader.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/api/src/org/labkey/api/module/ModuleLoader.java b/api/src/org/labkey/api/module/ModuleLoader.java index 594d2bb4b64..82b9d474834 100644 --- a/api/src/org/labkey/api/module/ModuleLoader.java +++ b/api/src/org/labkey/api/module/ModuleLoader.java @@ -475,6 +475,16 @@ public void updateModuleDirectory(File dir, File archive) } } + private record UpgradeInfo(String moduleName, double installedVersion) + { + @Override + public String toString() + { + return moduleName + " (from schema version " + ModuleContext.formatVersion(installedVersion) + ")"; + + } + } + /** Full web-server initialization */ private void doInit(Execution execution) throws ServletException { @@ -646,7 +656,7 @@ public void addStaticWarnings(@NotNull Warnings warnings, boolean showAllWarning upgradeLabKeySchemaInExternalDataSources(); ModuleContext coreCtx; - List modulesRequiringUpgrade = new LinkedList<>(); + List modulesRequiringUpgrade = new LinkedList<>(); List additionalSchemasRequiringUpgrade = new LinkedList<>(); List downgradedModules = new LinkedList<>(); @@ -675,7 +685,7 @@ public void addStaticWarnings(@NotNull Warnings warnings, boolean showAllWarning if (context.needsUpgrade(module.getSchemaVersion())) { context.setModuleState(ModuleState.InstallRequired); - modulesRequiringUpgrade.add(context.getName()); + modulesRequiringUpgrade.add(new UpgradeInfo(context.getName(), context.getInstalledVersion())); addModuleToLockFile(context.getName(), lockFile); } else @@ -771,11 +781,11 @@ private void warnAboutDuplicateSchemas(Collection values) /** * Does this module live in a repository that's managed by LabKey Corporation? * @param module a Module - * @return true if the module's VCS URL is non-null and starts with one of the GitHub organizations that LabKey manages + * @return true if the module's VCS URL is non-null and includes "github.com:LabKey/" */ private boolean isFromLabKeyRepository(Module module) { - return StringUtils.startsWithAny(module.getVcsUrl(), "https://github.com/LabKey/"); + return StringUtils.containsAny(module.getVcsUrl(), "github.com:LabKey/"); } /** From 19b4390f9143da52ed82e6c4a97dcbfd2c1abc6f Mon Sep 17 00:00:00 2001 From: Karl Lum Date: Thu, 9 Jan 2025 10:17:41 -0800 Subject: [PATCH 3/4] Support for running transform scripts during result updates. (#6196) --- .../assay/AbstractAssayTsvDataHandler.java | 33 +- .../plate/AssayPlateMetadataService.java | 29 ++ .../assay/transform/DataExchangeHandler.java | 43 +- .../assay/transform/DataTransformService.java | 8 +- .../labkey/api/qc/TsvDataExchangeHandler.java | 388 ++++++++++-------- .../api/assay/AssayResultUpdateService.java | 281 +++++++++++++ .../dilution/DilutionDataExchangeHandler.java | 22 +- .../src/org/labkey/assay/AssayController.java | 5 +- .../plate/AssayPlateMetadataServiceImpl.java | 103 +++-- 9 files changed, 667 insertions(+), 245 deletions(-) diff --git a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java index f1b742065e5..924f8e21263 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java @@ -120,6 +120,7 @@ import java.util.function.Function; import static java.util.stream.Collectors.toList; +import static org.labkey.api.assay.AssayRunUploadContext.ReImportOption.MERGE_DATA; import static org.labkey.api.exp.OntologyManager.NO_OP_ROW_CALLBACK; import static org.labkey.api.gwt.client.ui.PropertyType.SAMPLE_CONCEPT_URI; @@ -237,8 +238,9 @@ private DataIteratorBuilder parsePlateData( Container container = context.getContainer(); User user = context.getUser(); - Integer plateSetId = getPlateSetValueFromRunProps(context, provider, protocol); - DataIteratorBuilder dataRows = AssayPlateMetadataService.get().parsePlateData(container, user, ((AssayUploadXarContext)context).getContext(), data, provider, + AssayRunUploadContext runUploadContext = ((AssayUploadXarContext)context).getContext(); + Integer plateSetId = AssayPlateMetadataService.get().getPlateSetId(runUploadContext, provider, protocol); + DataIteratorBuilder dataRows = AssayPlateMetadataService.get().parsePlateData(container, user, runUploadContext, data, provider, protocol, plateSetId, dataFile, settings); // assays with plate metadata support will merge the plate metadata with the data rows to make it easier for @@ -251,21 +253,6 @@ private DataIteratorBuilder parsePlateData( return dataRows; } - @Nullable - private Integer getPlateSetValueFromRunProps(XarContext context, AssayProvider provider, ExpProtocol protocol) throws ExperimentException - { - Domain runDomain = provider.getRunDomain(protocol); - DomainProperty propertyPlateSet = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_SET_COLUMN_NAME); - if (propertyPlateSet == null) - { - throw new ExperimentException("The assay run domain for the assay '" + protocol.getName() + "' does not contain a plate set property."); - } - - Map runProps = ((AssayUploadXarContext)context).getContext().getRunProperties(); - Object plateSetVal = runProps.getOrDefault(propertyPlateSet, null); - return plateSetVal != null ? Integer.parseInt(String.valueOf(plateSetVal)) : null; - } - /** * Creates a DataLoader that can handle missing value indicators if the columns on the domain * are configured to support it. @@ -610,7 +597,19 @@ protected void insertRowData( { OntologyManager.UpdateableTableImportHelper importHelper = new SimpleAssayDataImportHelper(data, protocol, provider); if (provider.isPlateMetadataEnabled(protocol)) + { + if (context.getReRunId() != null) + { + // check if we are merging the re-imported data + if (context.getReImportOption() == MERGE_DATA) + { + DataIteratorBuilder mergedData = AssayPlateMetadataService.get().mergeReRunData(container, user, context, fileData, provider, protocol, data); + fileData = DataIteratorUtil.wrapMap(mergedData.getDataIterator(new DataIteratorContext()), false); + } + } + importHelper = AssayPlateMetadataService.get().getImportHelper(container, user, run, data, protocol, provider, context); + } if (tableInfo instanceof UpdateableTableInfo uti) { diff --git a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java index 535e53a2d66..9b1e30ad979 100644 --- a/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java +++ b/api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java @@ -7,6 +7,7 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.TableInfo; +import org.labkey.api.dataiterator.DataIterator; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.Lsid; @@ -66,6 +67,34 @@ DataIteratorBuilder mergePlateMetadata( ExpProtocol protocol ) throws ExperimentException; + /** + * Takes the current incoming data and combines it with any data uploaded in the previous run (re-run ID). Data + * can be combined for plates within a plate set, but only on a per plate boundary. If there is data for plates + * in both sets of data, the most recent data will take precedence. + * + * @param results The incoming data rows + * @return The new, combined data + */ + DataIteratorBuilder mergeReRunData( + Container container, + User user, + @NotNull AssayRunUploadContext context, + DataIterator results, + AssayProvider provider, + ExpProtocol protocol, + ExpData data + ) throws ExperimentException; + + /** + * Returns the plate set ID for the current run context. + */ + @Nullable + Integer getPlateSetId( + AssayRunUploadContext context, + AssayProvider provider, + ExpProtocol protocol + ) throws ExperimentException; + /** * Handles the validation and parsing of the plate data (or data file) including plate graphical formats as * well as cases where plate identifiers have not been supplied. diff --git a/api/src/org/labkey/api/assay/transform/DataExchangeHandler.java b/api/src/org/labkey/api/assay/transform/DataExchangeHandler.java index 539d5af9af2..07028fedd3f 100644 --- a/api/src/org/labkey/api/assay/transform/DataExchangeHandler.java +++ b/api/src/org/labkey/api/assay/transform/DataExchangeHandler.java @@ -16,6 +16,7 @@ package org.labkey.api.assay.transform; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayRunUploadContext; import org.labkey.api.data.TSVWriter; @@ -37,15 +38,47 @@ /** * Used to process input and output data between the server and externally executed qc and analysis scripts. - * User: Karl Lum - * Date: Jan 7, 2009 */ public interface DataExchangeHandler { - Pair> createTransformationRunInfo(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, Map runProperties, Map batchProperties) throws Exception; - void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewContext, FileLike scriptDir) throws Exception; + /** + * Create and serialize the run properties information that is made available to transform scripts. + * The file contains a variety of information based on the transform operation being specified. + * + * @param operation The transform operation being performed + * @param context Contains information about the import or update context + * @param scriptDir The folder that the transform script will be run in. + * @return The map of the run properties file to the set of other data files associated with the operation + * being performed. + */ + Pair> createTransformationRunInfo( + DataTransformService.TransformOperation operation, + AssayRunUploadContext context, + @Nullable ExpRun run, + FileLike scriptDir, + Map runProperties, + Map batchProperties + ) throws Exception; - TransformResult processTransformationOutput(AssayRunUploadContext context, FileLike runInfo, ExpRun run, FileLike scriptFile, TransformResult mergeResult, Set inputDataFiles) throws ValidationException; + /** + * Creates a test version of the run properties file for download + */ + void createSampleData( + DataTransformService.TransformOperation operation, + @NotNull ExpProtocol protocol, + ViewContext viewContext, + FileLike scriptDir + ) throws Exception; + + TransformResult processTransformationOutput( + DataTransformService.TransformOperation operation, + AssayRunUploadContext context, + FileLike runInfo, + @Nullable ExpRun run, + FileLike scriptFile, + TransformResult mergeResult, + Set inputDataFiles + ) throws ValidationException; DataSerializer getDataSerializer(); diff --git a/api/src/org/labkey/api/assay/transform/DataTransformService.java b/api/src/org/labkey/api/assay/transform/DataTransformService.java index 90c0be97e77..b78076a9fba 100644 --- a/api/src/org/labkey/api/assay/transform/DataTransformService.java +++ b/api/src/org/labkey/api/assay/transform/DataTransformService.java @@ -55,6 +55,7 @@ public static DataTransformService get() public static final String BASE_SERVER_URL_REPLACEMENT = "baseServerURL"; public static final String CONTAINER_PATH = "containerPath"; public static final String ORIGINAL_SOURCE_PATH = "OriginalSourcePath"; + public static final String TRANSFORM_OPERATION = "transformOperation"; public enum TransformOperation { @@ -134,7 +135,7 @@ public TransformResult transformAndValidate( Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); String script = sb.toString(); - Pair> files = dataHandler.createTransformationRunInfo(context, run, scriptDir, runProperties, batchProperties); + Pair> files = dataHandler.createTransformationRunInfo(operation, context, run, scriptDir, runProperties, batchProperties); FileLike runInfo = files.getKey(); bindings.put(ExternalScriptEngine.WORKING_DIRECTORY, scriptDir.toNioPathForWrite().toString()); @@ -144,6 +145,7 @@ public TransformResult transformAndValidate( // Issue 51543: Resolve windows path to run properties paramMap.put(RUN_INFO_REPLACEMENT, runInfo.toNioPathForWrite().toFile().getAbsolutePath().replaceAll("\\\\", "/")); + paramMap.put(TRANSFORM_OPERATION, operation.name()); addStandardParameters(context.getRequest(), context.getContainer(), scriptFile, session.getApiKey(), paramMap); @@ -151,7 +153,7 @@ public TransformResult transformAndValidate( Object output = engine.eval(script); - FileLike rewrittenScriptFile = null; + FileLike rewrittenScriptFile; if (bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE) instanceof File) { var rewrittenScriptFileObject = bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE); @@ -166,7 +168,7 @@ public TransformResult transformAndValidate( } // process any output from the transformation script - result = dataHandler.processTransformationOutput(context, runInfo, run, rewrittenScriptFile, result, files.getValue()); + result = dataHandler.processTransformationOutput(operation, context, runInfo, run, rewrittenScriptFile, result, files.getValue()); // Propagate any transformed batch properties on to the next script if (result.getBatchProperties() != null && !result.getBatchProperties().isEmpty()) diff --git a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java index 5a619eae4e6..253ffb3af7f 100644 --- a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java +++ b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java @@ -32,6 +32,7 @@ import org.labkey.api.assay.actions.AssayRunUploadForm; import org.labkey.api.assay.actions.ProtocolIdForm; import org.labkey.api.assay.transform.DataExchangeHandler; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.assay.transform.DefaultTransformResult; import org.labkey.api.assay.transform.TransformResult; import org.labkey.api.collections.CaseInsensitiveHashMap; @@ -156,7 +157,14 @@ public DataSerializer getDataSerializer() } @Override - public Pair> createTransformationRunInfo(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, Map runProperties, Map batchProperties) throws Exception + public Pair> createTransformationRunInfo( + DataTransformService.TransformOperation operation, + AssayRunUploadContext context, + @Nullable ExpRun run, + FileLike scriptDir, + Map runProperties, + Map batchProperties + ) throws Exception { FileLike runProps = scriptDir.resolveChild(VALIDATION_RUN_INFO_FILE); _filesToIgnore.add(runProps.toNioPathForRead().toFile()); @@ -169,7 +177,7 @@ public Pair> createTransformationRunInfo(AssayRunUploadC // Hack to get TSV values to be properly quoted if they include tabs TSVWriter writer = createTSVWriter(); - writeRunProperties(context, mergedProps, scriptDir, pw, writer); + writeRunProperties(operation, context, mergedProps, scriptDir, pw, writer); // add the run data entries Set dataFiles = writeRunData(context, run, scriptDir, pw, writer); @@ -194,20 +202,23 @@ public Pair> createTransformationRunInfo(AssayRunUploadC _filesToIgnore.add(errorFile.toNioPathForRead().toFile()); - // transformed run properties file location - FileLike transformedRunPropsFile = scriptDir.resolveChild(TRANSFORMED_RUN_INFO_FILE); - pw.append(Props.transformedRunPropertiesFile.name()); - pw.append('\t'); - pw.println(transformedRunPropsFile.toNioPathForRead().toString()); - _filesToIgnore.add(transformedRunPropsFile.toNioPathForRead().toFile()); + if (operation == DataTransformService.TransformOperation.INSERT) + { + // transformed run properties file location + FileLike transformedRunPropsFile = scriptDir.resolveChild(TRANSFORMED_RUN_INFO_FILE); + pw.append(Props.transformedRunPropertiesFile.name()); + pw.append('\t'); + pw.println(transformedRunPropsFile.toNioPathForRead().toString()); + _filesToIgnore.add(transformedRunPropsFile.toNioPathForRead().toFile()); - // error level initialization - pw.append(Props.severityLevel.name()); - pw.append('\t'); - if (context instanceof AssayRunUploadForm form && null != form.getSeverityLevel()) - pw.println(form.getSeverityLevel()); - else - pw.println(errLevel.WARN.name()); + // error level initialization + pw.append(Props.severityLevel.name()); + pw.append('\t'); + if (context instanceof AssayRunUploadForm form && null != form.getSeverityLevel()) + pw.println(form.getSeverityLevel()); + else + pw.println(errLevel.WARN.name()); + } return new Pair<>(runProps, dataFiles); } @@ -232,7 +243,7 @@ protected Set writeRunData(AssayRunUploadContext context, ExpRun ru * assay-saveAssayBatch.api: does not include uploadedData but may include an inputData file along with rawData rows. * assay-importRun.api: may include uploadedData or rawData (with or without an inputData file). */ - protected Set _writeRunData(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, PrintWriter pw, TSVWriter tsvWriter) throws Exception + private Set _writeRunData(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, PrintWriter pw, TSVWriter tsvWriter) throws Exception { List result = new ArrayList<>(); @@ -249,7 +260,8 @@ protected Set _writeRunData(AssayRunUploadContext context, ExpRun r if (iterator.next()) { rawDataHasRows = true; - dataInputs = run.getDataInputs().keySet(); + if (run != null) + dataInputs = run.getDataInputs().keySet(); } } } @@ -427,7 +439,14 @@ protected void addSampleProperties(String propertyName, List _sampleProperties.put(propertyName, MapDataIterator.of(rows)); } - protected void writeRunProperties(AssayRunUploadContext context, Map runProperties, FileLike scriptDir, PrintWriter pw, TSVWriter writer) + protected void writeRunProperties( + DataTransformService.TransformOperation operation, + AssayRunUploadContext context, + Map runProperties, + FileLike scriptDir, + PrintWriter pw, + TSVWriter writer + ) { // serialize the run properties to a tsv for (Map.Entry entry : runProperties.entrySet()) @@ -440,7 +459,7 @@ protected void writeRunProperties(AssayRunUploadContext } // additional context properties - for (Map.Entry entry : getContextProperties(context, scriptDir).entrySet()) + for (Map.Entry entry : getContextProperties(operation, context, scriptDir).entrySet()) { pw.append(writer.quoteValue(entry.getKey())); pw.append('\t'); @@ -450,7 +469,7 @@ protected void writeRunProperties(AssayRunUploadContext } } - public Map getRunProperties(AssayRunUploadContext context) throws ExperimentException + private Map getRunProperties(AssayRunUploadContext context) throws ExperimentException { Map runProperties = new HashMap<>(context.getRunProperties()); for (Map.Entry entry : runProperties.entrySet()) @@ -460,12 +479,19 @@ public Map getRunProperties(AssayRunUploadContext getContextProperties(AssayRunUploadContext context, FileLike scriptDir) + private Map getContextProperties(DataTransformService.TransformOperation operation, AssayRunUploadContext context, FileLike scriptDir) { Map map = new HashMap<>(); - map.put(Props.assayId.name(), StringUtils.defaultString(context.getName())); - map.put(Props.runComments.name(), StringUtils.defaultString(context.getComments())); + if (operation == DataTransformService.TransformOperation.INSERT) + { + map.put(Props.assayId.name(), StringUtils.defaultString(context.getName())); + map.put(Props.runComments.name(), StringUtils.defaultString(context.getComments())); + File originalFileLocation = context.getOriginalFileLocation(); + if (originalFileLocation != null) + map.put(Props.originalFileLocation.name(), originalFileLocation.getPath()); + } + map.put(Props.baseUrl.name(), AppProps.getInstance().getBaseServerUrl() + AppProps.getInstance().getContextPath()); map.put(Props.containerPath.name(), context.getContainer().getPath()); map.put(Props.assayType.name(), context.getProvider().getName()); @@ -475,21 +501,16 @@ private Map getContextProperties(AssayRunUploadContext contex map.put(Props.protocolId.name(), String.valueOf(context.getProtocol().getRowId())); map.put(Props.protocolDescription.name(), StringUtils.defaultString(context.getProtocol().getDescription())); map.put(Props.protocolLsid.name(), context.getProtocol().getLSID()); - File originalFileLocation = context.getOriginalFileLocation(); - if (originalFileLocation != null) - { - map.put(Props.originalFileLocation.name(), originalFileLocation.getPath()); - } return map; } - public RunInfo processRunInfo(FileLike runInfoFileLike) throws ValidationException + private RunInfo processWarningsFile(FileLike runInfoFile) throws ValidationException { RunInfo info = new RunInfo(); - if (runInfoFileLike.exists()) + if (runInfoFile.exists()) { - try (TabLoader loader = new TabLoader(runInfoFileLike.toNioPathForRead().toFile(), false)) + try (TabLoader loader = new TabLoader(runInfoFile.toNioPathForRead().toFile(), false)) { // Don't unescape file path names on windows (C:\foo\bar.tsv) loader.setUnescapeBackslashes(false); @@ -515,7 +536,6 @@ public RunInfo processRunInfo(FileLike runInfoFileLike) throws ValidationExcepti { throw new ValidationException(e.getMessage()); } - } return info; } @@ -560,19 +580,8 @@ private void processWarningsOutput( @Nullable File transformedFile, List files) throws ValidationException { - String maxSeverity = null; - - if (null != transformedProps) - { - for (Map.Entry row : transformedProps.entrySet()) - { - if (row.getKey().equals(Props.maximumSeverity.name())) - { - maxSeverity = row.getValue(); - break; - } - } - } + String warningSevLevel = info.getWarningSevLevel(); + String maxSeverity = transformedProps != null ? transformedProps.get(Props.maximumSeverity.name()) : null; // Look for error file and get contents String warning = null; @@ -594,7 +603,7 @@ private void processWarningsOutput( } // Display warnings case - if (null != info.getWarningSevLevel() && info.getWarningSevLevel().equals(errLevel.WARN.name()) && null != maxSeverity && maxSeverity.equals(errLevel.WARN.name())) + if (null != warningSevLevel && warningSevLevel.equals(errLevel.WARN.name()) && null != maxSeverity && maxSeverity.equals(errLevel.WARN.name())) { // Running in background does not support warnings if (info.isBackgroundUpload()) @@ -637,7 +646,7 @@ else if (null != maxSeverity && maxSeverity.equals(errLevel.ERROR.name())) } } - public void processValidationOutput(RunInfo info, @Nullable Logger log) throws ValidationException + private void processValidationOutput(RunInfo info, @Nullable Logger log) throws ValidationException { List errors = new ArrayList<>(); @@ -676,14 +685,13 @@ else if ("warn".equalsIgnoreCase(row.get("type").toString()) && log != null) if (!errors.isEmpty()) throw new ValidationException(errors); - } } /** * Ensures the property name recorded maps to a valid form field name */ - protected String mapPropertyName(String name) + private String mapPropertyName(String name) { if (Props.assayId.name().equals(name)) return "name"; @@ -696,7 +704,12 @@ protected String mapPropertyName(String name) } @Override - public void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewContext, FileLike scriptDir) throws Exception + public void createSampleData( + DataTransformService.TransformOperation operation, + @NotNull ExpProtocol protocol, + ViewContext viewContext, + FileLike scriptDir + ) throws Exception { final int SAMPLE_DATA_ROWS = 5; FileLike runProps = scriptDir.resolveChild(VALIDATION_RUN_INFO_FILE); @@ -708,48 +721,54 @@ public void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewCont { AssayRunUploadContext context = new SampleRunUploadContext(protocol, viewContext); - writeRunProperties(context, context.getRunProperties(), scriptDir, pw, writer); + writeRunProperties(operation, context, context.getRunProperties(), scriptDir, pw, writer); // create the sample run data AssayProvider provider = AssayService.get().getProvider(protocol); List> dataRows = new ArrayList<>(); - Domain runDataDomain = provider.getResultsDomain(protocol); - if (runDataDomain != null) + if (operation == DataTransformService.TransformOperation.INSERT) { - List properties = runDataDomain.getProperties(); - for (int i = 0; i < SAMPLE_DATA_ROWS; i++) + Domain runDataDomain = provider.getResultsDomain(protocol); + if (runDataDomain != null) { - Map row = new HashMap<>(); - for (DomainProperty prop : properties) - row.put(prop.getName(), getSampleValue(prop)); + List properties = runDataDomain.getProperties(); + for (int i = 0; i < SAMPLE_DATA_ROWS; i++) + { + Map row = new HashMap<>(); + for (DomainProperty prop : properties) + row.put(prop.getName(), getSampleValue(prop)); + + dataRows.add(row); + } + FileLike runData = scriptDir.resolveChild(RUN_DATA_FILE); + pw.append(Props.runDataFile.name()); + pw.append('\t'); + pw.println(runData.toNioPathForRead().toString()); - dataRows.add(row); + getDataSerializer().exportRunData(protocol, Collections.singletonList(MapDataIterator.of(dataRows)), runData, writer); } - FileLike runData = scriptDir.resolveChild(RUN_DATA_FILE); - pw.append(Props.runDataFile.name()); - pw.append('\t'); - pw.println(runData.toNioPathForRead().toString()); - getDataSerializer().exportRunData(protocol, Collections.singletonList(MapDataIterator.of(dataRows)), runData, writer); + // any additional sample property sets + for (Map.Entry set : _sampleProperties.entrySet()) + { + FileLike sampleData = scriptDir.resolveChild(set.getKey() + ".tsv"); + getDataSerializer().exportRunData(protocol, Collections.singletonList(set.getValue()), sampleData, writer); + + pw.append(set.getKey()); + pw.append('\t'); + pw.println(sampleData.toNioPathForRead().toString()); + } } - // any additional sample property sets - for (Map.Entry set : _sampleProperties.entrySet()) + // errors file location + if (operation == DataTransformService.TransformOperation.INSERT) { - FileLike sampleData = scriptDir.resolveChild(set.getKey() + ".tsv"); - getDataSerializer().exportRunData(protocol, Collections.singletonList(set.getValue()), sampleData, writer); - - pw.append(set.getKey()); + FileLike errorFile = scriptDir.resolveChild(ERRORS_FILE); + pw.append(Props.errorsFile.name()); pw.append('\t'); - pw.println(sampleData.toNioPathForRead().toString()); + pw.println(errorFile.toNioPathForRead().toString()); } - - // errors file location - FileLike errorFile = scriptDir.resolveChild(ERRORS_FILE); - pw.append(Props.errorsFile.name()); - pw.append('\t'); - pw.println(errorFile.toNioPathForRead().toString()); } } @@ -766,7 +785,7 @@ protected int write() }; } - protected List> parseRunInfo(File runInfo) + private List> parseRunInfo(File runInfo) { try (TabLoader loader = new TabLoader(runInfo, false)) { @@ -787,7 +806,7 @@ protected boolean isIgnorableOutput(File file) return _filesToIgnore.contains(file); } - public static String directoryKey(AssayRunUploadContext context) + private static String directoryKey(AssayRunUploadContext context) { if (context instanceof ProtocolIdForm && null != ((ProtocolIdForm) context).getUploadAttemptID()) return ((ProtocolIdForm) context).getUploadAttemptID(); @@ -796,7 +815,8 @@ public static String directoryKey(AssayRunUploadContext context) } @Nullable - public static FileLike getWorkingDirectory(AssayRunUploadContext context) { + private FileLike getWorkingDirectory(AssayRunUploadContext context) + { Pair containerFilePair = workingDirectories.get(directoryKey(context)); if (containerFilePair != null && containerFilePair.first.hasPermission("TsvDataExchangeHandler.getWorkingDirectory()", context.getUser(), ReadPermission.class)) @@ -807,7 +827,8 @@ public static FileLike getWorkingDirectory(AssayRunUploadContext context) { } @Nullable - public static File getWorkingDirectory(ProtocolIdForm form, User u) { + public static File getWorkingDirectory(ProtocolIdForm form, User u) + { Pair containerFilePair = workingDirectories.get(form.getUploadAttemptID()); if (containerFilePair != null && containerFilePair.first.hasPermission("TsvDataExchangeHandler.getWorkingDirectory()", u, ReadPermission.class)) @@ -818,7 +839,8 @@ public static File getWorkingDirectory(ProtocolIdForm form, User u) { } @Nullable - public static File removeWorkingDirectory(AssayRunUploadContext context) { + public static File removeWorkingDirectory(AssayRunUploadContext context) + { Pair containerFilePair = workingDirectories.get(directoryKey(context)); if (containerFilePair != null && containerFilePair.first.hasPermission("TsvDataExchangeHandler.removeWorkingDirectory()", context.getUser(), ReadPermission.class)) @@ -830,7 +852,8 @@ public static File removeWorkingDirectory(AssayRunUploadContext context) { } @Nullable - public static File removeWorkingDirectory(ProtocolIdForm form, User u) { + public static File removeWorkingDirectory(ProtocolIdForm form, User u) + { Pair containerFilePair = workingDirectories.get(form.getUploadAttemptID()); if (containerFilePair != null && containerFilePair.first.hasPermission("TsvDataExchangeHandler.removeWorkingDirectory()", u, ReadPermission.class)) @@ -841,71 +864,24 @@ public static File removeWorkingDirectory(ProtocolIdForm form, User u) { return null; } - public static void setWorkingDirectory(AssayRunUploadContext context, File dir) { + private void setWorkingDirectory(AssayRunUploadContext context, File dir) + { if (context.getContainer().hasPermission("TsvDataExchangeHandler.setWorkingDirectory()", context.getUser(), ReadPermission.class)) { workingDirectories.put(directoryKey(context), new Pair<>(context.getContainer(), dir)); } } - private class RunInfo { - private File _errorFile; - private String _warningSevLevel; - private File _originalFileLocation; - private ExpProtocol _protocol; - - public void setProtocol(ExpProtocol protocol) - { - _protocol = protocol; - } - - public boolean isSaveScriptFiles() - { - AssayProvider provider = AssayService.get().getProvider(_protocol); - return provider.isSaveScriptFiles(_protocol); - } - - public boolean isBackgroundUpload() - { - AssayProvider provider = AssayService.get().getProvider(_protocol); - return provider.isBackgroundUpload(_protocol); - } - - // Original file location used to determine if file was already on the server or not. Used for cleanup when - // there is an error. - public File getOriginalFileLocation() - { - return _originalFileLocation; - } - - public void setOriginalFileLocation(File originalFileLocation) - { - _originalFileLocation = originalFileLocation; - } - - public File getErrorFile() - { - return _errorFile; - } - - public void setErrorFile(File errorFile) - { - _errorFile = errorFile; - } - - public String getWarningSevLevel() - { - return _warningSevLevel; - } - - public void setWarningSevLevel(String warningSevLevel) - { - _warningSevLevel = warningSevLevel; - } - } - @Override - public TransformResult processTransformationOutput(AssayRunUploadContext context, FileLike runInfo, ExpRun run, FileLike scriptFile, TransformResult mergeResult, Set inputDataFiles) throws ValidationException + public TransformResult processTransformationOutput( + DataTransformService.TransformOperation operation, + AssayRunUploadContext context, + FileLike runInfo, + @Nullable ExpRun run, + FileLike scriptFile, + TransformResult mergeResult, + Set inputDataFiles + ) throws ValidationException { Logger log = context.getLogger(); if (log == null) @@ -915,49 +891,54 @@ public TransformResult processTransformationOutput(AssayRunUploadContext outputData = DefaultAssayRunCreator.createdRelatedOutputData(context, baseName, targetFile); - if (outputData != null) + if (run != null) { - outputData.getKey().setSourceApplication(scriptPA); - outputData.getKey().save(context.getUser()); + Pair outputData = DefaultAssayRunCreator.createdRelatedOutputData(context, baseName, targetFile); + if (outputData != null) + { + outputData.getKey().setSourceApplication(scriptPA); + outputData.getKey().save(context.getUser()); - outputProtocolApplication.addDataInput(context.getUser(), outputData.getKey(), outputData.getValue()); + outputProtocolApplication.addDataInput(context.getUser(), outputData.getKey(), outputData.getValue()); + } } } } @@ -1171,13 +1158,15 @@ else if (String.valueOf(row.get("name")).equalsIgnoreCase(Props.runDataUploadedF if (transformedProps.containsKey(Props.runComments.name())) result.setComments(transformedProps.get(Props.runComments.name())); } + if (!runDataUploadedFiles.isEmpty()) result.setUploadedFiles(FileSystemLike.wrapFiles(runDataUploadedFiles)); // Don't offer up input or other files as "outputs" of the script tempOutputFiles.removeAll(_filesToIgnore); - processWarningsOutput(result, transformedProps, info, transErrorFile, transformedFile, tempOutputFiles); + if (operation == DataTransformService.TransformOperation.INSERT) + processWarningsOutput(result, transformedProps, info, transErrorFile, transformedFile, tempOutputFiles); } catch (ValidationException e) { @@ -1206,6 +1195,63 @@ protected static String getSampleValue(DomainProperty prop) }; } + private static class RunInfo + { + private File _errorFile; + private String _warningSevLevel; + private File _originalFileLocation; + private ExpProtocol _protocol; + + public void setProtocol(ExpProtocol protocol) + { + _protocol = protocol; + } + + public boolean isSaveScriptFiles() + { + AssayProvider provider = AssayService.get().getProvider(_protocol); + return provider.isSaveScriptFiles(_protocol); + } + + public boolean isBackgroundUpload() + { + AssayProvider provider = AssayService.get().getProvider(_protocol); + return provider.isBackgroundUpload(_protocol); + } + + // Original file location used to determine if file was already on the server or not. Used for cleanup when + // there is an error. + public File getOriginalFileLocation() + { + return _originalFileLocation; + } + + public void setOriginalFileLocation(File originalFileLocation) + { + _originalFileLocation = originalFileLocation; + } + + public File getErrorFile() + { + return _errorFile; + } + + public void setErrorFile(File errorFile) + { + _errorFile = errorFile; + } + + public String getWarningSevLevel() + { + return _warningSevLevel; + } + + public void setWarningSevLevel(String warningSevLevel) + { + _warningSevLevel = warningSevLevel; + } + } + private static class SampleRunUploadContext implements AssayRunUploadContext { ExpProtocol _protocol; diff --git a/assay/api-src/org/labkey/api/assay/AssayResultUpdateService.java b/assay/api-src/org/labkey/api/assay/AssayResultUpdateService.java index d940cd31e3c..d017bc32212 100644 --- a/assay/api-src/org/labkey/api/assay/AssayResultUpdateService.java +++ b/assay/api-src/org/labkey/api/assay/AssayResultUpdateService.java @@ -15,22 +15,35 @@ */ package org.labkey.api.assay; +import jakarta.servlet.http.HttpServletRequest; import org.apache.commons.beanutils.ConversionException; import org.apache.commons.beanutils.ConvertUtils; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.sample.AssaySampleLookupContext; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.assay.transform.DefaultTransformResult; +import org.labkey.api.assay.transform.TransformResult; +import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.OntologyObject; import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.exp.api.ProvenanceService; +import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultQueryUpdateService; import org.labkey.api.query.FieldKey; @@ -42,10 +55,17 @@ import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.Permission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.view.ActionURL; import org.labkey.api.view.UnauthorizedException; +import org.labkey.vfs.FileLike; +import java.io.IOException; import java.nio.file.Path; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -55,6 +75,7 @@ public class AssayResultUpdateService extends DefaultQueryUpdateService { private final AssaySampleLookupContext _assaySampleLookupContext; + private final AssayProtocolSchema _schema; public AssayResultUpdateService(AssayProtocolSchema schema, FilteredTable table) { @@ -63,6 +84,7 @@ public AssayResultUpdateService(AssayProtocolSchema schema, FilteredTable table) throw new IllegalArgumentException("Expected AssayResultTable"); _assaySampleLookupContext = new AssaySampleLookupContext(); + _schema = schema; } @Override @@ -76,6 +98,8 @@ public List> updateRows( Map extraScriptContext ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException { + // handle transform scripts + rows = transform(container, user, rows); var result = super.updateRows(user, container, rows, oldKeys, errors, configParameters, extraScriptContext); _assaySampleLookupContext.syncLineage(container, user, errors); @@ -86,6 +110,128 @@ public List> updateRows( return result; } + private List> transform( + Container container, + User user, + List> rows + ) throws BatchValidationException, InvalidKeyException, QueryUpdateServiceException, SQLException + { + try + { + List> rowsForTransform = resolveRows(container, user, rows); + AssayTransformContext context = new AssayTransformContext(container, user, rowsForTransform, _schema.getProtocol(), _schema.getProvider()); + TransformResult result = DataTransformService.get().transformAndValidate(context, null, DataTransformService.TransformOperation.UPDATE); + Map transformedData = result.getTransformedData(); + + if (!transformedData.isEmpty()) + { + ColumnInfo keyCol = null; + for (ColumnInfo colInfo : getDbTable().getPkColumns()) + { + if (rows.get(0).containsKey(colInfo.getName())) + { + keyCol = colInfo; + break; + } + } + + if (keyCol == null) + throw new BatchValidationException((new ValidationException(String.format("The data does not contain a key field value for table : %s.", getQueryTable().getName())))); + + // merge any existing data with transformed rows + Map> newData = new LinkedHashMap<>(); + for (Map row : rows) + { + if (row.containsKey(keyCol.getName())) + newData.put(row.get(keyCol.getName()), new HashMap<>(row)); + else + throw new BatchValidationException(new ValidationException(String.format("Unable to find the key value : %s for a row being updated.", keyCol.getName()))); + } + + boolean dataTypeHandled = false; + for (Map.Entry entry : transformedData.entrySet()) + { + ExpData data = entry.getKey(); + + // match the transformed data by data types + if (data.getDataType().equals(context.getProvider().getDataType())) + { + boolean mergeData = false; + if (dataTypeHandled) + throw new BatchValidationException(new ValidationException(String.format("There was more than one transformed file found for the data type : %s.", context.getProvider().getDataType()))); + dataTypeHandled = true; + + try (var it = DataIteratorUtil.wrapMap(entry.getValue().getDataIterator(new DataIteratorContext()), false)) + { + while (it.next()) + { + // merge with original updated rows + Map row = it.getMap(); + Object key = row.get(keyCol.getName()); + if (key != null) + { + if (newData.containsKey(key)) + mergeData = true; + newData.put(key, new HashMap<>(row)); + } + else + throw new BatchValidationException(new ValidationException(String.format("Unable to find the key value : %s for a transformed data row.", keyCol.getName()))); + } + + if (mergeData) + { + // replace with merged data + rows = new ArrayList<>(newData.values()); + } + } + } + } + } + return rows; + } + catch (ValidationException ve) + { + throw new BatchValidationException(ve); + } + catch (IOException ioe) + { + throw new BatchValidationException(new ValidationException(ioe.getMessage())); + } + } + + /** + * Merge existing values with the rows being updated prior to handing off to any + * transform scripts. This is necessary because a transform script will need to see all + * values for each row (not just the changed values). + */ + private List> resolveRows( + Container container, + User user, + List> rows + ) throws InvalidKeyException, QueryUpdateServiceException, SQLException, ValidationException + { + Map columnInfoMap = new CaseInsensitiveHashMap<>(); + getQueryTable().getColumns().forEach(ci -> columnInfoMap.put(ci.getName(), ci)); + + for (Map row : rows) + { + var oldRow = getRow(user, container, row); + if (oldRow == null) + throw new ValidationException("Unable to find existing row"); + + for (Map.Entry entry : oldRow.entrySet()) + { + ColumnInfo col = columnInfoMap.get(entry.getKey()); + if (col != null && !row.containsKey(entry.getKey())) + { + // use column names for existing row values + row.put(col.getName(), entry.getValue()); + } + } + } + return rows; + } + @Override protected Map updateRow( User user, @@ -246,4 +392,139 @@ private void appendPropertyIfChanged(StringBuilder sb, String label, Object oldV sb.append(newValue == null ? "blank" : "'" + newValue + "'"); sb.append("."); } + + /** + * Context used during data transforms for update operations + */ + private static class AssayTransformContext implements AssayRunUploadContext + { + private final Container _container; + private final User _user; + private final ExpProtocol _protocol; + private final AssayProvider _provider; + private TransformResult _transformResult; + private final DataIteratorBuilder _data; + + public AssayTransformContext(Container container, User user, List> rows, ExpProtocol protocol, AssayProvider provider) + { + _container = container; + _user = user; + _protocol = protocol; + _provider = provider; + _data = MapDataIterator.of(rows); + } + + @Override + public @NotNull ExpProtocol getProtocol() + { + return _protocol; + } + + @Override + public Map getRunProperties() + { + return Collections.emptyMap(); + } + + @Override + public Map getBatchProperties() + { + return Collections.emptyMap(); + } + + @Override + public String getComments() + { + return null; + } + + @Override + public String getName() + { + return null; + } + + @Override + public User getUser() + { + return _user; + } + + @Override + public @NotNull Container getContainer() + { + return _container; + } + + @Override + public @Nullable HttpServletRequest getRequest() + { + return null; + } + + @Override + public ActionURL getActionURL() + { + return null; + } + + @Override + public @NotNull Map getUploadedData() + { + return Collections.emptyMap(); + } + + @Override + public @Nullable DataIteratorBuilder getRawData() + { + return _data; + } + + @Override + public @NotNull Map getInputDatas() + { + return Collections.emptyMap(); + } + + @Override + public AssayProvider getProvider() + { + return _provider; + } + + @Override + public String getTargetStudy() + { + return null; + } + + @Override + public TransformResult getTransformResult() + { + return _transformResult != null ? _transformResult : DefaultTransformResult.createEmptyResult(); + } + + @Override + public void setTransformResult(TransformResult result) + { + _transformResult = result; + } + + @Override + public Integer getReRunId() + { + return null; + } + + @Override + public void uploadComplete(ExpRun run) throws ExperimentException + { + } + + @Override + public @Nullable Logger getLogger() + { + return null; + } + } } diff --git a/assay/api-src/org/labkey/api/assay/dilution/DilutionDataExchangeHandler.java b/assay/api-src/org/labkey/api/assay/dilution/DilutionDataExchangeHandler.java index ca885af4e28..2d339f07b87 100644 --- a/assay/api-src/org/labkey/api/assay/dilution/DilutionDataExchangeHandler.java +++ b/assay/api-src/org/labkey/api/assay/dilution/DilutionDataExchangeHandler.java @@ -17,8 +17,10 @@ package org.labkey.api.assay.dilution; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.WellGroup; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.property.DomainProperty; @@ -43,7 +45,14 @@ public class DilutionDataExchangeHandler extends PlateBasedDataExchangeHandler { @Override - public Pair> createTransformationRunInfo(AssayRunUploadContext context, ExpRun run, FileLike scriptDir, Map runProperties, Map batchProperties) throws Exception + public Pair> createTransformationRunInfo( + DataTransformService.TransformOperation operation, + AssayRunUploadContext context, + @Nullable ExpRun run, + FileLike scriptDir, + Map runProperties, + Map batchProperties + ) throws Exception { DilutionRunUploadForm form = (DilutionRunUploadForm)context; @@ -57,11 +66,16 @@ public Pair> createTransformationRunInfo(AssayRunUploadC addSampleProperties(SAMPLE_DATA_PROP_NAME, GROUP_COLUMN_NAME, props, template, WellGroup.Type.SPECIMEN); - return super.createTransformationRunInfo(context, run, scriptDir, runProperties, batchProperties); + return super.createTransformationRunInfo(operation, context, run, scriptDir, runProperties, batchProperties); } @Override - public void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewContext, FileLike scriptDir) throws Exception + public void createSampleData( + DataTransformService.TransformOperation operation, + @NotNull ExpProtocol protocol, + ViewContext viewContext, + FileLike scriptDir + ) throws Exception { AssayProvider provider = AssayService.get().getProvider(protocol); if (provider instanceof AbstractPlateBasedAssayProvider) @@ -74,6 +88,6 @@ public void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewCont addSampleProperties(SAMPLE_DATA_PROP_NAME, GROUP_COLUMN_NAME, specimens, template, WellGroup.Type.SPECIMEN); } - super.createSampleData(protocol, viewContext, scriptDir); + super.createSampleData(operation, protocol, viewContext, scriptDir); } } diff --git a/assay/src/org/labkey/assay/AssayController.java b/assay/src/org/labkey/assay/AssayController.java index 27d75698a27..b30d2ba4321 100644 --- a/assay/src/org/labkey/assay/AssayController.java +++ b/assay/src/org/labkey/assay/AssayController.java @@ -63,6 +63,8 @@ import org.labkey.api.assay.plate.PlateBasedAssayProvider; import org.labkey.api.assay.sample.AssaySampleLookupContext; import org.labkey.api.assay.security.DesignAssayPermission; +import org.labkey.api.assay.transform.DataExchangeHandler; +import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.audit.AuditLogService; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; @@ -95,7 +97,6 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.assay.transform.DataExchangeHandler; import org.labkey.api.qc.DataState; import org.labkey.api.qc.DataStateManager; import org.labkey.api.query.BatchValidationException; @@ -912,7 +913,7 @@ public ModelAndView getView(ProtocolIdForm form, BindException errors) throws Ex FileLike tempDir = getTempFolder(); try { - handler.createSampleData(protocol, getViewContext(), tempDir); + handler.createSampleData(DataTransformService.TransformOperation.INSERT, protocol, getViewContext(), tempDir); File[] files = tempDir.toNioPathForRead().toFile().listFiles(); if (files.length > 0) diff --git a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java index 77ab3f7533b..c2f0ebfa9c0 100644 --- a/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java +++ b/assay/src/org/labkey/assay/plate/AssayPlateMetadataServiceImpl.java @@ -54,6 +54,7 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.data.statistics.MathStat; import org.labkey.api.data.statistics.StatsService; +import org.labkey.api.dataiterator.DataIterator; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; import org.labkey.api.dataiterator.DataIteratorUtil; @@ -225,6 +226,24 @@ public Map apply(Map row) }); } + private List getPlatesForPlateSet( + Container container, + User user, + Integer plateSetId, + ExpProtocol protocol + ) throws ExperimentException + { + // get the ordered list of plates for the plate set + ContainerFilter cf = PlateManager.get().getPlateContainerFilter(protocol, container, user); + PlateSet plateSet = PlateManager.get().getPlateSet(cf, plateSetId); + if (plateSet == null) + throw new ExperimentException("Plate set " + plateSetId + " not found."); + if (plateSet.isTemplate()) + throw new ExperimentException(String.format("Plate set \"%s\" is a template plate set. Template plate sets do not support associating assay data.", plateSet.getName())); + + return PlateManager.get().getPlatesForPlateSet(plateSet); + } + @Override public DataIteratorBuilder parsePlateData( Container container, @@ -239,33 +258,20 @@ public DataIteratorBuilder parsePlateData( ) throws ExperimentException { // get the ordered list of plates for the plate set - ContainerFilter cf = PlateManager.get().getPlateContainerFilter(protocol, container, user); - PlateSet plateSet = PlateManager.get().getPlateSet(cf, plateSetId); - if (plateSet == null) - throw new ExperimentException("Plate set " + plateSetId + " not found."); - if (plateSet.isTemplate()) - throw new ExperimentException(String.format("Plate set \"%s\" is a template plate set. Template plate sets do not support associating assay data.", plateSet.getName())); - - List plates = PlateManager.get().getPlatesForPlateSet(plateSet); + List plates = getPlatesForPlateSet(container, user, plateSetId, protocol); if (plates.isEmpty()) throw new ExperimentException("No plates were found for the plate set (" + plateSetId + ")."); + PlateSet plateSet = plates.get(0).getPlateSet(); List> rows = _parsePlateData(container, user, data, provider, protocol, plateSet, plates, dataFile, settings); - if (context.getReRunId() != null) + if (context.getReRunId() != null && context.getReImportOption() != MERGE_DATA) { - // check if we are merging the re-imported data - if (context.getReImportOption() == MERGE_DATA) - rows = mergeReRunData(container, user, context, rows, plates, provider, protocol, data, dataFile); - else - { - // remove hit selections from the replaced run - ExpRun prevRun = ExperimentService.get().getExpRun(context.getReRunId()); - if (prevRun != null) - PlateManager.get().deleteHits(FieldKey.fromParts("RunId"), List.of(prevRun)); - } + // remove hit selections if we are replacing a run + ExpRun prevRun = ExperimentService.get().getExpRun(context.getReRunId()); + if (prevRun != null) + PlateManager.get().deleteHits(FieldKey.fromParts("RunId"), List.of(prevRun)); } - return MapDataIterator.of(rows); } @@ -311,29 +317,37 @@ private List> _parsePlateData( } } - /** - * Takes the current incoming data and combines it with any data uploaded in the previous run (re-run ID). Data - * can be combined for plates within a plate set, but only on a per plate boundary. If there is data for plates - * in both sets of data, the most recent data will take precedence. - * - * @param rows The incoming data rows - * @param plates The list of plates in this plate set - * @param data The ExpData object for this run - * @param dataFile The current uploaded file - * @return The new, combined data - */ - private List> mergeReRunData( - Container container, - User user, - @NotNull AssayRunUploadContext context, - List> rows, - List plates, - AssayProvider provider, - ExpProtocol protocol, - ExpData data, - FileLike dataFile + @Override + public @Nullable Integer getPlateSetId(AssayRunUploadContext context, AssayProvider provider, ExpProtocol protocol) throws ExperimentException + { + Domain runDomain = provider.getRunDomain(protocol); + DomainProperty propertyPlateSet = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_SET_COLUMN_NAME); + if (propertyPlateSet == null) + { + throw new ExperimentException("The assay run domain for the assay '" + protocol.getName() + "' does not contain a plate set property."); + } + + Map runProps = context.getRunProperties(); + Object plateSetVal = runProps.getOrDefault(propertyPlateSet, null); + return plateSetVal != null ? Integer.parseInt(String.valueOf(plateSetVal)) : null; + } + + @Override + public DataIteratorBuilder mergeReRunData( + Container container, + User user, + @NotNull AssayRunUploadContext context, + DataIterator resultData, + AssayProvider provider, + ExpProtocol protocol, + ExpData data ) throws ExperimentException { + Integer plateSetId = getPlateSetId(context, provider, protocol); + List plates = getPlatesForPlateSet(container, user, plateSetId, protocol); + if (plates.isEmpty()) + throw new ExperimentException("No plates were found for the plate set (" + plateSetId + ")."); + ExpRun run = ExperimentService.get().getExpRun(context.getReRunId()); if (run == null) throw new ExperimentException(String.format("Unable to resolve the replaced run with ID : %d", context.getReRunId())); @@ -346,6 +360,7 @@ private List> mergeReRunData( plateMap.put(p.getPlateId(), p); } + List> rows = resultData.stream().toList(); Set incomingPlates = new HashSet<>(); // incoming plates may be either row IDs or plate IDs for (var row : rows) { @@ -363,7 +378,7 @@ private List> mergeReRunData( // The plate identifier is either a row ID or plate ID on incoming data, need to match that when merging existing data. FieldKey plateFieldKey = FieldKey.fromParts(AssayResultDomainKind.Column.Plate.name()); // Note that in the case where there is a transform script on the assay design, the LK data parsing might not have - // found any rows and we might be deferring to the transform script to do that parsing. This block of code should + // found any rows, and we might be deferring to the transform script to do that parsing. This block of code should // be able to proceed in that case by just passing through all run results to the transform script for the run being replaced. if (!rows.isEmpty()) { @@ -418,6 +433,8 @@ private List> mergeReRunData( { try (DbScope.Transaction tx = AssayDbSchema.getInstance().getScope().ensureTransaction()) { + FileLike dataFile = data.getFileLike(); + // replace the contents of the uploaded data file with the new combined data FileLike dir = dataFile.getParent() != null ? dataFile.getParent() : AssayFileWriter.ensureUploadDirectory(container); String newName = FileUtil.getBaseName(dataFile.toNioPathForRead().toFile()) + ".tsv"; @@ -468,7 +485,7 @@ private List> mergeReRunData( if (!prevPlateRowIDs.isEmpty()) rows = newRows; - return rows; + return MapDataIterator.of(rows); } /** From 1eb464a7383f0eed261f5be5e00dd8192b3c0931 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 9 Jan 2025 10:50:19 -0800 Subject: [PATCH 4/4] Add ManageVersion property to all module.properties files (#6203) --- .gitattributes | 10 ---------- announcements/module.properties | 1 + api/module.properties | 1 + assay/module.properties | 1 + audit/module.properties | 1 + core/module.properties | 1 + devtools/module.properties | 1 + experiment/module.properties | 1 + filecontent/module.properties | 1 + issues/module.properties | 1 + list/module.properties | 1 + mothership/module.properties | 1 + pipeline/module.properties | 1 + query/module.properties | 1 + search/module.properties | 1 + specimen/module.properties | 1 + study/module.properties | 1 + survey/module.properties | 2 +- timeline/module.properties | 1 + visualization/module.properties | 1 + wiki/module.properties | 1 + 21 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.gitattributes b/.gitattributes index e661498e54e..30e4f4c4428 100644 --- a/.gitattributes +++ b/.gitattributes @@ -274,7 +274,6 @@ api/src/org/labkey/api/assay/AssayWarningsDisplayColumn.java -text api/src/org/labkey/api/assay/AssayWellExclusionService.java -text api/src/org/labkey/api/assay/bulkPropertiesInput.jsp -text api/src/org/labkey/api/assay/DefaultAssayRunCreator.java -text -api/src/org/labkey/api/assay/DefaultDataTransformer.java -text api/src/org/labkey/api/assay/fileUpload.jsp -text api/src/org/labkey/api/assay/FileUploadDataCollector.java -text api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java -text @@ -932,12 +931,7 @@ api/src/org/labkey/api/pipeline/WorkDirFactory.java -text api/src/org/labkey/api/pipeline/XMLBeanTaskFactoryFactory.java -text api/src/org/labkey/api/portal/ProjectUrls.java -text api/src/org/labkey/api/premium/AntiVirusService.java -text -api/src/org/labkey/api/qc/DataExchangeHandler.java -text api/src/org/labkey/api/qc/DataLoaderSettings.java -text -api/src/org/labkey/api/qc/DataTransformer.java -text -api/src/org/labkey/api/qc/DefaultTransformResult.java -text -api/src/org/labkey/api/qc/TransformDataHandler.java -text -api/src/org/labkey/api/qc/TransformResult.java -text api/src/org/labkey/api/qc/TsvDataExchangeHandler.java -text api/src/org/labkey/api/qc/TsvDataSerializer.java -text api/src/org/labkey/api/qc/ValidationDataHandler.java -text @@ -1248,7 +1242,6 @@ api/src/org/labkey/api/security/SecurityUrls.java -text api/src/org/labkey/api/security/SessionApiKeyManager.java -text api/src/org/labkey/api/security/SessionKeyManager.java -text api/src/org/labkey/api/security/SessionReplacingRequest.java -text -api/src/org/labkey/api/security/TokenAuthenticationManager.java -text api/src/org/labkey/api/security/UserCache.java -text api/src/org/labkey/api/security/UserManager.java -text api/src/org/labkey/api/security/UserPrincipal.java -text @@ -2836,7 +2829,6 @@ study/src/org/labkey/study/query/VisitUpdateService.java -text study/src/org/labkey/study/reports/AssayProgressReport.java -text study/src/org/labkey/study/reports/BaseStudyView.java -text study/src/org/labkey/study/reports/DefaultAssayProgressSource.java -text -study/src/org/labkey/study/reports/ExternalReport.java -text study/src/org/labkey/study/reports/ParticipantReport.java -text study/src/org/labkey/study/reports/ParticipantReportDescriptor.java -text study/src/org/labkey/study/reports/QueryAssayProgressSource.java -text @@ -2881,7 +2873,6 @@ study/src/org/labkey/study/view/dataspace_studyfilter.jsp -text study/src/org/labkey/study/view/demoMode.jsp -text study/src/org/labkey/study/view/editSnapshot.jsp -text study/src/org/labkey/study/view/editVisit.jsp -text -study/src/org/labkey/study/view/externalReportDesigner.jsp -text study/src/org/labkey/study/view/importStudyBatch.jsp -text study/src/org/labkey/study/view/importVisitAliases.jsp -text study/src/org/labkey/study/view/importVisitMap.jsp -text @@ -3013,7 +3004,6 @@ study/test/src/org/labkey/test/tests/study/StudySimpleExportTest.java -text study/test/src/org/labkey/test/tests/study/StudyTest.java -text study/test/src/org/labkey/test/tests/study/StudyVisitManagementTest.java -text study/test/src/org/labkey/test/tests/study/StudyVisitTagTest.java -text -study/test/src/org/labkey/test/tests/study/VaccineProtocolTest.java -text study/webapp/study/MeasurePicker.js -text study/webapp/study/ParticipantFilterPanel.js -text study/webapp/study/ParticipantGroup.js -text diff --git a/announcements/module.properties b/announcements/module.properties index a0f9d351f05..0c8fc6530fa 100644 --- a/announcements/module.properties +++ b/announcements/module.properties @@ -14,3 +14,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/api/module.properties b/api/module.properties index 604e3f6c626..6fdc9a83118 100644 --- a/api/module.properties +++ b/api/module.properties @@ -8,3 +8,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/assay/module.properties b/assay/module.properties index 7f3b213e30f..510e33a467d 100644 --- a/assay/module.properties +++ b/assay/module.properties @@ -6,3 +6,4 @@ URL: https://www.labkey.org/Documentation/wiki-page.view?name=instrumentData License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/audit/module.properties b/audit/module.properties index 7628f70c0b3..da031be2f65 100644 --- a/audit/module.properties +++ b/audit/module.properties @@ -6,3 +6,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/core/module.properties b/core/module.properties index 4720cb08a5c..0d75d4beb95 100644 --- a/core/module.properties +++ b/core/module.properties @@ -10,3 +10,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/devtools/module.properties b/devtools/module.properties index 0bcb0410f6d..66c280d041d 100644 --- a/devtools/module.properties +++ b/devtools/module.properties @@ -4,3 +4,4 @@ Description: For use during development, not intended for production License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/experiment/module.properties b/experiment/module.properties index 62caf95c011..87f167513ac 100644 --- a/experiment/module.properties +++ b/experiment/module.properties @@ -13,3 +13,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/filecontent/module.properties b/filecontent/module.properties index f32c5c27ba5..fb54e858f5c 100644 --- a/filecontent/module.properties +++ b/filecontent/module.properties @@ -8,3 +8,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/issues/module.properties b/issues/module.properties index af06975cf7f..169fc450c1d 100644 --- a/issues/module.properties +++ b/issues/module.properties @@ -11,3 +11,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/list/module.properties b/list/module.properties index 1cbc6cfba5b..daf8d4a594d 100644 --- a/list/module.properties +++ b/list/module.properties @@ -14,3 +14,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/mothership/module.properties b/mothership/module.properties index 08a8637187e..b72aefd449d 100644 --- a/mothership/module.properties +++ b/mothership/module.properties @@ -5,3 +5,4 @@ Organization: LabKey OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: true diff --git a/pipeline/module.properties b/pipeline/module.properties index ab84a345609..f4b3047a989 100644 --- a/pipeline/module.properties +++ b/pipeline/module.properties @@ -12,3 +12,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/query/module.properties b/query/module.properties index e2a9cdde70b..e6cf64087b8 100644 --- a/query/module.properties +++ b/query/module.properties @@ -6,3 +6,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/search/module.properties b/search/module.properties index 766de71d199..120e6d9f57a 100644 --- a/search/module.properties +++ b/search/module.properties @@ -8,3 +8,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/specimen/module.properties b/specimen/module.properties index 3c8d60197d4..465594d3139 100644 --- a/specimen/module.properties +++ b/specimen/module.properties @@ -6,3 +6,4 @@ URL: https://www.labkey.com/products-services/sample-management-software/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/study/module.properties b/study/module.properties index fad052bd790..daf6cc26d53 100644 --- a/study/module.properties +++ b/study/module.properties @@ -8,3 +8,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/survey/module.properties b/survey/module.properties index e0de1a6f27d..df191053b11 100644 --- a/survey/module.properties +++ b/survey/module.properties @@ -7,5 +7,5 @@ Organization: LabKey OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 - SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/timeline/module.properties b/timeline/module.properties index 18b183c5354..455fff60ce9 100644 --- a/timeline/module.properties +++ b/timeline/module.properties @@ -6,3 +6,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/visualization/module.properties b/visualization/module.properties index 0482f0f2d8f..c131631603d 100644 --- a/visualization/module.properties +++ b/visualization/module.properties @@ -8,3 +8,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true diff --git a/wiki/module.properties b/wiki/module.properties index bb8c20a6728..660b15e57cc 100644 --- a/wiki/module.properties +++ b/wiki/module.properties @@ -9,3 +9,4 @@ OrganizationURL: https://www.labkey.com/ License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 SupportedDatabases: mssql, pgsql +ManageVersion: true