Merge branch 'develop' into fb_add_to_plate

.gitattributes

@@ -274,7 +274,6 @@ api/src/org/labkey/api/assay/AssayWarningsDisplayColumn.java -text
 api/src/org/labkey/api/assay/AssayWellExclusionService.java -text
 api/src/org/labkey/api/assay/bulkPropertiesInput.jsp -text
 api/src/org/labkey/api/assay/DefaultAssayRunCreator.java -text
-api/src/org/labkey/api/assay/DefaultDataTransformer.java -text
 api/src/org/labkey/api/assay/fileUpload.jsp -text
 api/src/org/labkey/api/assay/FileUploadDataCollector.java -text
 api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java -text
@@ -932,12 +931,7 @@ api/src/org/labkey/api/pipeline/WorkDirFactory.java -text
 api/src/org/labkey/api/pipeline/XMLBeanTaskFactoryFactory.java -text
 api/src/org/labkey/api/portal/ProjectUrls.java -text
 api/src/org/labkey/api/premium/AntiVirusService.java -text
-api/src/org/labkey/api/qc/DataExchangeHandler.java -text
 api/src/org/labkey/api/qc/DataLoaderSettings.java -text
-api/src/org/labkey/api/qc/DataTransformer.java -text
-api/src/org/labkey/api/qc/DefaultTransformResult.java -text
-api/src/org/labkey/api/qc/TransformDataHandler.java -text
-api/src/org/labkey/api/qc/TransformResult.java -text
 api/src/org/labkey/api/qc/TsvDataExchangeHandler.java -text
 api/src/org/labkey/api/qc/TsvDataSerializer.java -text
 api/src/org/labkey/api/qc/ValidationDataHandler.java -text
@@ -1248,7 +1242,6 @@ api/src/org/labkey/api/security/SecurityUrls.java -text
 api/src/org/labkey/api/security/SessionApiKeyManager.java -text
 api/src/org/labkey/api/security/SessionKeyManager.java -text
 api/src/org/labkey/api/security/SessionReplacingRequest.java -text
-api/src/org/labkey/api/security/TokenAuthenticationManager.java -text
 api/src/org/labkey/api/security/UserCache.java -text
 api/src/org/labkey/api/security/UserManager.java -text
 api/src/org/labkey/api/security/UserPrincipal.java -text
@@ -2836,7 +2829,6 @@ study/src/org/labkey/study/query/VisitUpdateService.java -text
 study/src/org/labkey/study/reports/AssayProgressReport.java -text
 study/src/org/labkey/study/reports/BaseStudyView.java -text
 study/src/org/labkey/study/reports/DefaultAssayProgressSource.java -text
-study/src/org/labkey/study/reports/ExternalReport.java -text
 study/src/org/labkey/study/reports/ParticipantReport.java -text
 study/src/org/labkey/study/reports/ParticipantReportDescriptor.java -text
 study/src/org/labkey/study/reports/QueryAssayProgressSource.java -text
@@ -2881,7 +2873,6 @@ study/src/org/labkey/study/view/dataspace_studyfilter.jsp -text
 study/src/org/labkey/study/view/demoMode.jsp -text
 study/src/org/labkey/study/view/editSnapshot.jsp -text
 study/src/org/labkey/study/view/editVisit.jsp -text
-study/src/org/labkey/study/view/externalReportDesigner.jsp -text
 study/src/org/labkey/study/view/importStudyBatch.jsp -text
 study/src/org/labkey/study/view/importVisitAliases.jsp -text
 study/src/org/labkey/study/view/importVisitMap.jsp -text
@@ -3013,7 +3004,6 @@ study/test/src/org/labkey/test/tests/study/StudySimpleExportTest.java -text
 study/test/src/org/labkey/test/tests/study/StudyTest.java -text
 study/test/src/org/labkey/test/tests/study/StudyVisitManagementTest.java -text
 study/test/src/org/labkey/test/tests/study/StudyVisitTagTest.java -text
-study/test/src/org/labkey/test/tests/study/VaccineProtocolTest.java -text
 study/webapp/study/MeasurePicker.js -text
 study/webapp/study/ParticipantFilterPanel.js -text
 study/webapp/study/ParticipantGroup.js -text

announcements/module.properties

@@ -14,3 +14,4 @@ OrganizationURL: https://www.labkey.com/
 License: Apache 2.0
 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0
 SupportedDatabases: mssql, pgsql
+ManageVersion: true

api/module.properties

@@ -8,3 +8,4 @@ OrganizationURL: https://www.labkey.com/
 License: Apache 2.0
 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0
 SupportedDatabases: mssql, pgsql
+ManageVersion: true

api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java

@@ -120,6 +120,7 @@
 import java.util.function.Function;
 
 import static java.util.stream.Collectors.toList;
+import static org.labkey.api.assay.AssayRunUploadContext.ReImportOption.MERGE_DATA;
 import static org.labkey.api.exp.OntologyManager.NO_OP_ROW_CALLBACK;
 import static org.labkey.api.gwt.client.ui.PropertyType.SAMPLE_CONCEPT_URI;
 
@@ -237,8 +238,9 @@ private DataIteratorBuilder parsePlateData(
         Container container = context.getContainer();
         User user = context.getUser();
 
-        Integer plateSetId = getPlateSetValueFromRunProps(context, provider, protocol);
-        DataIteratorBuilder dataRows = AssayPlateMetadataService.get().parsePlateData(container, user, ((AssayUploadXarContext)context).getContext(), data, provider,
+        AssayRunUploadContext<?> runUploadContext = ((AssayUploadXarContext)context).getContext();
+        Integer plateSetId = AssayPlateMetadataService.get().getPlateSetId(runUploadContext, provider, protocol);
+        DataIteratorBuilder dataRows = AssayPlateMetadataService.get().parsePlateData(container, user, runUploadContext, data, provider,
                 protocol, plateSetId, dataFile, settings);
 
         // assays with plate metadata support will merge the plate metadata with the data rows to make it easier for
@@ -251,21 +253,6 @@ private DataIteratorBuilder parsePlateData(
         return dataRows;
     }
 
-    @Nullable
-    private Integer getPlateSetValueFromRunProps(XarContext context, AssayProvider provider, ExpProtocol protocol) throws ExperimentException
-    {
-        Domain runDomain = provider.getRunDomain(protocol);
-        DomainProperty propertyPlateSet = runDomain.getPropertyByName(AssayPlateMetadataService.PLATE_SET_COLUMN_NAME);
-        if (propertyPlateSet == null)
-        {
-            throw new ExperimentException("The assay run domain for the assay '" + protocol.getName() + "' does not contain a plate set property.");
-        }
-
-        Map<DomainProperty, String> runProps = ((AssayUploadXarContext)context).getContext().getRunProperties();
-        Object plateSetVal = runProps.getOrDefault(propertyPlateSet, null);
-        return plateSetVal != null ? Integer.parseInt(String.valueOf(plateSetVal)) : null;
-    }
-
     /**
      * Creates a DataLoader that can handle missing value indicators if the columns on the domain
      * are configured to support it.
@@ -610,7 +597,19 @@ protected void insertRowData(
     {
         OntologyManager.UpdateableTableImportHelper importHelper = new SimpleAssayDataImportHelper(data, protocol, provider);
         if (provider.isPlateMetadataEnabled(protocol))
+        {
+            if (context.getReRunId() != null)
+            {
+                // check if we are merging the re-imported data
+                if (context.getReImportOption() == MERGE_DATA)
+                {
+                    DataIteratorBuilder mergedData = AssayPlateMetadataService.get().mergeReRunData(container, user, context, fileData, provider, protocol, data);
+                    fileData = DataIteratorUtil.wrapMap(mergedData.getDataIterator(new DataIteratorContext()), false);
+                }
+            }
+
             importHelper = AssayPlateMetadataService.get().getImportHelper(container, user, run, data, protocol, provider, context);
+        }
 
         if (tableInfo instanceof UpdateableTableInfo uti)
         {

api/src/org/labkey/api/assay/AssayProtocolSchema.java

@@ -89,6 +89,7 @@
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.util.Path;
 import org.labkey.api.util.StringExpressionFactory;
+import org.labkey.api.util.UnexpectedException;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.DataView;
 import org.labkey.api.view.NotFoundException;
@@ -746,13 +747,17 @@ public void addQCWarningIndicator(QueryView baseQueryView, ViewContext context,
                                     }
                                     catch(IOException e)
                                     {
-                                        throw new RuntimeException(e);
+                                        throw UnexpectedException.wrap(e);
                                     }
                                 });
                             }
                         }
                         catch (SQLException | IOException e)
                         {
+                            if (errors == null)
+                            {
+                                throw UnexpectedException.wrap(e);
+                            }
                             errors.reject(SpringActionController.ERROR_MSG, e.getMessage());
                         }
                     }

api/src/org/labkey/api/assay/plate/AssayPlateMetadataService.java

@@ -7,6 +7,7 @@
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
 import org.labkey.api.data.TableInfo;
+import org.labkey.api.dataiterator.DataIterator;
 import org.labkey.api.dataiterator.DataIteratorBuilder;
 import org.labkey.api.exp.ExperimentException;
 import org.labkey.api.exp.Lsid;
@@ -66,6 +67,34 @@ DataIteratorBuilder mergePlateMetadata(
         ExpProtocol protocol
     ) throws ExperimentException;
 
+    /**
+     * Takes the current incoming data and combines it with any data uploaded in the previous run (re-run ID). Data
+     * can be combined for plates within a plate set, but only on a per plate boundary. If there is data for plates
+     * in both sets of data, the most recent data will take precedence.
+     *
+     * @param results The incoming data rows
+     * @return The new, combined data
+     */
+    DataIteratorBuilder mergeReRunData(
+        Container container,
+        User user,
+        @NotNull AssayRunUploadContext<?> context,
+        DataIterator results,
+        AssayProvider provider,
+        ExpProtocol protocol,
+        ExpData data
+    ) throws ExperimentException;
+
+    /**
+     * Returns the plate set ID for the current run context.
+     */
+    @Nullable
+    Integer getPlateSetId(
+        AssayRunUploadContext<?> context,
+        AssayProvider provider,
+        ExpProtocol protocol
+    ) throws ExperimentException;
+
     /**
      * Handles the validation and parsing of the plate data (or data file) including plate graphical formats as
      * well as cases where plate identifiers have not been supplied.

api/src/org/labkey/api/assay/transform/DataExchangeHandler.java

@@ -16,6 +16,7 @@
 package org.labkey.api.assay.transform;
 
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.assay.AssayProvider;
 import org.labkey.api.assay.AssayRunUploadContext;
 import org.labkey.api.data.TSVWriter;
@@ -37,15 +38,47 @@
 
 /**
  * Used to process input and output data between the server and externally executed qc and analysis scripts.
- * User: Karl Lum
- * Date: Jan 7, 2009
  */
 public interface DataExchangeHandler
 {
-    Pair<FileLike, Set<FileLike>> createTransformationRunInfo(AssayRunUploadContext<? extends AssayProvider> context, ExpRun run, FileLike scriptDir, Map<DomainProperty, String> runProperties, Map<DomainProperty, String> batchProperties) throws Exception;
-    void createSampleData(@NotNull ExpProtocol protocol, ViewContext viewContext, FileLike scriptDir) throws Exception;
+    /**
+     * Create and serialize the run properties information that is made available to transform scripts.
+     * The file contains a variety of information based on the transform operation being specified.
+     *
+     * @param operation The transform operation being performed
+     * @param context Contains information about the import or update context
+     * @param scriptDir The folder that the transform script will be run in.
+     * @return The map of the run properties file to the set of other data files associated with the operation
+     * being performed.
+     */
+    Pair<FileLike, Set<FileLike>> createTransformationRunInfo(
+            DataTransformService.TransformOperation operation,
+            AssayRunUploadContext<? extends AssayProvider> context,
+            @Nullable ExpRun run,
+            FileLike scriptDir,
+            Map<DomainProperty, String> runProperties,
+            Map<DomainProperty, String> batchProperties
+    ) throws Exception;
 
-    TransformResult processTransformationOutput(AssayRunUploadContext<? extends AssayProvider> context, FileLike runInfo, ExpRun run, FileLike scriptFile, TransformResult mergeResult, Set<FileLike> inputDataFiles) throws ValidationException;
+    /**
+     * Creates a test version of the run properties file for download
+     */
+    void createSampleData(
+            DataTransformService.TransformOperation operation,
+            @NotNull ExpProtocol protocol,
+            ViewContext viewContext,
+            FileLike scriptDir
+    ) throws Exception;
+
+    TransformResult processTransformationOutput(
+            DataTransformService.TransformOperation operation,
+            AssayRunUploadContext<? extends AssayProvider> context,
+            FileLike runInfo,
+            @Nullable ExpRun run,
+            FileLike scriptFile,
+            TransformResult mergeResult,
+            Set<FileLike> inputDataFiles
+    ) throws ValidationException;
 
     DataSerializer getDataSerializer();
 

api/src/org/labkey/api/assay/transform/DataTransformService.java

@@ -55,6 +55,7 @@ public static DataTransformService get()
     public static final String BASE_SERVER_URL_REPLACEMENT = "baseServerURL";
     public static final String CONTAINER_PATH = "containerPath";
     public static final String ORIGINAL_SOURCE_PATH = "OriginalSourcePath";
+    public static final String TRANSFORM_OPERATION = "transformOperation";
 
     public enum TransformOperation
     {
@@ -134,7 +135,7 @@ public TransformResult transformAndValidate(
 
                         Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE);
                         String script = sb.toString();
-                        Pair<FileLike, Set<FileLike>> files = dataHandler.createTransformationRunInfo(context, run, scriptDir, runProperties, batchProperties);
+                        Pair<FileLike, Set<FileLike>> files = dataHandler.createTransformationRunInfo(operation, context, run, scriptDir, runProperties, batchProperties);
                         FileLike runInfo = files.getKey();
 
                         bindings.put(ExternalScriptEngine.WORKING_DIRECTORY, scriptDir.toNioPathForWrite().toString());
@@ -144,14 +145,15 @@ public TransformResult transformAndValidate(
 
                         // Issue 51543: Resolve windows path to run properties
                         paramMap.put(RUN_INFO_REPLACEMENT, runInfo.toNioPathForWrite().toFile().getAbsolutePath().replaceAll("\\\\", "/"));
+                        paramMap.put(TRANSFORM_OPERATION, operation.name());
 
                         addStandardParameters(context.getRequest(), context.getContainer(), scriptFile, session.getApiKey(), paramMap);
 
                         bindings.put(ExternalScriptEngine.PARAM_REPLACEMENT_MAP, paramMap);
 
                         Object output = engine.eval(script);
 
-                        FileLike rewrittenScriptFile = null;
+                        FileLike rewrittenScriptFile;
                         if (bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE) instanceof File)
                         {
                             var rewrittenScriptFileObject = bindings.get(ExternalScriptEngine.REWRITTEN_SCRIPT_FILE);
@@ -166,7 +168,7 @@ public TransformResult transformAndValidate(
                         }
 
                         // process any output from the transformation script
-                        result = dataHandler.processTransformationOutput(context, runInfo, run, rewrittenScriptFile, result, files.getValue());
+                        result = dataHandler.processTransformationOutput(operation, context, runInfo, run, rewrittenScriptFile, result, files.getValue());
 
                         // Propagate any transformed batch properties on to the next script
                         if (result.getBatchProperties() != null && !result.getBatchProperties().isEmpty())

api/src/org/labkey/api/module/ModuleLoader.java

@@ -475,6 +475,16 @@ public void updateModuleDirectory(File dir, File archive)
         }
     }
 
+    private record UpgradeInfo(String moduleName, double installedVersion)
+    {
+        @Override
+        public String toString()
+        {
+            return moduleName + " (from schema version " + ModuleContext.formatVersion(installedVersion) + ")";
+
+        }
+    }
+
     /** Full web-server initialization */
     private void doInit(Execution execution) throws ServletException
     {
@@ -646,7 +656,7 @@ public void addStaticWarnings(@NotNull Warnings warnings, boolean showAllWarning
         upgradeLabKeySchemaInExternalDataSources();
 
         ModuleContext coreCtx;
-        List<String> modulesRequiringUpgrade = new LinkedList<>();
+        List<UpgradeInfo> modulesRequiringUpgrade = new LinkedList<>();
         List<String> additionalSchemasRequiringUpgrade = new LinkedList<>();
         List<String> downgradedModules = new LinkedList<>();
 
@@ -675,7 +685,7 @@ public void addStaticWarnings(@NotNull Warnings warnings, boolean showAllWarning
                     if (context.needsUpgrade(module.getSchemaVersion()))
                     {
                         context.setModuleState(ModuleState.InstallRequired);
-                        modulesRequiringUpgrade.add(context.getName());
+                        modulesRequiringUpgrade.add(new UpgradeInfo(context.getName(), context.getInstalledVersion()));
                         addModuleToLockFile(context.getName(), lockFile);
                     }
                     else
@@ -771,11 +781,11 @@ private void warnAboutDuplicateSchemas(Collection<ModuleContext> values)
     /**
      * Does this module live in a repository that's managed by LabKey Corporation?
      * @param module a Module
-     * @return true if the module's VCS URL is non-null and starts with one of the GitHub organizations that LabKey manages
+     * @return true if the module's VCS URL is non-null and includes "github.com:LabKey/"
      */
     private boolean isFromLabKeyRepository(Module module)
     {
-        return StringUtils.startsWithAny(module.getVcsUrl(), "https://github.com/LabKey/");
+        return StringUtils.containsAny(module.getVcsUrl(), "github.com:LabKey/");
     }
 
     /**