Merge branch '7.6' into gh-4594-perms-bugs-and-enhancements_7.6

CHANGELOG.md

@@ -36,6 +36,14 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 
 * If you are upgrading from a previous v7.6 beta release you will need to run the following SQL. `update analytics_schema_history set checksum = '-86554219' where version = '07.06.00.405';` and `update processor_schema_history set checksum = '-175036745' where version = '07.06.00.305';`.
 
+* Issue **#4627** : Fix StroomQL function character escaping.
+
+* Issue **#4611** : Fix problem of changes to the conditional formatting rules of a duplicated dashboard table affecting the original table. This only affected the enabled/hide properties of the formatting rule.
+
+* Issue **#4612** : Fix stroomQL query not including all data points in the visualisation.
+
+* Issue **#4617** : Add debug to try to diagnose issue.
+
 * Issue **#4606** : Fix dashboard text pane that is set to Show as HTML not showing a vertical scrollbar.
 
 

stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/DetectionWriter.java

@@ -2,6 +2,7 @@
 
 import stroom.pipeline.errorhandler.ErrorReceiverProxy;
 import stroom.pipeline.filter.XMLFilter;
+import stroom.util.NullSafe;
 import stroom.util.shared.Severity;
 
 import jakarta.inject.Inject;
@@ -33,7 +34,6 @@ public class DetectionWriter implements DetectionConsumer {
     private static final String LOCATION = "file://detection-v1.1.xsd";
 //    private static final String VERSION = "2.0";
 
-
     private static final Attributes EMPTY_ATTS = new AttributesImpl();
     private static final AttributesImpl ROOT_ATTS = new AttributesImpl();
 
@@ -97,13 +97,13 @@ public synchronized void end() {
                 handler.endProcessing();
             }
         } catch (final SAXException e) {
-            log(Severity.ERROR, e.getMessage(), e);
+            logError(e.getMessage(), e);
         }
     }
 
-    private void log(final Severity severity, final String message, final Exception e) {
+    private void logError(final String message, final Exception e) {
         LOGGER.error(message, e);
-        errorReceiverProxy.log(severity, null,
+        errorReceiverProxy.log(Severity.ERROR, null,
                 getClass().getSimpleName(), message, e);
     }
 
@@ -140,12 +140,12 @@ public synchronized void accept(final Detection detection) {
             writeEndElement(DETECTION);
 
         } catch (final SAXException e) {
-            log(Severity.ERROR, e.getMessage(), e);
+            logError(e.getMessage(), e);
         }
     }
 
     private void writeValues(List<DetectionValue> values) throws SAXException {
-        if (values != null && values.size() > 0) {
+        if (NullSafe.hasItems(values)) {
             for (final DetectionValue value : values) {
                 writeValue(value);
             }
@@ -162,7 +162,7 @@ private void writeValue(final DetectionValue value) throws SAXException {
 
 
     private void writeLinkedEvents(final List<DetectionLinkedEvent> linkedEvents) throws SAXException {
-        if (linkedEvents != null && linkedEvents.size() > 0) {
+        if (NullSafe.hasItems(linkedEvents)) {
             writeStartElement(LINKED_EVENTS);
             for (final DetectionLinkedEvent linkedEvent : linkedEvents) {
                 writeLinkedEvent(linkedEvent);
@@ -201,7 +201,7 @@ private void writeOptionalDataElement(final String elementName,
 
     private void writeOptionalDataElement(final String elementName,
                                           final String text) throws SAXException {
-        if (text != null && text.length() > 0) {
+        if (NullSafe.isNonEmptyString(text)) {
             writeDataElement(elementName, EMPTY_ATTS, text);
         }
     }

stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/DetectionsWriter.java

@@ -4,14 +4,18 @@
 import stroom.docref.DocRef;
 import stroom.pipeline.writer.StreamAppender;
 import stroom.pipeline.writer.XMLWriter;
+import stroom.util.logging.LambdaLogger;
+import stroom.util.logging.LambdaLoggerFactory;
 
 import jakarta.inject.Inject;
 
 public class DetectionsWriter implements DetectionConsumer {
 
+    private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(DetectionsWriter.class);
+
     private final DetectionWriter detectionWriter;
     private final StreamAppender streamAppender;
-
+    private DocRef feed = null;
 
     @Inject
     public DetectionsWriter(final DetectionWriter detectionWriter,
@@ -27,21 +31,25 @@ public DetectionsWriter(final DetectionWriter detectionWriter,
     }
 
     public void setFeed(final DocRef feed) {
+        this.feed = feed;
         streamAppender.setFeed(feed);
     }
 
     @Override
     public void start() {
+        LOGGER.debug("Starting to write detections to feed: {}", feed);
         detectionWriter.start();
     }
 
     @Override
     public void end() {
         detectionWriter.end();
+        LOGGER.debug("Finished writing detections to feed: {}", feed);
     }
 
     @Override
     public void accept(final Detection detection) {
+        LOGGER.debug("Accepting detection to feed: {}", feed);
         detectionWriter.accept(detection);
     }
 }

stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/ScheduledQueryAnalyticExecutor.java

@@ -215,9 +215,10 @@ private Runnable createRunnable(final AnalyticRuleDoc analytic,
     private void execAnalytic(final AnalyticRuleDoc analytic,
                               final TaskContext parentTaskContext) {
         // Load schedules for the analytic.
+        final DocRef analyticDocRef = analytic.asDocRef();
         final ExecutionScheduleRequest request = ExecutionScheduleRequest
                 .builder()
-                .ownerDocRef(analytic.asDocRef())
+                .ownerDocRef(analyticDocRef)
                 .enabled(true)
                 .nodeName(StringMatch.equals(nodeInfo.getThisNodeName(), true))
                 .build();
@@ -229,6 +230,9 @@ private void execAnalytic(final AnalyticRuleDoc analytic,
                 try {
                     // We need to set the user again here as it will have been lost from the parent context as we are
                     // running within a new thread.
+                    LOGGER.debug(() -> LogUtil.message("analyticRuleDoc: {}, running as user: {}",
+                            analyticDocRef.toShortString(), executionSchedule.getRunAsUser()));
+
                     securityContext.asUser(executionSchedule.getRunAsUser(), () -> securityContext.useAsRead(() -> {
                         boolean success = true;
                         while (success && !parentTaskContext.isTerminated()) {
@@ -329,6 +333,10 @@ private boolean process(final AnalyticRuleDoc analytic,
                             final Instant effectiveExecutionTime,
                             final ExecutionSchedule executionSchedule,
                             final ExecutionTracker currentTracker) {
+        LOGGER.debug(() -> LogUtil.message(
+                "Executing analytic: {} with executionTime: {}, effectiveExecutionTime: {}, currentTracker: {}",
+                analytic.asDocRef().toShortString(), executionTime, effectiveExecutionTime, currentTracker));
+
         boolean success = false;
         final ErrorConsumer errorConsumer = new ErrorConsumerImpl();
         ExecutionResult executionResult = new ExecutionResult(null, null);

stroom-core-client/src/main/java/stroom/dashboard/client/main/DashboardPresenter.java

@@ -94,6 +94,7 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+@SuppressWarnings("PatternVariableCanBeUsed") // Cos GWT
 public class DashboardPresenter
         extends DocumentEditPresenter<DashboardView, DashboardDoc>
         implements
@@ -391,10 +392,7 @@ protected void onRead(final DocRef docRef, final DashboardDoc dashboard, final b
                         dashboardConfig.setComponents(componentConfigList);
                     }
 
-                    final String params = dashboardConfig.getParameters() == null
-                            ?
-                            ""
-                            : dashboardConfig.getParameters();
+                    final String params = GwtNullSafe.string(dashboardConfig.getParameters());
 
                     componentConfigList
                             .add(new ComponentConfig(
@@ -489,7 +487,7 @@ protected void onRead(final DocRef docRef, final DashboardDoc dashboard, final b
                     if (resultStoreInfo != null) {
                         final SearchRequestSource searchRequestSource = resultStoreInfo.getSearchRequestSource();
                         if (searchRequestSource != null &&
-                                component.getId().equals(searchRequestSource.getComponentId())) {
+                            component.getId().equals(searchRequestSource.getComponentId())) {
                             ((Queryable) component).setResultStoreInfo(resultStoreInfo);
                         }
                     }
@@ -518,8 +516,8 @@ protected void onRead(final DocRef docRef, final DashboardDoc dashboard, final b
 
             // Turn on design mode if this is a new dashboard.
             if (dashboardConfig != null &&
-                    dashboardConfig.getDesignMode() != null &&
-                    dashboardConfig.getDesignMode()) {
+                dashboardConfig.getDesignMode() != null &&
+                dashboardConfig.getDesignMode()) {
                 editModeButton.setState(true);
                 setDesignMode(true);
             }
@@ -649,7 +647,8 @@ public void duplicateTabs(final TabLayoutConfig tabLayoutConfig, final List<TabC
                 final ComponentType type = originalComponent.getComponentType();
 
                 final ComponentId componentId = componentNameSet.createUnique(type, originalComponent.getLabel());
-                final ComponentConfig componentConfig = originalComponent.getComponentConfig().copy()
+                final ComponentConfig componentConfig = originalComponent.getComponentConfig()
+                        .copy()
                         .id(componentId.id)
                         .name(componentId.name)
                         .build();
@@ -669,27 +668,32 @@ public void duplicateTabs(final TabLayoutConfig tabLayoutConfig, final List<TabC
             ComponentSettings settings = componentConfig.getSettings();
             if (settings instanceof TableComponentSettings) {
                 TableComponentSettings tableComponentSettings = (TableComponentSettings) settings;
-                if (tableComponentSettings.getQueryId() != null &&
-                        idMapping.containsKey(tableComponentSettings.getQueryId())) {
-                    settings = tableComponentSettings
-                            .copy().queryId(idMapping.get(tableComponentSettings.getQueryId())).build();
+                if (tableComponentSettings.getQueryId() != null
+                    && idMapping.containsKey(tableComponentSettings.getQueryId())) {
+                    settings = tableComponentSettings.copy()
+                            .queryId(idMapping.get(tableComponentSettings.getQueryId()))
+                            .build();
                 }
             } else if (settings instanceof VisComponentSettings) {
                 VisComponentSettings visComponentSettings = (VisComponentSettings) settings;
-                if (visComponentSettings.getTableId() != null &&
-                        idMapping.containsKey(visComponentSettings.getTableId())) {
-                    settings = visComponentSettings
-                            .copy().tableId(idMapping.get(visComponentSettings.getTableId())).build();
+                if (visComponentSettings.getTableId() != null
+                    && idMapping.containsKey(visComponentSettings.getTableId())) {
+                    settings = visComponentSettings.copy()
+                            .tableId(idMapping.get(visComponentSettings.getTableId()))
+                            .build();
                 }
             } else if (settings instanceof TextComponentSettings) {
                 TextComponentSettings textComponentSettings = (TextComponentSettings) settings;
-                if (textComponentSettings.getTableId() != null &&
-                        idMapping.containsKey(textComponentSettings.getTableId())) {
-                    settings = textComponentSettings
-                            .copy().tableId(idMapping.get(textComponentSettings.getTableId())).build();
+                if (textComponentSettings.getTableId() != null
+                    && idMapping.containsKey(textComponentSettings.getTableId())) {
+                    settings = textComponentSettings.copy()
+                            .tableId(idMapping.get(textComponentSettings.getTableId()))
+                            .build();
                 }
             }
-            modifiedComponents.add(componentConfig.copy().settings(settings).build());
+            modifiedComponents.add(componentConfig.copy()
+                    .settings(settings)
+                    .build());
         }
 
         // Now try and add all the duplicated components.
@@ -930,7 +934,7 @@ private TabConfig getFirstTabConfig(final LayoutConfig layoutConfig) {
                 final TabLayoutConfig tabLayoutConfig = (TabLayoutConfig) layoutConfig;
                 if (tabLayoutConfig.getTabs().size() > 0) {
                     if (tabLayoutConfig.getSelected() >= 0 &&
-                            tabLayoutConfig.getSelected() < tabLayoutConfig.getTabs().size()) {
+                        tabLayoutConfig.getSelected() < tabLayoutConfig.getTabs().size()) {
                         return tabLayoutConfig.get(tabLayoutConfig.getSelected());
                     } else {
                         return tabLayoutConfig.get(0);

stroom-core-client/src/main/java/stroom/dashboard/client/table/cf/RuleListPresenter.java

@@ -39,12 +39,16 @@
 import com.google.web.bindery.event.shared.HandlerRegistration;
 import com.gwtplatform.mvp.client.MyPresenterWidget;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 public class RuleListPresenter extends MyPresenterWidget<PagerView> implements HasDirtyHandlers {
 
     private final MyDataGrid<ConditionalFormattingRule> dataGrid;
     private final MultiSelectionModelImpl<ConditionalFormattingRule> selectionModel;
+    private List<ConditionalFormattingRule> rules = new ArrayList<>();
+    private Runnable dataChangeHandler = null;
 
     @Inject
     public RuleListPresenter(final EventBus eventBus,
@@ -106,9 +110,13 @@ private void addEnabledColumn() {
                         .build();
 
         enabledColumn.setFieldUpdater((index, row, tickBoxState) -> {
-            row.setEnabled(GwtNullSafe.isTrue(tickBoxState.toBoolean()));
-            setDirty(true);
-            dataGrid.redraw();
+            if (row != null) {
+                final ConditionalFormattingRule newRow = row
+                        .copy()
+                        .enabled(GwtNullSafe.isTrue(tickBoxState.toBoolean()))
+                        .build();
+                replaceDataGridRow(row, newRow);
+            }
         });
 
         dataGrid.addColumn(
@@ -128,9 +136,13 @@ private void addHideColumn() {
                         .build();
 
         hideColumn.setFieldUpdater((index, row, tickBoxState) -> {
-            row.setHide(GwtNullSafe.isTrue(tickBoxState.toBoolean()));
-            setDirty(true);
-            dataGrid.redraw();
+            if (row != null) {
+                final ConditionalFormattingRule newRow = row
+                        .copy()
+                        .hide(GwtNullSafe.isTrue(tickBoxState.toBoolean()))
+                        .build();
+                replaceDataGridRow(row, newRow);
+            }
         });
 
         dataGrid.addColumn(hideColumn,
@@ -140,9 +152,32 @@ private void addHideColumn() {
                 70);
     }
 
+    private void replaceDataGridRow(final ConditionalFormattingRule oldRule,
+                                    final ConditionalFormattingRule newRule) {
+        final int idx = rules.indexOf(oldRule);
+        rules.remove(idx);
+        rules.add(idx, newRule);
+
+        setDirty(true);
+        dataGrid.setRowData(0, rules);
+        final ConditionalFormattingRule selected = getSelectionModel().getSelected();
+        if (Objects.equals(selected, oldRule)) {
+            getSelectionModel().setSelected(newRule);
+        }
+        GwtNullSafe.run(dataChangeHandler);
+    }
+
     public void setData(final List<ConditionalFormattingRule> data) {
         dataGrid.setRowData(0, data);
         dataGrid.setRowCount(data.size());
+        rules = data;
+    }
+
+    /**
+     * A dataChangeHandler to be called if the dataGrid data is changed
+     */
+    public void setDataChangeHandler(final Runnable dataChangeHandler) {
+        this.dataChangeHandler = dataChangeHandler;
     }
 
     public MultiSelectionModel<ConditionalFormattingRule> getSelectionModel() {

stroom-core-client/src/main/java/stroom/dashboard/client/table/cf/RulesPresenter.java

@@ -74,6 +74,8 @@ public RulesPresenter(final EventBus eventBus,
                           final Provider<RulePresenter> editRulePresenterProvider) {
         super(eventBus, view);
         this.listPresenter = listPresenter;
+        // listPresenter can change the enabled/hide props, so we need to update the buttons accordingly
+        this.listPresenter.setDataChangeHandler(this::updateButtons);
         this.editRulePresenterProvider = editRulePresenterProvider;
 
         getView().setTableView(listPresenter.getView());
@@ -309,7 +311,9 @@ public ComponentConfig write(final ComponentConfig componentConfig) {
                 .copy()
                 .conditionalFormattingRules(rules)
                 .build();
-        return componentConfig.copy().settings(newSettings).build();
+        return componentConfig.copy()
+                .settings(newSettings)
+                .build();
     }
 
     @Override

stroom-core-shared/src/main/java/stroom/analytics/shared/ExecutionTracker.java

@@ -41,4 +41,13 @@ public long getLastEffectiveExecutionTimeMs() {
     public long getNextEffectiveExecutionTimeMs() {
         return nextEffectiveExecutionTimeMs;
     }
+
+    @Override
+    public String toString() {
+        return "ExecutionTracker{" +
+               "actualExecutionTimeMs=" + actualExecutionTimeMs +
+               ", lastEffectiveExecutionTimeMs=" + lastEffectiveExecutionTimeMs +
+               ", nextEffectiveExecutionTimeMs=" + nextEffectiveExecutionTimeMs +
+               '}';
+    }
 }