Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…

…9.6.x

Signed-off-by: Gordon Smith <[email protected]>

common/workunit/wujobq.cpp

@@ -1122,11 +1122,19 @@ class CJobQueue: public CJobQueueBase, implements IJobQueue
     void disconnect()   // signal no longer wil be dequeing (optional - done automatically on release)
     {
         Cconnlockblock block(this,true);
-        if (connected) {
+        if (connected)
+        {
             dounsubscribe();
-            ForEachQueue(qd) {
+            ForEachQueue(qd)
+            {
                 IPropertyTree *croot = queryClientRootSession(*qd);
-                croot->setPropInt64("@connected",croot->getPropInt64("@connected",0)-1);
+                unsigned connectedCount = croot->getPropInt64("@connected");
+                if (connectedCount) // should never be 0, but guard JIC
+                    connectedCount--;
+                if (connectedCount)
+                    croot->setPropInt64("@connected", connectedCount);
+                else
+                    qd->root->removeTree(croot);
             }
             connected = false;
         }

dali/base/dasds.cpp

@@ -4938,6 +4938,18 @@ void initializeInternals(IPropertyTree *root)
     root->addPropTree("Status/Servers",createPTree());
 }
 
+void clearStaleMeteData(IPropertyTree *root)
+{
+    // JobQueues
+    // Remove all Client entries from all queues. By definition they are stale (they should normally be removed when the client disconnects)
+    Owned<IPropertyTreeIterator> jobQueues = root->getElements("JobQueues/Queue");
+    ForEach(*jobQueues)
+    {
+        IPropertyTree &queue = jobQueues->query();
+        while (queue.removeProp("Client"));
+    }
+}
+
 IPropertyTree *loadStore(const char *storeFilename, unsigned edition, IPTreeMaker *iMaker, unsigned crcValidation, bool logErrorsOnly=false, const bool *abort=NULL)
 {
     CHECKEDCRITICALBLOCK(loadStoreCrit, fakeCritTimeout);
@@ -6543,6 +6555,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     }
     Owned<IRemoteConnection> conn = connect("/", 0, RTM_INTERNAL, INFINITE);
     initializeInternals(conn->queryRoot());
+    clearStaleMeteData(conn->queryRoot());
     conn.clear();
     initializeStorageGroups(oldEnvironment);
 }

esp/src/Login.html

@@ -112,6 +112,8 @@
             function (ready, nlsHPCCMod) {
                 var nlsHPCC = nlsHPCCMod.default;
                 ready(function () {
+                    window.localStorage.setItem("redirectAfterLogin", window.location.hash);
+
                     var loginStr = document.getElementById("loginStr");
                     var error = document.getElementById("hidden_msg");
                     var disabled = document.getElementById('disabled_msg');

esp/src/package.json

@@ -44,7 +44,7 @@
     "@hpcc-js/chart": "2.83.3",
     "@hpcc-js/codemirror": "2.61.4",
     "@hpcc-js/common": "2.71.17",
-    "@hpcc-js/comms": "2.92.1",
+    "@hpcc-js/comms": "2.92.2",
     "@hpcc-js/dataflow": "8.1.6",
     "@hpcc-js/eclwatch": "2.74.3",
     "@hpcc-js/graph": "2.85.15",

esp/src/src-react/components/ECLPlayground.tsx

@@ -1,14 +1,17 @@
 import * as React from "react";
 import { ReflexContainer, ReflexElement, ReflexSplitter } from "../layouts/react-reflex";
-import { PrimaryButton, IconButton, IIconProps, Link, Dropdown, IDropdownOption, TextField, useTheme } from "@fluentui/react";
+import { IconButton, IIconProps, Link, Dropdown, IDropdownOption, TextField, useTheme } from "@fluentui/react";
+import { Button } from "@fluentui/react-components";
+import { CheckmarkCircleRegular, DismissCircleRegular, QuestionCircleRegular } from "@fluentui/react-icons";
 import { scopedLogger } from "@hpcc-js/util";
 import { useOnEvent } from "@fluentui/react-hooks";
 import { mergeStyleSets } from "@fluentui/style-utilities";
 import { ECLEditor, IPosition } from "@hpcc-js/codemirror";
-import { Workunit, WUUpdate } from "@hpcc-js/comms";
+import { Workunit, WUUpdate, WorkunitsService } from "@hpcc-js/comms";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { DojoAdapter } from "../layouts/DojoAdapter";
 import { pushUrl } from "../util/history";
+import { debounce } from "../util/throttle";
 import { darkTheme } from "../themes";
 import { InfoGrid } from "./InfoGrid";
 import { TabbedResults } from "./Results";
@@ -77,6 +80,9 @@ const playgroundStyles = mergeStyleSets({
                     borderRight: borderStyle
                 }
             },
+            ".fui-Button": {
+                height: "min-content"
+            },
             ".ms-Label": {
                 marginRight: "12px"
             },
@@ -155,51 +161,69 @@ const warningIcon: IIconProps = { title: nlsHPCC.ErrorWarnings, ariaLabel: nlsHP
 const resultsIcon: IIconProps = { title: nlsHPCC.Outputs, ariaLabel: nlsHPCC.Outputs, iconName: "Table" };
 const graphIcon: IIconProps = { title: nlsHPCC.Visualizations, ariaLabel: nlsHPCC.Visualizations, iconName: "BarChartVerticalFill" };
 
-const displayErrors = (wu, editor) => {
+const displayErrors = async (wu = null, editor, errors = []) => {
     if (!editor) return;
-    wu.fetchECLExceptions().then(errors => {
-        errors.forEach(err => {
-            const lineError = err.LineNo;
-            const lineErrorNum = lineError > 0 ? lineError - 1 : 0;
-            const startPos: IPosition = {
-                ch: (err.Column > 0) ? err.Column - 1 : 0,
-                line: lineErrorNum
-            };
-            const endPos: IPosition = {
-                ch: editor.getLineLength(lineErrorNum),
-                line: lineErrorNum
-            };
-
-            switch (err.Severity) {
-                case "Info":
-                    editor.highlightInfo(startPos, endPos);
-                    break;
-                case "Warning":
-                    editor.highlightWarning(startPos, endPos);
-                    break;
-                case "Error":
-                default:
-                    editor.highlightError(startPos, endPos);
-                    break;
-            }
-        });
+    if (wu) {
+        errors = await wu.fetchECLExceptions();
+    }
+    if (!errors.length) {
+        editor.removeAllHighlight();
+    }
+    errors.forEach(err => {
+        const lineError = err.LineNo;
+        const lineErrorNum = lineError > 0 ? lineError - 1 : 0;
+        const startPos: IPosition = {
+            ch: (err.Column > 0) ? err.Column - 1 : 0,
+            line: lineErrorNum
+        };
+        const endPos: IPosition = {
+            ch: editor.getLineLength(lineErrorNum),
+            line: lineErrorNum
+        };
+
+        switch (err.Severity) {
+            case "Info":
+                editor.highlightInfo(startPos, endPos);
+                break;
+            case "Warning":
+                editor.highlightWarning(startPos, endPos);
+                break;
+            case "Error":
+            default:
+                editor.highlightError(startPos, endPos);
+                break;
+        }
     });
 };
 
+const service = new WorkunitsService({ baseUrl: "" });
+
+enum SyntaxCheckResult {
+    Unknown,
+    Failed,
+    Passed
+}
+
 interface ECLEditorToolbarProps {
     editor: ECLEditor;
     outputMode: OutputMode;
     setOutputMode: (_: OutputMode) => void;
     workunit: Workunit;
     setWorkunit: (_: Workunit) => void;
+    setSyntaxErrors: (_: any) => void;
+    syntaxStatusIcon: number;
+    setSyntaxStatusIcon: (_: number) => void;
 }
 
 const ECLEditorToolbar: React.FunctionComponent<ECLEditorToolbarProps> = ({
     editor,
     outputMode,
     setOutputMode,
     workunit,
-    setWorkunit
+    setWorkunit,
+    setSyntaxErrors,
+    syntaxStatusIcon,
+    setSyntaxStatusIcon
 }) => {
 
     const [cluster, setCluster] = React.useState("");
@@ -258,6 +282,24 @@ const ECLEditorToolbar: React.FunctionComponent<ECLEditorToolbarProps> = ({
         }
     }, [cluster, editor, playgroundResults, queryName, setQueryNameErrorMsg]);
 
+    const checkSyntax = React.useCallback(() => {
+        service.WUSyntaxCheckECL({
+            ECL: editor.ecl(),
+            Cluster: cluster
+        }).then(response => {
+            if (response.Errors) {
+                setSyntaxStatusIcon(SyntaxCheckResult.Failed);
+                setSyntaxErrors(response.Errors.ECLException);
+                displayErrors(null, editor, response.Errors.ECLException);
+                setOutputMode(OutputMode.ERRORS);
+            } else {
+                setSyntaxStatusIcon(SyntaxCheckResult.Passed);
+                setSyntaxErrors([]);
+                displayErrors(null, editor, []);
+            }
+        });
+    }, [cluster, editor, setOutputMode, setSyntaxErrors, setSyntaxStatusIcon]);
+
     const handleKeyUp = React.useCallback((evt) => {
         switch (evt.key) {
             case "Enter":
@@ -282,10 +324,19 @@ const ECLEditorToolbar: React.FunctionComponent<ECLEditorToolbarProps> = ({
     return <div className={playgroundStyles.toolBar}>
         <div className={playgroundStyles.controlsWrapper}>
             {showSubmitBtn ? (
-                <PrimaryButton text={nlsHPCC.Submit} onClick={submitWU} />
+                <Button appearance="primary" onClick={submitWU}>{nlsHPCC.Submit}</Button>
             ) : (
-                <PrimaryButton text={nlsHPCC.Publish} onClick={publishWU} />
+                <Button appearance="primary" onClick={publishWU}>{nlsHPCC.Publish}</Button>
             )}
+            <Button style={{ marginLeft: 6 }} onClick={checkSyntax} iconPosition="after"
+                icon={
+                    syntaxStatusIcon === SyntaxCheckResult.Passed ? <CheckmarkCircleRegular style={{ color: "green" }} /> :
+                        syntaxStatusIcon === SyntaxCheckResult.Failed ? <DismissCircleRegular style={{ color: "red" }} /> :
+                            <QuestionCircleRegular style={{ color: "initial" }} />
+                }
+            >
+                {nlsHPCC.Syntax}
+            </Button>
             <TargetClusterTextField
                 key="target-cluster"
                 label={nlsHPCC.Target}
@@ -350,6 +401,8 @@ export const ECLPlayground: React.FunctionComponent<ECLPlaygroundProps> = (props
     const [query, setQuery] = React.useState("");
     const [selectedEclSample, setSelectedEclSample] = React.useState("");
     const [eclContent, setEclContent] = React.useState("");
+    const [syntaxErrors, setSyntaxErrors] = React.useState<any[]>([]);
+    const [syntaxStatusIcon, setSyntaxStatusIcon] = React.useState(SyntaxCheckResult.Unknown);
     const [eclSamples, setEclSamples] = React.useState<IDropdownOption[]>([]);
 
     React.useEffect(() => {
@@ -417,6 +470,13 @@ export const ECLPlayground: React.FunctionComponent<ECLPlaygroundProps> = (props
     }, [editor]);
     useOnEvent(document, "eclwatch-theme-toggle", handleThemeToggle);
 
+    const handleEclChange = React.useMemo(() => debounce((evt) => {
+        if (editor.hasFocus()) {
+            setSyntaxStatusIcon(SyntaxCheckResult.Unknown);
+        }
+    }, 300), [editor]);
+    useOnEvent(window, "keyup", handleEclChange);
+
     return <div className={playgroundStyles.root}>
         <div className={playgroundStyles.titleBar}>
             <h1 className={playgroundStyles.title}>{nlsHPCC.title_ECLPlayground}</h1>
@@ -437,7 +497,8 @@ export const ECLPlayground: React.FunctionComponent<ECLPlaygroundProps> = (props
                             main={<ECLSourceEditor text={query} setEditor={setEditor} />}
                             footer={
                                 <ECLEditorToolbar
-                                    editor={editor}
+                                    editor={editor} setSyntaxErrors={setSyntaxErrors}
+                                    syntaxStatusIcon={syntaxStatusIcon} setSyntaxStatusIcon={setSyntaxStatusIcon}
                                     workunit={workunit} setWorkunit={setWorkunit}
                                     outputMode={outputMode} setOutputMode={setOutputMode}
                                 />
@@ -453,7 +514,7 @@ export const ECLPlayground: React.FunctionComponent<ECLPlaygroundProps> = (props
             <ReflexSplitter />
             <ReflexElement propagateDimensions={true} minSize={100}>
                 {outputMode === OutputMode.ERRORS ? (
-                    <InfoGrid wuid={workunit?.Wuid} />
+                    <InfoGrid wuid={workunit?.Wuid} syntaxErrors={syntaxErrors} />
 
                 ) : outputMode === OutputMode.RESULTS ? (
                     <TabbedResults wuid={workunit?.Wuid} filter={filter} />

esp/src/src-react/components/InfoGrid.tsx

@@ -32,11 +32,13 @@ interface FilterCounts {
 }
 
 interface InfoGridProps {
-    wuid: string;
+    wuid?: string;
+    syntaxErrors?: any[];
 }
 
 export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({
-    wuid
+    wuid = null,
+    syntaxErrors = []
 }) => {
 
     const [costChecked, setCostChecked] = React.useState(true);
@@ -46,6 +48,7 @@ export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({
     const [otherChecked, setOtherChecked] = React.useState(true);
     const [filterCounts, setFilterCounts] = React.useState<FilterCounts>({ cost: 0, penalty: 0, error: 0, warning: 0, info: 0, other: 0 });
     const [exceptions] = useWorkunitExceptions(wuid);
+    const [errors, setErrors] = React.useState<any[]>([]);
     const [data, setData] = React.useState<any[]>([]);
     const {
         selection, setSelection,
@@ -61,6 +64,14 @@ export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({
         { key: "others", onRender: () => <Checkbox defaultChecked label={`${filterCounts.other || 0} ${nlsHPCC.Others}`} onChange={(ev, value) => setOtherChecked(value)} styles={{ root: { paddingTop: 8, paddingRight: 8 } }} /> }
     ], [filterCounts.cost, filterCounts.error, filterCounts.info, filterCounts.other, filterCounts.warning]);
 
+    React.useEffect(() => {
+        if (syntaxErrors.length) {
+            setErrors(syntaxErrors);
+        } else {
+            setErrors(exceptions);
+        }
+    }, [syntaxErrors, exceptions]);
+
     //  Grid ---
     const columns = React.useMemo((): FluentColumns => {
         return {
@@ -137,7 +148,7 @@ export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({
             info: 0,
             other: 0
         };
-        const filteredExceptions = exceptions.map((row, idx) => {
+        const filteredExceptions = errors?.map((row, idx) => {
             if (row.Source === "Cost Optimizer") {
                 row.Severity = "Cost";
             }
@@ -199,7 +210,7 @@ export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({
         });
         setData(filteredExceptions);
         setFilterCounts(filterCounts);
-    }, [costChecked, errorChecked, exceptions, infoChecked, otherChecked, warningChecked]);
+    }, [costChecked, errorChecked, errors, infoChecked, otherChecked, warningChecked]);
 
     React.useEffect(() => {
         if (data.length) {

esp/src/src-react/components/Logs.tsx

@@ -13,7 +13,7 @@ import { Filter } from "./forms/Filter";
 import { Fields } from "./forms/Fields";
 import { ShortVerticalDivider } from "./Common";
 
-const maximumTimeUntilRefresh = 8 * 60 * 60 * 1000;
+const eightHours = 8 * 60 * 60 * 1000;
 const startTimeOffset = 1 * 60 * 60 * 1000;
 const endTimeOffset = 23 * 60 * 60 * 1000;
 const defaultStartDate = new Date(new Date().getTime() - startTimeOffset);
@@ -116,8 +116,9 @@ export const Logs: React.FunctionComponent<LogsProps> = ({
             if (typeof filter.StartDate === "string") {
                 filter.StartDate = new Date(filter.StartDate + ":00Z");
             }
-            if (filter.StartDate && now.getTime() - filter.StartDate.getTime() > maximumTimeUntilRefresh) {
-                filter.StartDate = new Date(now.getTime() - startTimeOffset);
+            if (!filter.StartDate) {
+                //assign a reasonable default start date if one isn't set
+                filter.StartDate = new Date(now.getTime() - eightHours);
             }
             if (!filter.EndDate) {
                 filter.EndDate = new Date(now.getTime() + endTimeOffset);

esp/src/src-react/components/Queries.tsx

@@ -180,7 +180,7 @@ export const Queries: React.FunctionComponent<QueriesProps> = ({
 
     const [DeleteConfirm, setShowDeleteConfirm] = useConfirm({
         title: nlsHPCC.Delete,
-        message: nlsHPCC.DeleteSelectedWorkunits,
+        message: nlsHPCC.DeleteSelectedQueries,
         items: selection.map(s => s.Id),
         onSubmit: React.useCallback(() => {
             WsWorkunits.WUQuerysetQueryAction(selection, "Delete").then(() => refreshTable.call(true));