Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Gordon Smith <[email protected]> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # helm/hpcc/templates/dafilesrv.yaml # helm/hpcc/templates/dali.yaml # helm/hpcc/templates/dfuserver.yaml # helm/hpcc/templates/eclagent.yaml # helm/hpcc/templates/eclccserver.yaml # helm/hpcc/templates/eclscheduler.yaml # helm/hpcc/templates/esp.yaml # helm/hpcc/templates/localroxie.yaml # helm/hpcc/templates/roxie.yaml # helm/hpcc/templates/sasha.yaml # helm/hpcc/templates/thor.yaml # version.cmake
AttilaVamos · Dec 14, 2023 · 7552030 · 7552030
2 parents bc133d7 + 9cdf065
commit 7552030
Show file tree

Hide file tree

Showing 50 changed files with 1,737 additions and 644 deletions.
diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml
diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml
@@ -6,23 +6,26 @@
       <primary>SOAPCALL</primary>
     </indexterm></title>
 
-  <para><emphasis>result</emphasis><emphasis role="bold"> :=
-  SOAPCALL<indexterm>
-      <primary>SOAPCALL Function</primary>
-    </indexterm>( [ </emphasis><emphasis>recset, </emphasis><emphasis
-  role="bold">]</emphasis><emphasis> url, service, instructure,
-  </emphasis><emphasis role="bold">[ </emphasis><emphasis>transform,
-  </emphasis><emphasis role="bold">]</emphasis><emphasis role="bold">
-  DATASET<indexterm>
-      <primary>DATASET</primary>
-    </indexterm>(</emphasis><emphasis>outstructure</emphasis><emphasis
-  role="bold">) </emphasis><emphasis>| outstructure </emphasis><emphasis
-  role="bold">[</emphasis><emphasis>, options </emphasis><emphasis
-  role="bold">]</emphasis> <emphasis role="bold">[, UNORDERED |
+  <para><emphasis>result</emphasis><emphasis role="bold"> </emphasis><emphasis
+  role="bold"> := SOAPCALL( [ </emphasis><emphasis>recset,
+  </emphasis><emphasis role="bold">]</emphasis><emphasis> url, service,
+  instructure, </emphasis><emphasis role="bold">[
+  </emphasis><emphasis>transform, </emphasis><emphasis role="bold">]
+  DATASET(</emphasis><emphasis>outstructure</emphasis><emphasis role="bold">)
+  </emphasis><emphasis>| outstrucuture </emphasis><emphasis
+  role="bold">[</emphasis>,options <emphasis role="bold">[, UNORDERED |
   ORDERED(</emphasis> <emphasis>bool </emphasis><emphasis role="bold">) ] [,
   STABLE | UNSTABLE ] [, PARALLEL [ (</emphasis> <emphasis>numthreads
   </emphasis><emphasis role="bold">) ] ] [, ALGORITHM(</emphasis>
-  <emphasis>name </emphasis><emphasis role="bold">) ] );</emphasis></para>
+  <emphasis>name</emphasis><emphasis role="bold"> )] [, PERSIST [ (
+  </emphasis><emphasis>option</emphasis> <emphasis role="bold">) ] ] ) ;
+  <indexterm>
+      <primary>PERSIST</primary>
+    </indexterm><indexterm>
+      <primary>SOAPCALL Function</primary>
+    </indexterm><indexterm>
+      <primary>DATASET</primary>
+    </indexterm></emphasis></para>
 
   <para><emphasis role="bold">SOAPCALL( [ </emphasis><emphasis>recset,
   </emphasis><emphasis role="bold">]</emphasis><emphasis> url, service,
@@ -34,7 +37,9 @@
   ORDERED(</emphasis> <emphasis>bool </emphasis><emphasis role="bold">) ] [,
   STABLE | UNSTABLE ] [, PARALLEL [ (</emphasis> <emphasis>numthreads
   </emphasis><emphasis role="bold">) ] ] [, ALGORITHM(</emphasis>
-  <emphasis>name </emphasis><emphasis role="bold">) ] );</emphasis></para>
+  <emphasis>name</emphasis><emphasis role="bold"> )] [, PERSIST [ (
+  </emphasis><emphasis>option</emphasis> <emphasis role="bold">) ] ] ) ;
+  </emphasis></para>
 
   <para><informaltable colsep="1" frame="all" rowsep="1">
       <tgroup cols="2">
@@ -202,6 +207,22 @@
             UNSTABLE options.</entry>
           </row>
 
+          <row>
+            <entry><emphasis role="bold">PERSIST</emphasis></entry>
+
+            <entry>Optional. Use persistent connections. </entry>
+          </row>
+
+          <row>
+            <entry><emphasis>option</emphasis></entry>
+
+            <entry>Optional. If omitted, it uses the default number of
+            connections. If TRUE, it enables persistent connections. If FALSE
+            or 0, it disables persistent connections. If set to an integer, it
+            enables persistent connections and sets the number of active
+            connections. </entry>
+          </row>
+
           <row>
             <entry>Return:</entry>
 
@@ -509,6 +530,9 @@ DATASET(outRecord), ONFAIL(genDefault1())));
 OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(genDefault2(LEFT))));
 OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP)));
 
+OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP),PERSIST(12))); 
+                                                                      //use 12 persistent connections
+
 //Using HTTPHEADER to pass Authorization info
 OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP),
                 HTTPHEADER('Authorization','Basic dXNlcm5hbWU6cGFzc3dvcmQ='),

diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/Copy.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/Copy.xml
@@ -28,6 +28,8 @@
   [</emphasis> <emphasis>,preserveCompression</emphasis> <emphasis
   role="bold">] [</emphasis> <emphasis>,noSplit</emphasis> <emphasis
   role="bold">] [</emphasis> <emphasis>,expireDays</emphasis> <emphasis
+  role="bold">] [</emphasis>
+  <emphasis>,ensure</emphasis> <emphasis
   role="bold">]);</emphasis></para>
 
   <para><emphasis>dfuwuid</emphasis> <emphasis role="bold"> :=
@@ -55,7 +57,9 @@
   [</emphasis> <emphasis>,preserveCompression</emphasis> <emphasis
   role="bold">] [</emphasis> <emphasis>,noSplit</emphasis> <emphasis
   role="bold">] [</emphasis> <emphasis>,expireDays</emphasis> <emphasis
-  role="bold">]);</emphasis></para>
+  role="bold">] [</emphasis>
+  <emphasis>,ensure</emphasis><emphasis role="bold">]);
+  </emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">
@@ -204,6 +208,13 @@
           role="bold">expiryDefault</emphasis> setting.</entry>
         </row>
 
+        <row>
+          <entry><emphasis>ensure</emphasis></entry>
+
+          <entry>Optional. Copies logical file, but does not copy file parts
+          if they already exist. Default is FALSE.</entry>
+        </row>
+
         <row>
           <entry><emphasis>dfuwuid</emphasis></entry>
 

diff --git a/docs/EN_US/HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml b/docs/EN_US/HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml
@@ -645,7 +645,7 @@ dfuplus action=spray srcplane=lzstorageplane
                   If omitted, the information must be supplied by the
                   <emphasis>dstxml</emphasis> or <emphasis>dstplane</emphasis>
                   parameter. Deprecated, you should use
-                  <emphasis>dstplane</emphasis> instead. </entry>
+                  <emphasis>dstplane</emphasis> instead.</entry>
                 </row>
 
                 <row>
@@ -868,6 +868,13 @@ dfuplus action=despray srcname=mytest::test:spraytest
                   to preserve the compression of the source file. If omitted,
                   the default is 1.</entry>
                 </row>
+
+                <row>
+                  <entry><emphasis>ensure</emphasis></entry>
+
+                  <entry>Optional. Copies logical file, but does not copy file
+                  parts if they already exist. Default is FALSE.</entry>
+                </row>
               </tbody>
             </tgroup>
           </informaltable>

diff --git a/ecl/hql/hqlgram.y b/ecl/hql/hqlgram.y
@@ -3822,6 +3822,19 @@ soapFlag
                             $3.unwindCommaList(args);
                             $$.setExpr(createExprAttribute(jsonAtom, args), $1);
                         }
+    | PERSIST
+                        {
+                            $$.setExpr(createAttribute(persistAtom), $1);
+                        }
+    | PERSIST '(' expression ')'
+                        {
+                            //Allow either bool or an integer as the parameter
+                            if ($3.queryExprType()->isBoolean())
+                                parser->normalizeExpression($3, type_boolean, true);
+                            else
+                                parser->normalizeExpression($3, type_int, true);
+                            $$.setExpr(createExprAttribute(persistAtom, $3.getExpr()), $1);
+                        }
     ;
 
 onFailAction

diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp
@@ -18231,6 +18231,20 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
         OwnedHqlExpr xpathHintsExpr = createConstant(createStringValue(xpathHints.str(), xpathHints.length()));
         doBuildVarStringFunction(instance->startctx, "getXpathHintsXml", xpathHintsExpr);
     }
+
+    IHqlExpression * persistArg = queryAttributeChild(expr, persistAtom, 0);
+    if (persistArg)
+    {
+        // PERSIST(false) maps to PERSIST(0).
+        if (matchesBoolean(persistArg, false))
+            persistArg = queryZero(); // Do not generate the function, but set the pool flag so the default base function is called.
+        // PERSIST(true) maps to PERSIST with no argument,
+        else if (matchesBoolean(persistArg, true))
+            persistArg = nullptr;
+        else if (!matchesConstantValue(persistArg, 0)) // Avoid generating 0 since that is the default implementation
+            doBuildUnsignedFunction(instance->createctx, "getPersistPoolSize", persistArg);
+    }
+
     //virtual unsigned getFlags()
     {
         StringBuffer flags;
@@ -18280,6 +18294,12 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
                 flags.append("|SOAPFformEncoded");
                 break;
         }
+        if (expr->hasAttribute(persistAtom))
+        {
+            flags.append("|SOAPFpersist");
+            if (persistArg)
+                flags.append("|SOAPFpersistPool");
+        }
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
     }

diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp
@@ -3985,7 +3985,6 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
         helper.createSegmentMonitors(manager, row);
         manager->finishSegmentMonitors();
         manager->reset();
-        manager->resetCounts();
     }
 
     virtual void doneManager(IKeyManager * manager)

diff --git a/ecl/regress/soapcall9.ecl b/ecl/regress/soapcall9.ecl
@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+outRecord :=
+    RECORD
+        string name{xpath('Name')};
+        unsigned6 id{xpath('ADL')};
+        real8 score;
+    END;
+
+string myUserName := '' : stored('myUserName');
+string myPassword := '' : stored('myPassword');
+
+outRecord genDefault1() := TRANSFORM
+    SELF.name := FAILMESSAGE;
+    SELF.id := FAILCODE;
+    SELF.score := (real8)FAILMESSAGE('ip');
+    END;
+
+
+//Test all the different PERSIST variants.
+output(SOAPCALL('ip', 'service', { string20 surname := 'Hawthorn', string20 forename := 'Gavin';}, DATASET(outRecord), PERSIST));
+output(SOAPCALL('ip', 'service', { string20 surname := 'Hawthorn', string20 forename := 'Gavin';}, DATASET(outRecord), PERSIST(true)));
+output(SOAPCALL('ip', 'service', { string20 surname := 'Hawthorn', string20 forename := 'Gavin';}, DATASET(outRecord), PERSIST(false)));
+output(SOAPCALL('ip', 'service', { string20 surname := 'Hawthorn', string20 forename := 'Gavin';}, DATASET(outRecord), PERSIST(0)));
+output(SOAPCALL('ip', 'service', { string20 surname := 'Hawthorn', string20 forename := 'Gavin';}, DATASET(outRecord), PERSIST(99)));
diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp
@@ -2691,11 +2691,24 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         StringBuffer destFolder, destTitle, defaultFolder, defaultReplicateFolder;
         StringBuffer srcNodeGroup, destNodeGroup;
         bool bRoxie = false;
+        const char* srcDali = req.getSourceDali();
         const char* destNodeGroupReq = req.getDestGroup();
-        if(!destNodeGroupReq || !*destNodeGroupReq)
+        if (isEmptyString(destNodeGroupReq))
         {
-            getNodeGroupFromLFN(context, srcname, destNodeGroup);
-            DBGLOG("Destination node group not specified, using source node group %s", destNodeGroup.str());
+            CDfsLogicalFileName lfn;
+            lfn.set(srcname);
+            if (!isEmptyString(srcDali) || lfn.isForeign() || lfn.isRemote())
+            {
+                //makes no sense to get the srcname's current group, if a logical file is from a different environment.
+                if (!getDefaultStoragePlane(destNodeGroup))
+                    throw makeStringException(ECLWATCH_INVALID_INPUT, "Destination node group not specified. And no default node group found.");
+                DBGLOG("Destination node group not specified, using default node group %s", destNodeGroup.str());
+            }
+            else
+            {
+                getNodeGroupFromLFN(context, srcname, destNodeGroup);
+                DBGLOG("Destination node group not specified, using source node group %s", destNodeGroup.str());
+            }
         }
         else
         {
@@ -2721,7 +2734,6 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         constructFileMask(destTitle.str(), fileMask);
 
         Owned<IUserDescriptor> udesc=createUserDescriptor();
-        const char* srcDali = req.getSourceDali();
         const char* srcu = req.getSrcusername();
         if (!isEmptyString(srcDali) && !isEmptyString(srcu))
         {

diff --git a/esp/src/eclwatch/ClusterProcessesQueryWidget.js b/esp/src/eclwatch/ClusterProcessesQueryWidget.js
@@ -17,7 +17,9 @@ define([
     "hpcc/DelayLoadWidget",
     "hpcc/PreflightDetailsWidget",
     "hpcc/MachineInformationWidget",
-    "hpcc/IFrameWidget"
+    "hpcc/IFrameWidget",
+
+    "dijit/Dialog",
 ], function (declare, nlsHPCCMod, topic,
     registry,
     tree, selector,

diff --git a/esp/src/eclwatch/DFUQueryWidget.js b/esp/src/eclwatch/DFUQueryWidget.js
@@ -759,7 +759,8 @@ define([
                             return "";
                         }
                     },
-                    Modified: { label: this.i18n.ModifiedUTCGMT, width: 162 },
+                    Modified: { label: this.i18n.ModifiedUTCGMT, width: 160 },
+                    Accessed: { label: this.i18n.LastAccessed, width: 160 },
                     AtRestCost: {
                         label: nlsHPCC.FileCostAtRest, width: 100,
                         formatter: function (cost, row) {

diff --git a/esp/src/eclwatch/DiskUsageWidget.js b/esp/src/eclwatch/DiskUsageWidget.js
@@ -16,6 +16,7 @@ define([
     "dijit/layout/BorderContainer",
     "dijit/layout/TabContainer",
     "dijit/layout/ContentPane",
+    "dijit/Dialog",
     "dijit/Toolbar",
     "dijit/ToolbarSeparator",
     "dijit/form/Button",

diff --git a/esp/src/eclwatch/TargetClustersQueryWidget.js b/esp/src/eclwatch/TargetClustersQueryWidget.js
@@ -17,7 +17,9 @@ define([
     "hpcc/DelayLoadWidget",
     "src/ESPUtil",
     "hpcc/MachineInformationWidget",
-    "hpcc/IFrameWidget"
+    "hpcc/IFrameWidget",
+
+    "dijit/Dialog",
 ], function (declare, nlsHPCCMod, topic,
     registry,
     tree, selector,

diff --git a/esp/src/eclwatch/css/hpcc.css b/esp/src/eclwatch/css/hpcc.css
@@ -1322,7 +1322,7 @@ table.miniSelect span {
     left: 0;
     right: 0;
     height: auto;
-    border-color: rgb(158, 158, 158);
+    border-style: none;
 }
 
 .flat .dgrid-page-size {
@@ -1928,22 +1928,28 @@ span.dijitReset.dijitInline.dijitIcon.fa.disabled {
     border-top: 4px solid white;
 }
 
-.flat .phosphor_WidgetAdapter {
-    background: inherit;
+.flat .phosphor_WidgetAdapter.p-DockPanel-widget {
+    border-top: none;
+    box-shadow: 0 -1px var(--colorNeutralStroke1), var(--shadow2);
 }
 
-.flat .p-TabBar-tab {
-    background: rgb(229, 229, 229);
+.flat .phosphor_WidgetAdapter {
+    background-color: var(--colorNeutralBackground1);
+    border-color: var(--colorNeutralStroke1)
 }
 
-.flat-dark .p-TabBar-tab {
-    background: rgb(48, 48, 48);
+.flat .p-TabBar-tab {
+    background-color: var(--colorNeutralBackground3);
+    border-bottom-style: solid;
+    border-color: var(--colorNeutralStroke1);
+    border-width: 1px;
 }
 
-.flat-dark .p-TabBar-tab:hover:not(.p-mod-current) {
-    background: rgb(72, 72, 72);
+.flat .p-TabBar-tab:hover:not(.p-mod-current) {
+    background-color: var(--colorNeutralBackground3Hover);
 }
 
 .flat .p-TabBar-tab.p-mod-current {
-    background: inherit;
+    background-color: var(--colorNeutralBackground1);
+    border-bottom-color: var(--colorNeutralStroke1);
 }