Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…

…9.6.x

Signed-off-by: Gavin Halliday <[email protected]>

dali/base/dasds.cpp

@@ -1027,15 +1027,15 @@ void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned re
         {
             exception.setown(e);
             StringBuffer s(msg);
-            IERRLOG(e, s.append("writeDelta, failed").str());
+            OWARNLOG(e, s.append("writeDelta, failed").str());
         }
         if (!exception.get())
             break;
         if (0 == retrySecs)
             return;
         if (0 == --_retryAttempts)
         {
-            IWARNLOG("writeDelta, too many retry attempts [%d]", retryAttempts);
+            DISLOG("writeDelta, too many retry attempts [%d]", retryAttempts);
             return;
         }
         exception.clear();
@@ -1218,7 +1218,7 @@ class CDeltaWriter : implements IThreaded
             {
                 exception.setown(e);
                 StringBuffer err("Saving external (backup): ");
-                LOG(MCoperatorError, e, err.append(rL).str());
+                OERRLOG(e, err.append(rL).str());
             }
             if (!exception.get())
                 break;
@@ -1254,7 +1254,7 @@ class CDeltaWriter : implements IThreaded
             {
                 exception.setown(e);
                 StringBuffer err("Removing external (backup): ");
-                LOG(MCoperatorWarning, e, err.append(rL).str());
+                OWARNLOG(e, err.append(rL).str());
             }
             if (!exception.get())
                 break;
@@ -1492,7 +1492,7 @@ class CLegacyBinaryFileExternal : public CExternalFile, implements IExternalHand
             if (*_name)
                 s.append("in property ").append(_name);
             Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-            LOG(MCoperatorWarning, e, s.str());
+            OWARNLOG(e, s.str());
             if (withValue)
             {
                 StringBuffer str("EXTERNAL BINARY FILE: \"");
@@ -1574,7 +1574,7 @@ class CXMLFileExternal : public CExternalFile, implements IExternalHandler
             if (name && *name)
                 s.append("in property ").append(name);
             Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-            LOG(MCoperatorWarning, e, s.str());
+            OWARNLOG(e, s.str());
             StringBuffer str("EXTERNAL XML FILE: \"");
             str.append(filename.str()).append("\" MISSING");
             tree.setown(createPTree(owner.queryName()));
@@ -2102,7 +2102,7 @@ void CBinaryFileExternal::readValue(const char *name, MemoryBuffer &mb)
     {
         StringBuffer s("Missing external file ");
         Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-        LOG(MCoperatorWarning, e, s.str());
+        OWARNLOG(e, s.str());
         StringBuffer str("EXTERNAL BINARY FILE: \"");
         str.append(filename.str()).append("\" MISSING");
         CPTValue v(str.length()+1, str.str(), false);
@@ -2148,7 +2148,7 @@ void CBinaryFileExternal::read(const char *name, IPropertyTree &owner, MemoryBuf
             if (*_name)
                 s.append("in property ").append(_name);
             Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-            LOG(MCoperatorWarning, e, s.str());
+            OWARNLOG(e, s.str());
             StringBuffer str("EXTERNAL BINARY FILE: \"");
             str.append(filename.str()).append("\" MISSING");
             CPTValue v(str.length()+1, str.str(), false);
@@ -2597,7 +2597,7 @@ class CServerRemoteTree : public CRemoteTreeBase
             try { SDSManager->deleteExternal(index); }
             catch (IException *e)
             {
-                LOG(MCoperatorWarning, e, StringBuffer("Deleting external reference for ").append(queryName()).str());
+                OWARNLOG(e, StringBuffer("Deleting external reference for ").append(queryName()).str());
                 e->Release();
             }
         }
@@ -4723,7 +4723,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
         mb.append(e->errorMessage(s).str());
         StringBuffer clientUrl("EXCEPTION in reply to client ");
         mb.getSender().getEndpointHostText(clientUrl);
-        LOG(MCoperatorError, e);
+        OERRLOG(e);
     }
     try {
         CheckTime block10("DAMP_REQUEST reply");
@@ -4959,7 +4959,7 @@ IPropertyTree *loadStore(const char *storeFilename, unsigned edition, IPTreeMake
     catch (DALI_CATCHALL)
     {
         IException *e = MakeStringException(0, "Unknown exception - loading store file : %s", storeFilename);
-        LOG(MCoperatorDisaster, e, "");
+        DISLOG(e);
         if (!logErrorsOnly)
             throw;
         e->Release();
@@ -5109,7 +5109,7 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface
                             t += idlePeriodSecs;
                             if (t/3600 >= STORENOTSAVE_WARNING_PERIOD && ((t-lastWarning)/3600>(STORENOTSAVE_WARNING_PERIOD/2)))
                             {
-                                OWARNLOG("Store has not been saved for %d hours", t/3600);
+                                OERRLOG("Store has not been saved for %d hours", t/3600);
                                 lastWarning = t;
                             }
                         }
@@ -5662,7 +5662,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             }
             catch (IException *e)
             {
-                OERRLOG(e, "Exception(1) - Error saving store file");
+                DISLOG(e, "Exception(1) - Error saving store file");
                 iFileIOTmpStore.clear();
                 iFileTmpStore->remove();
                 throw;
@@ -5675,7 +5675,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             refreshStoreInfo();
             if (storeInfo.edition != edition)
             {
-                OWARNLOG("Another process has updated the edition whilst saving the store: %s", newStoreNamePath.str());
+                WARNLOG("Another process has updated the edition whilst saving the store: %s", newStoreNamePath.str());
                 iFileTmpStore->remove();
                 return;
             }
@@ -5701,7 +5701,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
                 catch (IException *e)
                 {
                     StringBuffer s("Exception(2) - Error saving store file");
-                    OERRLOG(e, s.str());
+                    DISLOG(e, s.str());
                     e->Release();
                     return;
                 }
@@ -5710,7 +5710,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
                     try { renameDelta(edition, newEdition, remoteBackupLocation); }
                     catch (IException *e)
                     {
-                        LOG(MCoperatorError, e, "Failure handling backup");
+                        OERRLOG(e, "Failure handling backup");
                         e->Release();
                     }
                 }
@@ -5740,7 +5740,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             catch (IException *e)
             {
                 StringBuffer s;
-                LOG(MCoperatorError, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation));
+                OERRLOG(e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation));
                 e->Release();
             }
 
@@ -5753,7 +5753,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
         catch (IException *e)
         {
             StringBuffer s("Exception(3) - Error saving store file");
-            OERRLOG(e, s.str());
+            DISLOG(e, s.str());
             e->Release();
         }
         if (done)
@@ -6484,12 +6484,12 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     }
     catch (IException *e)
     {
-        OERRLOG(e, "Exception - Failed to load main store");
+        DISLOG(e, "Exception - Failed to load main store");
         throw;
     }
     catch (DALI_CATCHALL)
     {
-        OERRLOG("Unknown exception - Failed to load main store");
+        DISLOG("Unknown exception - Failed to load main store");
         throw;
     }
 
@@ -6503,7 +6503,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     if (remoteBackupLocation.length())
     {
         try { validateBackup(); }
-        catch (IException *e) { LOG(MCoperatorError, e, "Validating backup"); e->Release(); }
+        catch (IException *e) { OERRLOG(e, "Validating backup"); e->Release(); }
 
         StringBuffer deltaFilename(dataPath);
         iStoreHelper->getCurrentDeltaFilename(deltaFilename);
@@ -8700,7 +8700,7 @@ bool CCovenSDSManager::fireException(IException *e)
         {
             if (handled)
             {
-                LOG(MCoperatorDisaster, e, "FATAL, too many exceptions");
+                DISLOG(e, "FATAL, too many exceptions");
                 return false; // did not successfully handle.
             }
             IERRLOG(e, "Exception while restarting or shutting down");
@@ -8737,7 +8737,7 @@ bool CCovenSDSManager::fireException(IException *e)
                 }
                 manager.unhandledThread.clear();
             }
-            catch (IException *_e) { LOG(MCoperatorError, _e, "Exception while restarting or shutting down"); _e->Release(); }
+            catch (IException *_e) { OERRLOG(_e, "Exception while restarting or shutting down"); _e->Release(); }
             catch (DALI_CATCHALL) { IERRLOG("Unknown exception while restarting or shutting down"); }
             if (!restart)
             {
@@ -8846,7 +8846,7 @@ bool CDeltaWriter::save(std::queue<Owned<CTransactionItem>> &todo)
         }
         catch (IException *e)
         {
-            LOG(MCoperatorWarning, e, "save: failed to touch delta in progress file");
+            OERRLOG(e, "save: failed to touch delta in progress file");
             e->Release();
         }
         // here if exception only
@@ -8909,14 +8909,14 @@ bool CDeltaWriter::save(std::queue<Owned<CTransactionItem>> &todo)
     }
     catch (IException *e)
     {
-        LOG(MCoperatorWarning, e, "save: failure whilst committing deltas to disk! Remedial action must be taken");
+        OERRLOG("save: failure whilst committing deltas to disk! Remedial action must be taken");
         e->Release();
         // this is really an attempt at disaster recovery at this point
         forceBlockingSave = true;
     }
     if (forceBlockingSave)
     {
-        LOG(MCoperatorWarning, "Due to earlier failures, attempting forced/blocking save of Dali store");
+        OWARNLOG("Due to earlier failures, attempting forced/blocking save of Dali store");
         while (todo.size())
             todo.pop();
         SDSManager->saveStore(nullptr, false, false);

dali/base/dasess.cpp

@@ -1963,11 +1963,11 @@ bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned time
                 return true;
             }
         }
-        StringBuffer str;
-        OERRLOG("Waiting for Dali to be available - server: %s", comm->queryGroup().queryNode(r).endpoint().getEndpointHostText(str).str());
+        StringBuffer str("Waiting for Dali to be available - server: ");
+        comm->queryGroup().queryNode(r).endpoint().getEndpointHostText(str);
         if (tm.timedout())
         {
-            PROGLOG("%s", str.append(" Timed out.").str());
+            OWARNLOG("%s", str.append(" Timed out.").str());
             break;
         }
         else if (0 == nextLog)

dockerfiles/image.sh

@@ -97,6 +97,7 @@ finalize_platform_core_image() {
             hpccsystems/platform-core:$GIT_BRANCH-$MODE-$crc "tail -f /dev/null")
         docker exec --user root --workdir /hpcc-dev $CONTAINER /bin/bash -c "rm -rf /hpcc-dev/HPCC-Platform && mkdir /hpcc-dev/HPCC-Platform && chown -R hpcc:hpcc /hpcc-dev/HPCC-Platform"
         docker exec --workdir /hpcc-dev $CONTAINER /bin/bash -c "git config --global --add safe.directory /hpcc-dev/HPCC-Platform-local"
+        docker exec --workdir /hpcc-dev $CONTAINER /bin/bash -c "git config --global --add safe.directory /hpcc-dev/HPCC-Platform-local/.git"
         docker exec --workdir /hpcc-dev $CONTAINER /bin/bash -c "git clone --single-branch file:///hpcc-dev/HPCC-Platform-local /hpcc-dev/HPCC-Platform"
         docker exec --workdir /hpcc-dev/HPCC-Platform $CONTAINER /bin/bash -c "git reset --hard --recurse-submodules"
         docker exec --workdir /hpcc-dev/HPCC-Platform-local $CONTAINER /bin/bash -c "git ls-files --modified --exclude-standard -z | xargs -0 -I {} cp {} /hpcc-dev/HPCC-Platform/{}"

esp/src/eclwatch/GetDFUWorkunitsWidget.js

@@ -369,8 +369,6 @@ define([
         initWorkunitsGrid: function () {
             var context = this;
             var filter = this.filter.toObject();
-            filter.includeTimings = true;
-            filter.includeTransferRate = true;
             var store = this.params.searchResults ? this.params.searchResults : new ESPDFUWorkunit.CreateWUQueryStore();
             this.workunitsGrid = new declare([ESPUtil.Grid(true, true, false, false, "GetDFUWorkunitsWidget")])({
                 store: store,
@@ -477,8 +475,6 @@ define([
 
         refreshGrid: function (clearSelection) {
             var filter = this.filter.toObject();
-            filter.includeTimings = true;
-            filter.includeTransferRate = true;
             this.workunitsGrid.set("query", filter);
             if (clearSelection) {
                 this.workunitsGrid.clearSelection();

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -69,7 +69,7 @@
                                         <input id="${id}CopyPreserveCompression" title="${i18n.PreserveCompression}:" checked="true" name="preserveCompression" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyTargetReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyExpireDays" title="${i18n.ExpireDays}:" name="ExpireDays" data-dojo-type="dijit.form.NumberTextBox" />
-                                        <input id="${i18n.NoCommon}" title="${i18n.NoCommon}:" checked="true" name="noCommon" data-dojo-type="dijit.form.CheckBox"/>
+                                        <input title="${i18n.NoCommon}:" checked="true" name="noCommon" data-dojo-type="dijit.form.CheckBox"/>
                                         <input id="${id}MaxConnections" title="${i18n.MaxConnections}:" name="maxConnections" data-dojo-type="dijit.form.NumberTextBox" />
                                     </div>
                                 </div>

esp/src/src-react/components/DFUWorkunits.tsx

@@ -39,8 +39,6 @@ function formatQuery(_filter): { [id: string]: any } {
     if (filter.Type === true) {
         filter.Type = "archived workunits";
     }
-    filter.includeTimings = true;
-    filter.includeTransferRate = true;
     return filter;
 }
 
@@ -83,7 +81,7 @@ export const DFUWorkunits: React.FunctionComponent<DFUWorkunitsProps> = ({
 
     //  Grid ---
     const gridStore = React.useMemo(() => {
-        return store || ESPDFUWorkunit.CreateWUQueryStore({});
+        return store || ESPDFUWorkunit.CreateWUQueryStore();
     }, [store]);
 
     const query = React.useMemo(() => {

esp/src/src/ESPDFUWorkunit.ts

@@ -8,6 +8,11 @@ import * as FileSpray from "./FileSpray";
 import nlsHPCC from "./nlsHPCC";
 import * as Utility from "./Utility";
 
+import { FileSprayService, FileSpray as FileSprayNS } from "@hpcc-js/comms";
+
+import { Paged } from "./store/Paged";
+import { BaseStore } from "./store/Store";
+
 const i18n = nlsHPCC;
 
 class Store extends ESPRequest.Store {
@@ -297,14 +302,37 @@ export function isInstanceOfWorkunit(obj) {
 export function Get(wuid, data?) {
     const store = new Store();
     const retVal = store.get(wuid);
-    if (data) {
+    if (data && !retVal.__hpcc_id) {
         retVal.updateData(data);
     }
     return retVal;
 }
 
-export function CreateWUQueryStore(options) {
+export function CreateWUQueryStoreLegacy(options) {
     let store = new Store(options);
     store = new Observable(store);
     return store;
 }
+
+const service = new FileSprayService({ baseUrl: "" });
+
+export type WUQueryStore = BaseStore<FileSprayNS.GetDFUWorkunits, typeof Workunit>;
+
+export function CreateWUQueryStore(): BaseStore<FileSprayNS.GetDFUWorkunits, typeof Workunit> {
+    const store = new Paged<FileSprayNS.GetDFUWorkunits, typeof Workunit>({
+        start: "PageStartFrom",
+        count: "PageSize",
+        sortBy: "Sortby",
+        descending: "Descending"
+    }, "ID", request => {
+        request.includeTimings = true;
+        request.includeTransferRate = true;
+        return service.GetDFUWorkunits(request).then(response => {
+            return {
+                data: response.results.DFUWorkunit.map(wu => Get(wu.ID, wu)),
+                total: response.NumWUs
+            };
+        });
+    });
+    return new Observable(store);
+}

esp/src/src/ESPLogicalFile.ts

@@ -472,7 +472,7 @@ export function Get(Cluster, Name, data?) {
     }
     const store = new Store();
     const retVal = store.get(createID(Cluster, Name));
-    if (data) {
+    if (data && !retVal.__hpcc_id) {
         lang.mixin(data, {
             __hpcc_id: createID(data.NodeGroup, data.Name),
             __hpcc_isDir: false,

esp/src/src/FileSpray.ts

@@ -327,6 +327,8 @@ export function CreateLandingZonesFilterStore(options) {
 }
 
 export function GetDFUWorkunits(params) {
+    params.request.includeTimings = true;
+    params.request.includeTransferRate = true;
     return ESPRequest.send("FileSpray", "GetDFUWorkunits", params);
 }
 

helm/hpcc/templates/dfuserver.yaml

@@ -47,7 +47,7 @@ kind: Deployment
 metadata:
   name: {{ .name | quote }}
 spec:
-  replicas: 1
+  replicas: {{ .replicas | default 1 }}
   selector:
     matchLabels:
       run: {{ .name | quote }}