Merge remote-tracking branch 'origin/candidate-9.6.x'

Signed-off-by: Gavin Halliday <[email protected]>

dali/base/dasds.cpp

@@ -1027,15 +1027,15 @@ void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned re
         {
             exception.setown(e);
             StringBuffer s(msg);
-            IERRLOG(e, s.append("writeDelta, failed").str());
+            OWARNLOG(e, s.append("writeDelta, failed").str());
         }
         if (!exception.get())
             break;
         if (0 == retrySecs)
             return;
         if (0 == --_retryAttempts)
         {
-            IWARNLOG("writeDelta, too many retry attempts [%d]", retryAttempts);
+            DISLOG("writeDelta, too many retry attempts [%d]", retryAttempts);
             return;
         }
         exception.clear();
@@ -1218,7 +1218,7 @@ class CDeltaWriter : implements IThreaded
             {
                 exception.setown(e);
                 StringBuffer err("Saving external (backup): ");
-                LOG(MCoperatorError, e, err.append(rL).str());
+                OERRLOG(e, err.append(rL).str());
             }
             if (!exception.get())
                 break;
@@ -1254,7 +1254,7 @@ class CDeltaWriter : implements IThreaded
             {
                 exception.setown(e);
                 StringBuffer err("Removing external (backup): ");
-                LOG(MCoperatorWarning, e, err.append(rL).str());
+                OWARNLOG(e, err.append(rL).str());
             }
             if (!exception.get())
                 break;
@@ -1492,7 +1492,7 @@ class CLegacyBinaryFileExternal : public CExternalFile, implements IExternalHand
             if (*_name)
                 s.append("in property ").append(_name);
             Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-            LOG(MCoperatorWarning, e, s.str());
+            OWARNLOG(e, s.str());
             if (withValue)
             {
                 StringBuffer str("EXTERNAL BINARY FILE: \"");
@@ -1574,7 +1574,7 @@ class CXMLFileExternal : public CExternalFile, implements IExternalHandler
             if (name && *name)
                 s.append("in property ").append(name);
             Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-            LOG(MCoperatorWarning, e, s.str());
+            OWARNLOG(e, s.str());
             StringBuffer str("EXTERNAL XML FILE: \"");
             str.append(filename.str()).append("\" MISSING");
             tree.setown(createPTree(owner.queryName()));
@@ -2102,7 +2102,7 @@ void CBinaryFileExternal::readValue(const char *name, MemoryBuffer &mb)
     {
         StringBuffer s("Missing external file ");
         Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-        LOG(MCoperatorWarning, e, s.str());
+        OWARNLOG(e, s.str());
         StringBuffer str("EXTERNAL BINARY FILE: \"");
         str.append(filename.str()).append("\" MISSING");
         CPTValue v(str.length()+1, str.str(), false);
@@ -2148,7 +2148,7 @@ void CBinaryFileExternal::read(const char *name, IPropertyTree &owner, MemoryBuf
             if (*_name)
                 s.append("in property ").append(_name);
             Owned<IException> e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str());
-            LOG(MCoperatorWarning, e, s.str());
+            OWARNLOG(e, s.str());
             StringBuffer str("EXTERNAL BINARY FILE: \"");
             str.append(filename.str()).append("\" MISSING");
             CPTValue v(str.length()+1, str.str(), false);
@@ -2597,7 +2597,7 @@ class CServerRemoteTree : public CRemoteTreeBase
             try { SDSManager->deleteExternal(index); }
             catch (IException *e)
             {
-                LOG(MCoperatorWarning, e, StringBuffer("Deleting external reference for ").append(queryName()).str());
+                OWARNLOG(e, StringBuffer("Deleting external reference for ").append(queryName()).str());
                 e->Release();
             }
         }
@@ -4723,7 +4723,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
         mb.append(e->errorMessage(s).str());
         StringBuffer clientUrl("EXCEPTION in reply to client ");
         mb.getSender().getEndpointHostText(clientUrl);
-        LOG(MCoperatorError, e);
+        OERRLOG(e);
     }
     try {
         CheckTime block10("DAMP_REQUEST reply");
@@ -4959,7 +4959,7 @@ IPropertyTree *loadStore(const char *storeFilename, unsigned edition, IPTreeMake
     catch (DALI_CATCHALL)
     {
         IException *e = MakeStringException(0, "Unknown exception - loading store file : %s", storeFilename);
-        LOG(MCoperatorDisaster, e, "");
+        DISLOG(e);
         if (!logErrorsOnly)
             throw;
         e->Release();
@@ -5109,7 +5109,7 @@ class CLightCoalesceThread : implements ICoalesce, public CInterface
                             t += idlePeriodSecs;
                             if (t/3600 >= STORENOTSAVE_WARNING_PERIOD && ((t-lastWarning)/3600>(STORENOTSAVE_WARNING_PERIOD/2)))
                             {
-                                OWARNLOG("Store has not been saved for %d hours", t/3600);
+                                OERRLOG("Store has not been saved for %d hours", t/3600);
                                 lastWarning = t;
                             }
                         }
@@ -5662,7 +5662,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             }
             catch (IException *e)
             {
-                OERRLOG(e, "Exception(1) - Error saving store file");
+                DISLOG(e, "Exception(1) - Error saving store file");
                 iFileIOTmpStore.clear();
                 iFileTmpStore->remove();
                 throw;
@@ -5675,7 +5675,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             refreshStoreInfo();
             if (storeInfo.edition != edition)
             {
-                OWARNLOG("Another process has updated the edition whilst saving the store: %s", newStoreNamePath.str());
+                WARNLOG("Another process has updated the edition whilst saving the store: %s", newStoreNamePath.str());
                 iFileTmpStore->remove();
                 return;
             }
@@ -5701,7 +5701,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
                 catch (IException *e)
                 {
                     StringBuffer s("Exception(2) - Error saving store file");
-                    OERRLOG(e, s.str());
+                    DISLOG(e, s.str());
                     e->Release();
                     return;
                 }
@@ -5710,7 +5710,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
                     try { renameDelta(edition, newEdition, remoteBackupLocation); }
                     catch (IException *e)
                     {
-                        LOG(MCoperatorError, e, "Failure handling backup");
+                        OERRLOG(e, "Failure handling backup");
                         e->Release();
                     }
                 }
@@ -5740,7 +5740,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
             catch (IException *e)
             {
                 StringBuffer s;
-                LOG(MCoperatorError, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation));
+                OERRLOG(e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation));
                 e->Release();
             }
 
@@ -5753,7 +5753,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
         catch (IException *e)
         {
             StringBuffer s("Exception(3) - Error saving store file");
-            OERRLOG(e, s.str());
+            DISLOG(e, s.str());
             e->Release();
         }
         if (done)
@@ -6484,12 +6484,12 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     }
     catch (IException *e)
     {
-        OERRLOG(e, "Exception - Failed to load main store");
+        DISLOG(e, "Exception - Failed to load main store");
         throw;
     }
     catch (DALI_CATCHALL)
     {
-        OERRLOG("Unknown exception - Failed to load main store");
+        DISLOG("Unknown exception - Failed to load main store");
         throw;
     }
 
@@ -6503,7 +6503,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     if (remoteBackupLocation.length())
     {
         try { validateBackup(); }
-        catch (IException *e) { LOG(MCoperatorError, e, "Validating backup"); e->Release(); }
+        catch (IException *e) { OERRLOG(e, "Validating backup"); e->Release(); }
 
         StringBuffer deltaFilename(dataPath);
         iStoreHelper->getCurrentDeltaFilename(deltaFilename);
@@ -8700,7 +8700,7 @@ bool CCovenSDSManager::fireException(IException *e)
         {
             if (handled)
             {
-                LOG(MCoperatorDisaster, e, "FATAL, too many exceptions");
+                DISLOG(e, "FATAL, too many exceptions");
                 return false; // did not successfully handle.
             }
             IERRLOG(e, "Exception while restarting or shutting down");
@@ -8737,7 +8737,7 @@ bool CCovenSDSManager::fireException(IException *e)
                 }
                 manager.unhandledThread.clear();
             }
-            catch (IException *_e) { LOG(MCoperatorError, _e, "Exception while restarting or shutting down"); _e->Release(); }
+            catch (IException *_e) { OERRLOG(_e, "Exception while restarting or shutting down"); _e->Release(); }
             catch (DALI_CATCHALL) { IERRLOG("Unknown exception while restarting or shutting down"); }
             if (!restart)
             {
@@ -8846,7 +8846,7 @@ bool CDeltaWriter::save(std::queue<Owned<CTransactionItem>> &todo)
         }
         catch (IException *e)
         {
-            LOG(MCoperatorWarning, e, "save: failed to touch delta in progress file");
+            OERRLOG(e, "save: failed to touch delta in progress file");
             e->Release();
         }
         // here if exception only
@@ -8909,14 +8909,14 @@ bool CDeltaWriter::save(std::queue<Owned<CTransactionItem>> &todo)
     }
     catch (IException *e)
     {
-        LOG(MCoperatorWarning, e, "save: failure whilst committing deltas to disk! Remedial action must be taken");
+        OERRLOG("save: failure whilst committing deltas to disk! Remedial action must be taken");
         e->Release();
         // this is really an attempt at disaster recovery at this point
         forceBlockingSave = true;
     }
     if (forceBlockingSave)
     {
-        LOG(MCoperatorWarning, "Due to earlier failures, attempting forced/blocking save of Dali store");
+        OWARNLOG("Due to earlier failures, attempting forced/blocking save of Dali store");
         while (todo.size())
             todo.pop();
         SDSManager->saveStore(nullptr, false, false);

dali/base/dasess.cpp

@@ -1963,11 +1963,11 @@ bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned time
                 return true;
             }
         }
-        StringBuffer str;
-        OERRLOG("Waiting for Dali to be available - server: %s", comm->queryGroup().queryNode(r).endpoint().getEndpointHostText(str).str());
+        StringBuffer str("Waiting for Dali to be available - server: ");
+        comm->queryGroup().queryNode(r).endpoint().getEndpointHostText(str);
         if (tm.timedout())
         {
-            PROGLOG("%s", str.append(" Timed out.").str());
+            OWARNLOG("%s", str.append(" Timed out.").str());
             break;
         }
         else if (0 == nextLog)

dockerfiles/image.sh

@@ -97,6 +97,7 @@ finalize_platform_core_image() {
             hpccsystems/platform-core:$GIT_BRANCH-$MODE-$crc "tail -f /dev/null")
         docker exec --user root --workdir /hpcc-dev $CONTAINER /bin/bash -c "rm -rf /hpcc-dev/HPCC-Platform && mkdir /hpcc-dev/HPCC-Platform && chown -R hpcc:hpcc /hpcc-dev/HPCC-Platform"
         docker exec --workdir /hpcc-dev $CONTAINER /bin/bash -c "git config --global --add safe.directory /hpcc-dev/HPCC-Platform-local"
+        docker exec --workdir /hpcc-dev $CONTAINER /bin/bash -c "git config --global --add safe.directory /hpcc-dev/HPCC-Platform-local/.git"
         docker exec --workdir /hpcc-dev $CONTAINER /bin/bash -c "git clone --single-branch file:///hpcc-dev/HPCC-Platform-local /hpcc-dev/HPCC-Platform"
         docker exec --workdir /hpcc-dev/HPCC-Platform $CONTAINER /bin/bash -c "git reset --hard --recurse-submodules"
         docker exec --workdir /hpcc-dev/HPCC-Platform-local $CONTAINER /bin/bash -c "git ls-files --modified --exclude-standard -z | xargs -0 -I {} cp {} /hpcc-dev/HPCC-Platform/{}"

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml

@@ -509,7 +509,7 @@
 
               <entry>components</entry>
 
-              <entry></entry>
+              <entry/>
             </row>
           </tbody>
         </tgroup>
@@ -839,8 +839,43 @@
         format:</para>
 
         <programlisting>ds := DATASET('~remote::hpcc2::somescope1::somelfn1', rec);</programlisting>
+      </sect3>
+
+      <sect3>
+        <title>Preferred Storage</title>
+
+        <para>The <emphasis>preferredReadPlanes</emphasis> option is available
+        for each type of cluster--hThor, Thor, and Roxie. </para>
+
+        <para>This option is only significant for logical files which reside
+        on multiple storage planes. When specified, the HPCC Systems platform
+        will seek to read files from the preferred plane(s) if a file resides
+        on them. These preferred planes must exist and be defined in
+        <emphasis>storage.planes </emphasis></para>
+
+        <para>The following is an example of a Thor cluster configured with
+        the preferredDataReadPlanes option.</para>
+
+        <programlisting>thor: 
+- name: thor 
+  prefix: thor 
+  numWorkers: 2
+  maxJobs: 4
+  maxGraphs: 3 
+  preferredDataReadPlanes: 
+  - data-copy
+  - indexdata-copy
+</programlisting>
+
+        <para>In the above example, running a query that reads a file that
+        resides on both "data" and "data-copy" (in that order) normally would
+        read the first copy on "data". With that
+        <emphasis>preferredDataReadPlanes</emphasis> specified, if that file
+        also resides on "data-copy", Thor will read that copy.</para>
 
-        <para></para>
+        <para>This can be useful when there are multiple copies of files on
+        different planes with different characteristics that can impact
+        performance.</para>
       </sect3>
     </sect2>
 
@@ -1031,7 +1066,7 @@ thor:
         integrating applications in different domains. CORS defines how client
         web applications in one domain can interact with resources in another
         domain. You can configure CORS support settings in the ESP section of
-        the values.yaml file as illustrated below: </para>
+        the values.yaml file as illustrated below:</para>
 
         <programlisting>esp:
 - name: eclwatch
@@ -1297,9 +1332,9 @@ thor:
 
         <informaltable colsep="1" frame="all" rowsep="1">
           <tgroup cols="2">
-            <colspec colwidth="125.55pt" />
+            <colspec colwidth="125.55pt"/>
 
-            <colspec />
+            <colspec/>
 
             <tbody>
               <row>

esp/bindings/http/platform/httptransport.cpp

@@ -2123,22 +2123,23 @@ int CHttpRequest::processHeaders(IMultiException *me)
 
 bool CHttpRequest::readContentToBuffer(MemoryBuffer& buffer, __int64& bytesNotRead)
 {
-    char buf[1024 + 1];
-    __int64 buflen = 1024;
-    if (buflen > bytesNotRead)
-        buflen = bytesNotRead;
-
-    int readlen = m_bufferedsocket->read(buf, (int) buflen);
-    if(readlen < 0)
-        DBGLOG("Failed to read from socket");
-
-    if(readlen <= 0)
-       return false;
-
-    buf[readlen] = 0;
-    buffer.append(readlen, buf);//'buffer' may have some left-over from previous read
+    constexpr size32_t readChunkSize = 0x100000;
+    size32_t sizeToRead = bytesNotRead > readChunkSize ? readChunkSize: (size32_t)bytesNotRead;
+    size32_t prevLen = buffer.length();
+
+    // BufferedSocket::read buffer must be at least one larger than its maxlen argument
+    char * target = (char *)buffer.reserve(sizeToRead + 1);
+    int readLen = m_bufferedsocket->read(target, sizeToRead);
+    if(readLen <= 0)
+    {
+        if(readLen < 0)
+            DBGLOG("Failed to read from socket");
+        buffer.setLength(prevLen);
+        return false;
+    }
 
-    bytesNotRead -= readlen;
+    buffer.setLength(prevLen + readLen);
+    bytesNotRead -= readLen;
     return true;
 }
 

esp/src/eclwatch/GetDFUWorkunitsWidget.js

@@ -369,8 +369,6 @@ define([
         initWorkunitsGrid: function () {
             var context = this;
             var filter = this.filter.toObject();
-            filter.includeTimings = true;
-            filter.includeTransferRate = true;
             var store = this.params.searchResults ? this.params.searchResults : new ESPDFUWorkunit.CreateWUQueryStore();
             this.workunitsGrid = new declare([ESPUtil.Grid(true, true, false, false, "GetDFUWorkunitsWidget")])({
                 store: store,
@@ -477,8 +475,6 @@ define([
 
         refreshGrid: function (clearSelection) {
             var filter = this.filter.toObject();
-            filter.includeTimings = true;
-            filter.includeTransferRate = true;
             this.workunitsGrid.set("query", filter);
             if (clearSelection) {
                 this.workunitsGrid.clearSelection();

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -69,7 +69,7 @@
                                         <input id="${id}CopyPreserveCompression" title="${i18n.PreserveCompression}:" checked="true" name="preserveCompression" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyTargetReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyExpireDays" title="${i18n.ExpireDays}:" name="ExpireDays" data-dojo-type="dijit.form.NumberTextBox" />
-                                        <input id="${i18n.NoCommon}" title="${i18n.NoCommon}:" checked="true" name="noCommon" data-dojo-type="dijit.form.CheckBox"/>
+                                        <input title="${i18n.NoCommon}:" checked="true" name="noCommon" data-dojo-type="dijit.form.CheckBox"/>
                                         <input id="${id}MaxConnections" title="${i18n.MaxConnections}:" name="maxConnections" data-dojo-type="dijit.form.NumberTextBox" />
                                     </div>
                                 </div>