Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Jake Smith <[email protected]>

.gitignore

@@ -16,5 +16,6 @@ eclcc.log
 *.pyc
 ./helm/examples/azure/sa/env-sa
 ./dockerfiles/platform-build-incremental/hpcc.gitpatch
+dockerfiles/platform-build-incremental/hpcc.gitpatch
 .env
 /vcpkg.json

common/workunit/workunit.cpp

@@ -4423,6 +4423,8 @@ class CLockedWorkUnit : implements ILocalWorkUnit, implements IExtendedWUInterfa
             { return c->getHash(); }
     virtual IStringIterator *getLogs(const char *type, const char *instance) const
             { return c->getLogs(type, instance); }
+    virtual IPropertyTreeIterator *getProcessTypes() const
+            { return c->getProcessTypes(); }
     virtual IStringIterator *getProcesses(const char *type) const
             { return c->getProcesses(type); }
     virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const
@@ -8616,6 +8618,13 @@ IStringIterator *CLocalWorkUnit::getLogs(const char *type, const char *instance)
         return new CStringPTreeAttrIterator(p->getElements(xpath.str()), "@log");
 }
 
+IPropertyTreeIterator* CLocalWorkUnit::getProcessTypes() const
+{
+    VStringBuffer xpath("Process/*");
+    CriticalBlock block(crit);
+    return p->getElements(xpath.str());
+}
+
 IPropertyTreeIterator* CLocalWorkUnit::getProcesses(const char *type, const char *instance) const
 {
     VStringBuffer xpath("Process/%s/", type);

common/workunit/workunit.hpp

@@ -1292,6 +1292,7 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo
     virtual const IPropertyTree * getXmlParams() const = 0;
     virtual unsigned __int64 getHash() const = 0;
     virtual IStringIterator *getLogs(const char *type, const char *instance=NULL) const = 0;
+    virtual IPropertyTreeIterator *getProcessTypes() const = 0;
     virtual IStringIterator *getProcesses(const char *type) const = 0;
     virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const = 0;
 

common/workunit/workunit.ipp

@@ -267,6 +267,7 @@ public:
     virtual const IPropertyTree *getXmlParams() const;
     virtual unsigned __int64 getHash() const;
     virtual IStringIterator *getLogs(const char *type, const char *component) const;
+    virtual IPropertyTreeIterator *getProcessTypes() const;
     virtual IStringIterator *getProcesses(const char *type) const;
     virtual IPropertyTreeIterator* getProcesses(const char *type, const char *instance) const;
     virtual IStringVal & getSnapshot(IStringVal & str) const;

dali/base/dadfs.cpp

@@ -11173,12 +11173,21 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I
                 Owned<IPropertyTree> tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false);
                 if (tree)
                 {
-#ifdef _CONTAINERIZED
-                    // This is for bare-metal clients using ~foreign pointing at a containerized/k8s setup,
-                    // asking for the returned meta data to be remapped to point to the dafilesrv service.
-                    if (hasMask(opts, GetFileTreeOpts::remapToService))
-                        remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
-#endif
+                    if (isContainerized())
+                    {
+                        // This is for bare-metal clients using ~foreign pointing at a containerized/k8s setup,
+                        // asking for the returned meta data to be remapped to point to the dafilesrv service.
+                        if (hasMask(opts, GetFileTreeOpts::remapToService))
+                        {
+                            remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
+
+                            const char *remotePlaneName = tree->queryProp("@group");
+                            Owned<IPropertyTree> filePlane = getStoragePlane(remotePlaneName);
+                            assertex(filePlane);
+                            // Used by DFS clients to determine if stripe and/or alias translation needed
+                            tree->setPropTree("Attr/_remoteStoragePlane", createPTreeFromIPT(filePlane));
+                        }
+                    }
 
                     Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES);
                     mb.append((int)1); // 1 == standard file

dali/dfu/dfurun.cpp

@@ -1442,6 +1442,8 @@ class CDFUengine: public CInterface, implements IDFUengine
                 }
                 break;
             }
+
+            bool ensureLfnAlreadyPublished = false;
             // fill dstfile for commands that need it
             switch (cmd) {
             case DFUcmd_copymerge:
@@ -1542,6 +1544,14 @@ class CDFUengine: public CInterface, implements IDFUengine
                                 Owned<IDistributedFile> oldfile = wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE);
                                 if (oldfile)
                                 {
+                                    if (options->getEnsure())
+                                    {
+                                        // logical file already exists.
+                                        ensureLfnAlreadyPublished = true;
+                                        dstFile.setown(oldfile.getClear());
+                                        dstName.set(tmp);
+                                        break;
+                                    }
                                     StringBuffer reason;
                                     bool canRemove = oldfile->canRemove(reason);
                                     oldfile.clear();
@@ -1700,12 +1710,35 @@ class CDFUengine: public CInterface, implements IDFUengine
                             }
                         }
                         else {
-                            fsys.copy(srcFile,dstFile,recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid);
-                            if (!abortnotify.abortRequested()) {
-                                if (needrep)
-                                    replicating = true;
+                            bool performCopy = true;
+                            if (options->getEnsure())
+                            {
+                                if (ensureLfnAlreadyPublished)
+                                    performCopy = false;
                                 else
-                                    dstFile->attach(dstName.get(),userdesc);
+                                {
+                                    if (dstFile->existsPhysicalPartFiles(0))
+                                    {
+                                        dstFile->attach(dstName.get(), userdesc);
+                                        performCopy = false;
+                                    }
+                                }
+                                if (!performCopy)
+                                {
+                                    feedback.repmode=cProgressReporter::REPnone;
+                                    feedback.displaySummary(nullptr, 0);
+                                    Audit("COPYENSURE", userdesc, srcFile?srcName.str():nullptr, dstName.get());
+                                }
+                            }
+                            if (performCopy)
+                            {
+                                fsys.copy(srcFile,dstFile,recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid);
+                                if (!abortnotify.abortRequested()) {
+                                    if (needrep)
+                                        replicating = true;
+                                    else
+                                        dstFile->attach(dstName.get(),userdesc);
+                                }
                                 Audit("COPY",userdesc,srcFile?srcName.str():NULL,dstName.get());
                             }
                         }

dali/dfu/dfuwu.cpp

@@ -2002,6 +2002,11 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
         return queryRoot()->getPropInt("@overwrite")!=0;
     }
 
+    bool getEnsure() const
+    {
+        return queryRoot()->getPropInt("@ensure")!=0;
+    }
+
     DFUreplicateMode getReplicateMode(StringBuffer &cluster, bool &repeatlast,bool &onlyrepeated) const
     {
         repeatlast = false;
@@ -2146,7 +2151,7 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
         queryRoot()->setPropInt("@throttle",val);
     }
 
-    void setTransferBufferSize(unsigned val)
+    void setTransferBufferSize(size32_t val)
     {
         queryRoot()->setPropInt("@transferBufferSize",val);
     }
@@ -2161,6 +2166,11 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
         queryRoot()->setPropInt("@overwrite",val?1:0);
     }
 
+    void setEnsure(bool val=true)
+    {
+        queryRoot()->setPropInt("@ensure",val?1:0);
+    }
+
     void setReplicateMode(DFUreplicateMode val,const char *cluster=NULL,bool repeatlast=false,bool onlyrepeated=false)
     {
         queryRoot()->setPropInt("@replicatemode",(int)val);

dali/dfu/dfuwu.hpp

@@ -153,6 +153,7 @@ interface IConstDFUoptions : extends IInterface
     virtual size32_t getTransferBufferSize() const = 0;
     virtual bool getVerify() const = 0;
     virtual bool getOverwrite() const = 0;
+    virtual bool getEnsure() const = 0;
     virtual DFUreplicateMode getReplicateMode(StringBuffer &cluster, bool &repeatlast,bool &onlyrepeated) const = 0;
     virtual const char *queryPartFilter() const = 0;
     virtual bool getKeepHeader() const = 0;
@@ -195,6 +196,7 @@ interface IDFUoptions : extends IConstDFUoptions
     virtual void setTransferBufferSize(size32_t val) = 0;
     virtual void setVerify(bool val=true) = 0;
     virtual void setOverwrite(bool val=true) = 0;
+    virtual void setEnsure(bool val=true) = 0;
     virtual void setReplicateMode(DFUreplicateMode val,const char *cluster=NULL,bool repeatlast=false,bool onlyrepeated=false) = 0;
     virtual void setPartFilter(const char *filter) = 0;             // format n,n-n,n etc
     virtual void setKeepHeader(bool val=true) = 0;

esp/bindings/http/platform/httpbinding.cpp

@@ -1124,6 +1124,8 @@ void EspHttpBinding::handleHttpPost(CHttpRequest *request, CHttpResponse *respon
     IEspCache *cacheClient = nullptr;
     IEspContext &context = *request->queryContext();
 
+    request->annotateSpan("http.request.method", "POST");
+
     IEspContainer *espContainer = getESPContainer();
     if (espContainer->hasCacheClient() && (cacheMethods > 0)
         && queryCacheSeconds(request->queryServiceMethod(), cacheSeconds)) //ESP cache is needed for this method
@@ -1190,6 +1192,7 @@ int EspHttpBinding::onGet(CHttpRequest* request, CHttpResponse* response)
 {
     IEspContext& context = *request->queryContext();
 
+    request->annotateSpan("http.request.method", "GET");
     // At this time, the request is already received and fully passed, and
     // the user authenticated
     LogLevel level = getEspLogLevel(&context);

esp/bindings/http/platform/httpservice.cpp

@@ -386,6 +386,9 @@ int CEspHttpServer::processRequest()
                     return 0;
                 }
 
+                //Avoids unrestrictedSSType requests
+                m_request->startSpan();
+
                 if(stricmp(method.str(), POST_METHOD)==0)
                     thebinding->handleHttpPost(m_request.get(), m_response.get());
                 else if(!stricmp(method.str(), GET_METHOD))

esp/bindings/http/platform/httptransport.cpp

@@ -1915,6 +1915,18 @@ int CHttpRequest::receive(IMultiException *me)
     return 0;
 }
 
+void CHttpRequest::startSpan()
+{
+    //MORE: The previous code would be better off querying httpHeaders...
+    Owned<IProperties> httpHeaders = getHeadersAsProperties(m_headers);
+    Owned<ISpan> requestSpan = queryTraceManager().createServerSpan("HTTPRequest", httpHeaders, SpanFlags::EnsureGlobalId);
+    m_context->setActiveSpan(requestSpan);
+}
+
+void CHttpRequest::annotateSpan(const char * key, const char * value)
+{
+    m_context->queryActiveSpan()->setSpanAttribute(key, value);
+}
 
 void CHttpRequest::updateContext()
 {
@@ -1982,11 +1994,6 @@ void CHttpRequest::updateContext()
         m_context->setUseragent(useragent.str());
         getHeader("Accept-Language", acceptLanguage);
         m_context->setAcceptLanguage(acceptLanguage.str());
-
-        //MORE: The previous code would be better off querying httpHeaders...
-        Owned<IProperties> httpHeaders = getHeadersAsProperties(m_headers);
-        Owned<ISpan> requestSpan = queryTraceManager().createServerSpan("request", httpHeaders, SpanFlags::EnsureGlobalId);
-        m_context->setActiveSpan(requestSpan);
     }
 }
 

esp/bindings/http/platform/httptransport.ipp

@@ -369,7 +369,9 @@ public:
 
     virtual int receive(IMultiException *me);
 
+    void startSpan();
     void updateContext();
+    void annotateSpan(const char * key, const char * value);
 
     virtual void setMaxRequestEntityLength(int len) {m_MaxRequestEntityLength = len;}
     virtual int getMaxRequestEntityLength() { return m_MaxRequestEntityLength; }

esp/platform/espcontext.cpp

@@ -114,6 +114,7 @@ class CEspContext : public CInterface, implements IEspContext
         updateTraceSummaryHeader();
         m_secureContext.setown(secureContext);
         m_SecurityHandler.setSecureContext(secureContext);
+        m_activeSpan.set(getNullSpan());
     }
 
     ~CEspContext()
@@ -633,29 +634,21 @@ class CEspContext : public CInterface, implements IEspContext
     {
         return m_activeSpan;
     }
-    //GH Can these be deleted?
+
     virtual const char* getGlobalId() const override
     {
-        if (!m_activeSpan)
-            return nullptr;
         return m_activeSpan->queryGlobalId();
     }
     virtual const char* getCallerId() const override
     {
-        if (!m_activeSpan)
-            return nullptr;
         return m_activeSpan->queryCallerId();
     }
     virtual const char* getLocalId() const override
     {
-        if (!m_activeSpan)
-            return nullptr;
         return m_activeSpan->queryLocalId();
     }
     virtual IProperties * getClientSpanHeaders() const override
     {
-        if (!m_activeSpan)
-            return nullptr;
         return ::getClientHeaders(m_activeSpan);
     }
 };

esp/scm/ws_fs.ecm

@@ -481,6 +481,7 @@ ESPrequest [nil_remove] Copy
     string srcusername;
     string srcpassword;
     bool   overwrite;
+    bool   ensure;
     bool   replicate;
     int    ReplicateOffset(1);
 

esp/scm/ws_workunits.ecm

@@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp);
 
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.97"), default_client_version("1.97"), cache_group("ESPWsWUs"),
+    version("1.98"), default_client_version("1.98"), cache_group("ESPWsWUs"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/scm/ws_workunits_req_resp.ecm

@@ -357,6 +357,7 @@ ESPrequest WUInfoRequest
     [min_ver("1.66")] bool IncludeAllowedClusters(true);
     [min_ver("1.73")] bool IncludeTotalClusterTime(true);
     [min_ver("1.78")] bool IncludeServiceNames(false);
+    [min_ver("1.98")] bool IncludeProcesses(false);
     [min_ver("1.16")] bool SuppressResultSchemas(false);
     [min_ver("1.25")] string ThorSlaveIP;
 };

esp/scm/ws_workunits_struct.ecm

@@ -339,6 +339,18 @@ ESPStruct [nil_remove] ThorLogInfo
     int    NumberSlaves;
 };
 
+ESPStruct [nil_remove] ECLWUProcess
+{
+    string Name;
+    string Type;
+    string PodName;           //containerized only
+    int    InstanceNumber(1); //containerized only
+    string Log;               //bare metal only
+    string PID;               //bare metal only
+    string Pattern;           //bare metal only
+    int    Max(1);            //bare metal only
+};
+
 ESPStruct [nil_remove] ECLWorkunitLW
 {
     string Wuid;
@@ -442,6 +454,7 @@ ESPStruct [nil_remove] ECLWorkunit
     [min_ver("1.85")] double FileAccessCost;
     [min_ver("1.87")] double CompileCost;
     [min_ver("1.91")] bool NoAccess(false);
+    [min_ver("1.98")] ESParray<ESPstruct ECLWUProcess> ECLWUProcessList;
 };
 
 ESPStruct [nil_remove] WUECLAttribute

esp/services/ws_fs/ws_fsService.cpp

@@ -2790,6 +2790,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         wuFSpecDest->setLogicalName(dstname);
         wuFSpecDest->setFileMask(fileMask.str());
         wuOptions->setOverwrite(req.getOverwrite());
+        wuOptions->setEnsure(req.getEnsure());
         wuOptions->setPreserveCompression(req.getPreserveCompression());
         if (!req.getExpireDays_isNull())
             wuOptions->setExpireDays(req.getExpireDays());