Merge pull request #18242 from jakesmith/HPCC-30996-roxie-copy-from-wsdfs

HPCC-30996 Fix issues with roxie copying from remote storage source

Reviewed-By: Anthony Fishbeck <[email protected]>
Reviewed-by: Gavin Halliday <[email protected]>
Merged-by: Gavin Halliday <[email protected]>

Author: ghalliday

dali/dfu/dfuutil.cpp

@@ -406,7 +406,7 @@ class CFileCloner
             throw afor2.exc.getClear();
     }
 
-    void updateCloneFrom(const char *lfn, IPropertyTree &attrs, IFileDescriptor *srcfdesc, INode *srcdali, const char *srcCluster)
+    void updateCloneFrom(const char *lfn, IPropertyTree &attrs, IFileDescriptor *srcfdesc, const IPropertyTree *srcTree, INode *srcdali, const char *srcCluster)
     {
         DBGLOG("updateCloneFrom %s", lfn);
         if (!srcdali || srcdali->endpoint().isNull())
@@ -421,8 +421,17 @@ class CFileCloner
             if (prefix.length())
                 attrs.setProp("@cloneFromPrefix", prefix.get());
 
-            while(attrs.removeProp("cloneFromGroup"));
 
+            // for now, only use source file descriptor as cloned source if it's from
+            // wsdfs file backed by remote storage using dafilesrv (NB: if it is '_remoteStoragePlane' will be set)
+            // JCSMORE: it may be this can replace the need for the other 'clone*' attributes altogether.
+            if (srcfdesc->queryProperties().hasProp("_remoteStoragePlane"))
+            {
+                attrs.setPropTree("cloneFromFDesc", createPTreeFromIPT(srcTree));
+                return;
+            }
+
+            while(attrs.removeProp("cloneFromGroup"));
             unsigned numClusters = srcfdesc->numClusters();
             for (unsigned clusterNum = 0; clusterNum < numClusters; clusterNum++)
             {
@@ -438,15 +447,15 @@ class CFileCloner
             }
         }
     }
-    void updateCloneFrom(IDistributedFile *dfile, IFileDescriptor *srcfdesc, INode *srcdali, const char *srcCluster)
+    void updateCloneFrom(IDistributedFile *dfile, IFileDescriptor *srcfdesc, const IPropertyTree *srcTree, INode *srcdali, const char *srcCluster)
     {
         DistributedFilePropertyLock lock(dfile);
         IPropertyTree &attrs = lock.queryAttributes();
-        updateCloneFrom(dfile->queryLogicalName(), attrs, srcfdesc, srcdali, srcCluster);
+        updateCloneFrom(dfile->queryLogicalName(), attrs, srcfdesc, srcTree, srcdali, srcCluster);
     }
-    void updateCloneFrom(const char *lfn, IFileDescriptor *dstfdesc, IFileDescriptor *srcfdesc, INode *srcdali, const char *srcCluster)
+    void updateCloneFrom(const char *lfn, IFileDescriptor *dstfdesc, IFileDescriptor *srcfdesc, const IPropertyTree *srcTree, INode *srcdali, const char *srcCluster)
     {
-        updateCloneFrom(lfn, dstfdesc->queryProperties(), srcfdesc, srcdali, srcCluster);
+        updateCloneFrom(lfn, dstfdesc->queryProperties(), srcfdesc, srcTree, srcdali, srcCluster);
     }
 
     void cloneSubFile(IPropertyTree *ftree,const char *destfilename, INode *srcdali, const char *srcCluster)   // name already has prefix added
@@ -455,7 +464,11 @@ class CFileCloner
         const char * kind = srcfdesc->queryProperties().queryProp("@kind");
         bool iskey = kind&&(strcmp(kind,"key")==0);
 
-        Owned<IFileDescriptor> dstfdesc = createFileDescriptor(srcfdesc->getProperties());
+        Owned<IPropertyTree> dstProps = createPTreeFromIPT(&srcfdesc->queryProperties());
+        // If present, we do not want this as part of the cloned properties of this new local roxie file
+        dstProps->removeProp("_remoteStoragePlane");
+        Owned<IFileDescriptor> dstfdesc = createFileDescriptor(dstProps.getClear());
+
         if (!nameprefix.isEmpty())
             dstfdesc->queryProperties().setProp("@roxiePrefix", nameprefix.get());
         if (!copyphysical)
@@ -507,7 +520,7 @@ class CFileCloner
         }
 
         if (!copyphysical) //cloneFrom tells roxie where to copy from.. it's unnecessary if we already did the copy
-            updateCloneFrom(destfilename, dstfdesc, srcfdesc, srcdali, srcCluster);
+            updateCloneFrom(destfilename, dstfdesc, srcfdesc, ftree, srcdali, srcCluster);
         else
         {
             DBGLOG("copyphysical dst=%s", destfilename);
@@ -771,7 +784,7 @@ class CFileCloner
                 {
                     Owned<IFileDescriptor> dstfdesc=dfile->getFileDescriptor();
                     Owned<IFileDescriptor> srcfdesc = deserializeFileDescriptorTree(ftree, NULL, 0);
-                    updateCloneFrom(filename, dstfdesc, srcfdesc, srcdali, srcCluster);
+                    updateCloneFrom(filename, dstfdesc, srcfdesc, ftree, srcdali, srcCluster);
                 }
                 return;
             }
@@ -832,6 +845,16 @@ class CFileCloner
         }
         else
         {
+            IPropertyTree *dstClonedFDesc = dfile->queryAttributes().queryPropTree("cloneFromFDesc");
+            IPropertyTree *srcClonedFDesc = srcfdesc->queryProperties().queryPropTree("cloneFromFDesc");
+            if (dstClonedFDesc && srcClonedFDesc)
+            {
+                // if both based on cloneFromFDesc, no need to check other varieties.
+                return !areMatchingPTrees(dstClonedFDesc, srcClonedFDesc);
+            }
+            else if (dstClonedFDesc || srcClonedFDesc) // one has cloneFromFDesc, the other doesn't
+                return true;
+            // else - neither based on cloneFromFDesc
             StringBuffer s;
             if (checkValueChanged(dfile->queryAttributes().queryProp("@cloneFrom"), srcdali->endpoint().getUrlStr(s).str()))
                 return true;
@@ -921,7 +944,7 @@ class CFileCloner
                 {
                     Owned<IFileDescriptor> srcfdesc = deserializeFileDescriptorTree(ftree, NULL, 0);
                     if (checkCloneFromChanged(dfile, srcfdesc, srcdali, srcCluster))
-                        updateCloneFrom(dfile, srcfdesc, srcdali, srcCluster);
+                        updateCloneFrom(dfile, srcfdesc, ftree, srcdali, srcCluster);
                 }
                 return;
             }

roxie/ccd/ccddali.cpp

@@ -356,6 +356,13 @@ class CRoxieDaliHelper : implements IRoxieDaliHelper, public CInterface
 
     IFileDescriptor *recreateCloneSource(IFileDescriptor *srcfdesc, const char *destfilename)
     {
+        IPropertyTree *clonedFDescTree = srcfdesc->queryProperties().queryPropTree("cloneFromFDesc");
+        if (clonedFDescTree)
+        {
+            Owned<IFileDescriptor> srcFDesc = deserializeFileDescriptorTree(clonedFDescTree);
+            return srcFDesc.getClear();
+        }
+
         Owned<IFileDescriptor> dstfdesc = createFileDescriptor(srcfdesc->getProperties());
         // calculate dest dir
 
@@ -596,7 +603,7 @@ class CRoxieDaliHelper : implements IRoxieDaliHelper, public CInterface
             return resolveCachedLFN(foreignLfn);  // Note - cache only used when no dali connection available
         try
         {
-            if (fdesc->queryProperties().hasProp("cloneFromGroup") && fdesc->queryProperties().hasProp("@cloneFromDir"))
+            if (fdesc->queryProperties().hasProp("cloneFromFDesc") || (fdesc->queryProperties().hasProp("cloneFromGroup") && fdesc->queryProperties().hasProp("@cloneFromDir")))
             {
                 Owned<IFileDescriptor> ret = recreateCloneSource(fdesc, _lfn);
                 if (cacheIt)