Vdb 5758

test/external/prefetch/CMakeLists.txt

@@ -117,6 +117,38 @@ if ( NOT WIN32 )
 			WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
 	endif()
 
+	add_test( NAME Test_Prefetch_Bad_Kart
+        COMMAND
+         env
+          dbgap|110600={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110600\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]}
+          dbgap|110601={\"\":500,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110601\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e\"}]}]}]}
+          dbgap|110602={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110602\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]}
+          dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]}
+          ./test-bad-kart.sh ${DIRTOTEST} prefetch
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
+	if( TARGET prefetch-asan )
+      add_test( NAME Test_Prefetch_Bad_Kart-asan
+        COMMAND
+         env
+          dbgap|110600={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110600\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]}
+          dbgap|110601={\"\":500,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110601\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e\"}]}]}]}
+          dbgap|110602={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110602\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]}
+          dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]}
+          ./test-bad-kart.sh ${DIRTOTEST} prefetch-asan
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
+	endif()
+	if( TARGET prefetch-tsan )
+      add_test( NAME Test_Prefetch_Bad_Kart-tsan
+        COMMAND
+         env
+          dbgap|110600={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110600\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]}
+          dbgap|110601={\"\":500,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110601\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e\"}]}]}]}
+          dbgap|110602={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110602\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]}
+          dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]}
+          ./test-bad-kart.sh ${DIRTOTEST} prefetch-tsan
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
+	endif()
+
 	add_test( NAME SlowTest_Prefetch_vdbcache
 		COMMAND perl vdbcache.pl ${DIRTOTEST} prefetch 0 # 23
 		WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )

test/external/prefetch/expected/err

@@ -0,0 +1,4 @@
+file not found while opening file within network system module - cannot open remote file [https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found]
+file unexpected while opening file within network system module - cannot open remote file [https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e]
+file not found while opening file within network system module - cannot open remote file [https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found]
+file unexpected while opening file within network system module - cannot open remote file [https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi]

test/external/prefetch/expected/out

@@ -0,0 +1,11 @@
+Downloading kart file 'data/bad-kart.krt'
+Checking the sizes of kart files...
+1) Resolving 'Study_Report.phs001237.TOPMed_WGS_WHI.v4.p2.MULTI.pdf'...
+Current preference is set to retrieve SRA Normalized Format files with full base quality scores
+1) Failed to resolve 'Study_Report.phs001237.TOPMed_WGS_WHI.v4.p2.MULTI.pdf'...
+2) Resolving 'Release_Notes.phs001237.TOPMed_WHI.v4.p2.MULTI.pdf'...
+2) Failed to resolve 'Release_Notes.phs001237.TOPMed_WHI.v4.p2.MULTI.pdf'...
+3) Resolving 'manifest_phs001237.TOPMed_WGS_WHI.v4.p2.c2.HMB-IRB-NPU.pdf'...
+3) Failed to resolve 'manifest_phs001237.TOPMed_WGS_WHI.v4.p2.c2.HMB-IRB-NPU.pdf'...
+4) Resolving 'phs001237.v4.pht005987.v3.TOPMed_WHI_Subject.data_dict.xml'...
+4) Failed to resolve 'phs001237.v4.pht005987.v3.TOPMed_WHI_Subject.data_dict.xml'...

test/external/prefetch/test-bad-kart.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+if [ $# -lt 2 ]; then
+  echo "Usage $0 <bin-dir> <tool-name>"
+  exit 1
+fi
+
+bin_dir=$1
+prefetch=$2
+tool=$1/$2
+
+echo "testing processing of a bad kart file via $tool"
+
+if [ "" != "" ]; then
+    perl -e 'print $ENV{"dbgap|110600"};print "=dbgap-110600\n"'
+    perl -e 'print $ENV{"dbgap|110601"};print "=dbgap-110601\n"'
+    perl -e 'print $ENV{"dbgap|110602"};print "=dbgap-110602\n"'
+    perl -e 'print $ENV{"dbgap|110603"};print "=dbgap-110603\n"'
+fi
+
+i=0; while [ $i -le 3 ]; do
+# echo "dbgap|11060$i"
+  E=$(perl -e "print \$ENV{'dbgap|11060$i'}")
+  if [ "$E" == "" ]; then
+    echo "dbgap|11060$i env.var. is not set"
+    i=$((i+2))
+    exit $i
+    fi
+  i=$((i+1))
+done
+
+mkdir -p actual || exit 4
+
+$tool --ngc data/prj_phs710EA_test.ngc data/bad-kart.krt > actual/out \
+                                                        2> actual/err
+res=$?
+if [ $res -eq 0 ] ; then
+  echo "prefetch of a bad kart file succeed when failure was expected"
+  exit 7
+fi
+
+cat actual/out | perl -e'foreach(<>){s/.*: //;print}' > actual/outer
+cat actual/err | perl -e \
+     'foreach (<>) { next if /_ItemResolveResolved/; s/.*: //; print }' \
+    > actual/errer
+diff actual/outer expected/out || exit 8
+diff actual/errer expected/err || exit 9
+
+rm -r actual || exit 10
+
+exit 0

tools/external/prefetch/PrfMain.h

@@ -40,7 +40,8 @@ typedef struct {
 
 typedef struct {
     ERunType type;
-    char *name; /* name to resolve */
+    char *name;     /* name to resolve */
+    const char *id; /* numeric ID for kart items */
 
     VPathStr      local;
     const struct String *cache;
@@ -193,7 +194,7 @@ rc_t PrfMainFini(PrfMain *self);
 #define DISP_RC(rc, err) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, err))
 
 #define DISP_RC2(rc, msg, name) (void)((rc == 0) ? 0 : \
-    PLOGERR(klogInt, (klogInt,rc, "$(msg): $(name)","msg=%s,name=%s",msg,name)))
+   PLOGERR(klogInt, (klogInt,rc, "$(msg) [$(name)]","msg=%s,name=%s",msg,name)))
 
 #define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
     if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)

tools/external/prefetch/prefetch.c

@@ -347,6 +347,8 @@ static rc_t V_ResolverRemote(const VResolver *self,
         id = item -> seq_id;
     }
 
+    if (id == NULL)
+        id = resolved->id;
     if ( id == NULL )
         id = resolved -> name;
 
@@ -2133,8 +2135,11 @@ static rc_t ItemInit(Item *self, const char *obj) {
     return 0;
 }
 
-static char* ItemName(const Item *self) {
+static char* ItemName(const Item *self, const char **id) {
     char *c = NULL;
+    const char *dummy = NULL;
+    if (id == NULL)
+        id = &dummy;
     assert(self);
     if (self->desc != NULL)
         return string_dup_measure(self->desc, NULL);
@@ -2144,6 +2149,8 @@ static char* ItemName(const Item *self) {
         rc_t rc = 0;
         const String *elem = NULL;
         assert(self->item);
+        rc = KartItemItemId(self->item, &elem);
+        *id = StringCheck(elem, rc);
         /*
         rc = KartItemItemDesc(self->item, &elem);
         c = StringCheck(elem, rc);
@@ -2157,6 +2164,11 @@ static char* ItemName(const Item *self) {
             return c;
         }
 
+        rc = KartItemName(self->item, &elem);
+        c = StringCheck(elem, rc);
+        if (c != NULL)
+            return c;
+
         rc = KartItemItemId(self->item, &elem);
         return StringCheck(elem, rc);
     }
@@ -2500,6 +2512,19 @@ static rc_t _ItemResolveResolved(VResolver *resolver,
                         return rc;
                 }
             }
+
+            if (rc == 0
+                && (
+                    rc3 == RC(rcNS, rcFile, rcOpening, rcFile, rcNotFound)
+                    /* 404 */
+                    ||
+                    rc3 == RC(rcNS, rcFile, rcOpening, rcFile, rcUnexpected)
+                    /* Unexpected HTTP status */
+                    )
+                )
+            {
+                rc = rc3;
+            }
         }
     }
 
@@ -2546,7 +2571,7 @@ static rc_t ItemInitResolved(Item *self, VResolver *resolver, KDirectory *dir,
     assert(self && self->mane);
 
     resolved = &self->resolved;
-    resolved->name = ItemName(self);
+    resolved->name = ItemName(self, &resolved->id);
 
     assert(resolved->type != eRunTypeUnknown);
 
@@ -3069,7 +3094,7 @@ rc_t ItemResolveResolvedAndDownloadOrProcess(Item *self, int32_t row)
     rc_t rc = 0;
     char * itemName = NULL;
     assert(self);
-    itemName = ItemName(self);
+    itemName = ItemName(self, NULL);
 
     if (dbgRc == 0xffffffff) {
         if (getenv("VDB5693") != NULL)
@@ -3180,7 +3205,7 @@ static rc_t ItemDownloadDependencies(Item *item) {
     }
 
     if (resolved->path.str != NULL) {
-        char * itemName = ItemName(item);
+        char * itemName = ItemName(item, NULL);
         STSMSG(STS_TOP, ("%d) Resolving '%s's dependencies...",
             item->number, itemName));
         rc = PrfMainDependenciesList(item->mane, resolved, &deps);
@@ -3753,12 +3778,11 @@ static rc_t PrfMainRun ( PrfMain * self, const char * arg, const char * realArg,
                 if (it.kart != NULL) {
                     STSMSG(STS_TOP, ("Downloading kart file '%s'", realArg));
                     if (type == eRunTypeGetSize)
-                        STSMSG(STS_TOP, ("Checking sizes of kart files..."));
+                        STSMSG(STS_TOP,("Checking the sizes of kart files..."));
                 }
                 else if (self->jwtCart != NULL)
                     STSMSG(STS_TOP, (
                         "Downloading jwt cart file '%s'", self->jwtCart));
-            //    OUTMSG(("\n"));
             }
 
 #ifdef DBGNG