From 0b26b17d84bb789bb41b38802e0edd62c1c2a41e Mon Sep 17 00:00:00 2001 From: Andrew Klymenko Date: Thu, 26 Dec 2024 12:51:18 -0500 Subject: [PATCH 1/3] VDB-5758: Updated status message when downloading a kart; don't try to download after resolve failure --- tools/external/prefetch/PrfMain.h | 5 +++-- tools/external/prefetch/prefetch.c | 36 +++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/tools/external/prefetch/PrfMain.h b/tools/external/prefetch/PrfMain.h index 31b348b2d..8fa5a6fdd 100644 --- a/tools/external/prefetch/PrfMain.h +++ b/tools/external/prefetch/PrfMain.h @@ -40,7 +40,8 @@ typedef struct { typedef struct { ERunType type; - char *name; /* name to resolve */ + char *name; /* name to resolve */ + const char *id; /* numeric ID for kart items */ VPathStr local; const struct String *cache; @@ -193,7 +194,7 @@ rc_t PrfMainFini(PrfMain *self); #define DISP_RC(rc, err) (void)((rc == 0) ? 0 : LOGERR(klogInt, rc, err)) #define DISP_RC2(rc, msg, name) (void)((rc == 0) ? 0 : \ - PLOGERR(klogInt, (klogInt,rc, "$(msg): $(name)","msg=%s,name=%s",msg,name))) + PLOGERR(klogInt, (klogInt,rc, "$(msg) [$(name)]","msg=%s,name=%s",msg,name))) #define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false) diff --git a/tools/external/prefetch/prefetch.c b/tools/external/prefetch/prefetch.c index 95223cc02..ed7161289 100644 --- a/tools/external/prefetch/prefetch.c +++ b/tools/external/prefetch/prefetch.c @@ -347,6 +347,8 @@ static rc_t V_ResolverRemote(const VResolver *self, id = item -> seq_id; } + if (id == NULL) + id = resolved->id; if ( id == NULL ) id = resolved -> name; @@ -2133,8 +2135,11 @@ static rc_t ItemInit(Item *self, const char *obj) { return 0; } -static char* ItemName(const Item *self) { +static char* ItemName(const Item *self, const char **id) { char *c = NULL; + const char *dummy = NULL; + if (id == NULL) + id = &dummy; assert(self); if (self->desc != NULL) return string_dup_measure(self->desc, NULL); @@ -2144,6 +2149,8 @@ static char* ItemName(const Item *self) { rc_t rc = 0; const String *elem = NULL; assert(self->item); + rc = KartItemItemId(self->item, &elem); + *id = StringCheck(elem, rc); /* rc = KartItemItemDesc(self->item, &elem); c = StringCheck(elem, rc); @@ -2157,6 +2164,11 @@ static char* ItemName(const Item *self) { return c; } + rc = KartItemName(self->item, &elem); + c = StringCheck(elem, rc); + if (c != NULL) + return c; + rc = KartItemItemId(self->item, &elem); return StringCheck(elem, rc); } @@ -2500,6 +2512,19 @@ static rc_t _ItemResolveResolved(VResolver *resolver, return rc; } } + + if (rc == 0 + && ( + rc3 == RC(rcNS, rcFile, rcOpening, rcFile, rcNotFound) + /* 404 */ + || + rc3 == RC(rcNS, rcFile, rcOpening, rcFile, rcUnexpected) + /* Unexpected HTTP status */ + ) + ) + { + rc = rc3; + } } } @@ -2546,7 +2571,7 @@ static rc_t ItemInitResolved(Item *self, VResolver *resolver, KDirectory *dir, assert(self && self->mane); resolved = &self->resolved; - resolved->name = ItemName(self); + resolved->name = ItemName(self, &resolved->id); assert(resolved->type != eRunTypeUnknown); @@ -3069,7 +3094,7 @@ rc_t ItemResolveResolvedAndDownloadOrProcess(Item *self, int32_t row) rc_t rc = 0; char * itemName = NULL; assert(self); - itemName = ItemName(self); + itemName = ItemName(self, NULL); if (dbgRc == 0xffffffff) { if (getenv("VDB5693") != NULL) @@ -3180,7 +3205,7 @@ static rc_t ItemDownloadDependencies(Item *item) { } if (resolved->path.str != NULL) { - char * itemName = ItemName(item); + char * itemName = ItemName(item, NULL); STSMSG(STS_TOP, ("%d) Resolving '%s's dependencies...", item->number, itemName)); rc = PrfMainDependenciesList(item->mane, resolved, &deps); @@ -3753,12 +3778,11 @@ static rc_t PrfMainRun ( PrfMain * self, const char * arg, const char * realArg, if (it.kart != NULL) { STSMSG(STS_TOP, ("Downloading kart file '%s'", realArg)); if (type == eRunTypeGetSize) - STSMSG(STS_TOP, ("Checking sizes of kart files...")); + STSMSG(STS_TOP,("Checking the sizes of kart files...")); } else if (self->jwtCart != NULL) STSMSG(STS_TOP, ( "Downloading jwt cart file '%s'", self->jwtCart)); - // OUTMSG(("\n")); } #ifdef DBGNG From 1858db9a7600ce3e4c5d6e7c138168fc2c1bdafe Mon Sep 17 00:00:00 2001 From: Andrew Klymenko Date: Thu, 26 Dec 2024 12:54:08 -0500 Subject: [PATCH 2/3] VDB-5758: test of a bad kart file --- test/external/prefetch/CMakeLists.txt | 10 +++++ test/external/prefetch/data/bad-kart.krt | Bin 0 -> 294 bytes test/external/prefetch/expected/err | 4 ++ test/external/prefetch/expected/out | 11 +++++ test/external/prefetch/test-bad-kart.sh | 51 +++++++++++++++++++++++ 5 files changed, 76 insertions(+) create mode 100644 test/external/prefetch/data/bad-kart.krt create mode 100644 test/external/prefetch/expected/err create mode 100644 test/external/prefetch/expected/out create mode 100755 test/external/prefetch/test-bad-kart.sh diff --git a/test/external/prefetch/CMakeLists.txt b/test/external/prefetch/CMakeLists.txt index a1d50f218..345b6d2c3 100644 --- a/test/external/prefetch/CMakeLists.txt +++ b/test/external/prefetch/CMakeLists.txt @@ -117,6 +117,16 @@ if ( NOT WIN32 ) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) endif() + add_test( NAME Test_Prefetch_Bad_Kart + COMMAND + env + dbgap|110600={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110600\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]} + dbgap|110601={\"\":500,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110601\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e\"}]}]}]} + dbgap|110602={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110602\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]} + dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]} + ./test-bad-kart.sh ${DIRTOTEST} prefetch + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + add_test( NAME SlowTest_Prefetch_vdbcache COMMAND perl vdbcache.pl ${DIRTOTEST} prefetch 0 # 23 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/test/external/prefetch/data/bad-kart.krt b/test/external/prefetch/data/bad-kart.krt new file mode 100644 index 0000000000000000000000000000000000000000..aef25a572e0e41be763e0b87a6057be5cfafe1f7 GIT binary patch literal 294 zcmV+>0ond;V`6D*VRCdIiwFP*n`CDI1GSMsYlAQthVMfE!*>{LJ2XUHw{Ax-Q)s(V zEgnN=J~2ZRF=@9#e*Cl^+JdmtZplmDJWui+u`^OD@Co(8MX(H=@6%^WolEUOoY)Rq z-C5FBhEhKWUr6?`U87+6d&=@SA;%}u24uZoWeI6TIkuoXx1iJ=2>q>U%>@*?X|+OS z&Fq@wJGa9cxk0wk7R>}|0<(!%Q3B(EH^_B^>fS(%ytWl2N?OUSJR&^w<;Mw!-tf!> zH85|40_Pg8WQoSIKb~C#Bwk1JWEagh+x>rwh0a}k27$^_x_DjooABzdSbF&U!om{4 sEoVaZtDmNJS%okpFeLrNd3PG;(`N=P+}t+v?obKu7q{_{yd44n00e4_mH+?% literal 0 HcmV?d00001 diff --git a/test/external/prefetch/expected/err b/test/external/prefetch/expected/err new file mode 100644 index 000000000..7fe2370dc --- /dev/null +++ b/test/external/prefetch/expected/err @@ -0,0 +1,4 @@ +file not found while opening file within network system module - cannot open remote file [https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found] +file unexpected while opening file within network system module - cannot open remote file [https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e] +file not found while opening file within network system module - cannot open remote file [https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found] +file unexpected while opening file within network system module - cannot open remote file [https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi] diff --git a/test/external/prefetch/expected/out b/test/external/prefetch/expected/out new file mode 100644 index 000000000..ba95d5188 --- /dev/null +++ b/test/external/prefetch/expected/out @@ -0,0 +1,11 @@ +Downloading kart file 'data/bad-kart.krt' +Checking the sizes of kart files... +1) Resolving 'Study_Report.phs001237.TOPMed_WGS_WHI.v4.p2.MULTI.pdf'... +Current preference is set to retrieve SRA Normalized Format files with full base quality scores +1) Failed to resolve 'Study_Report.phs001237.TOPMed_WGS_WHI.v4.p2.MULTI.pdf'... +2) Resolving 'Release_Notes.phs001237.TOPMed_WHI.v4.p2.MULTI.pdf'... +2) Failed to resolve 'Release_Notes.phs001237.TOPMed_WHI.v4.p2.MULTI.pdf'... +3) Resolving 'manifest_phs001237.TOPMed_WGS_WHI.v4.p2.c2.HMB-IRB-NPU.pdf'... +3) Failed to resolve 'manifest_phs001237.TOPMed_WGS_WHI.v4.p2.c2.HMB-IRB-NPU.pdf'... +4) Resolving 'phs001237.v4.pht005987.v3.TOPMed_WHI_Subject.data_dict.xml'... +4) Failed to resolve 'phs001237.v4.pht005987.v3.TOPMed_WHI_Subject.data_dict.xml'... diff --git a/test/external/prefetch/test-bad-kart.sh b/test/external/prefetch/test-bad-kart.sh new file mode 100755 index 000000000..1560e9cc1 --- /dev/null +++ b/test/external/prefetch/test-bad-kart.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +if [ $# -lt 2 ]; then + echo "Usage $0 " + exit 1 +fi + +bin_dir=$1 +prefetch=$2 +tool=$1/$2 + +echo "testing processing of a bad kart file via $tool" + +if [ "" != "" ]; then + perl -e 'print $ENV{"dbgap|110600"};print "=dbgap-110600\n"' + perl -e 'print $ENV{"dbgap|110601"};print "=dbgap-110601\n"' + perl -e 'print $ENV{"dbgap|110602"};print "=dbgap-110602\n"' + perl -e 'print $ENV{"dbgap|110603"};print "=dbgap-110603\n"' +fi + +i=0; while [ $i -le 3 ]; do +# echo "dbgap|11060$i" + E=$(perl -e "print \$ENV{'dbgap|11060$i'}") + if [ "$E" == "" ]; then + echo "dbgap|11060$i env.var. is not set" + i=$((i+2)) + exit $i + fi + i=$((i+1)) +done + +mkdir -p actual || exit 4 + +$tool --ngc data/prj_phs710EA_test.ngc data/bad-kart.krt > actual/out \ + 2> actual/err +res=$? +if [ $res -eq 0 ] ; then + echo "prefetch of a bad kart file succeed when failure was expected" + exit 7 +fi + +cat actual/out | perl -e'foreach(<>){s/.*: //;print}' > actual/outer +cat actual/err | perl -e \ + 'foreach (<>) { next if /_ItemResolveResolved/; s/.*: //; print }' \ + > actual/errer +diff actual/outer expected/out || exit 8 +diff actual/errer expected/err || exit 9 + +rm -r actual || exit 10 + +exit 0 From 9a0624e9c11966bd890a463e1b93af156f177ba2 Mon Sep 17 00:00:00 2001 From: Andrew Klymenko Date: Thu, 26 Dec 2024 13:12:17 -0500 Subject: [PATCH 3/3] VDB-5758: added Test_Prefetch_Bad_Kart-asan/-tsan --- test/external/prefetch/CMakeLists.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/external/prefetch/CMakeLists.txt b/test/external/prefetch/CMakeLists.txt index 345b6d2c3..abe8e82db 100644 --- a/test/external/prefetch/CMakeLists.txt +++ b/test/external/prefetch/CMakeLists.txt @@ -126,6 +126,28 @@ if ( NOT WIN32 ) dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]} ./test-bad-kart.sh ${DIRTOTEST} prefetch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + if( TARGET prefetch-asan ) + add_test( NAME Test_Prefetch_Bad_Kart-asan + COMMAND + env + dbgap|110600={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110600\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]} + dbgap|110601={\"\":500,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110601\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e\"}]}]}]} + dbgap|110602={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110602\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]} + dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]} + ./test-bad-kart.sh ${DIRTOTEST} prefetch-asan + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + endif() + if( TARGET prefetch-tsan ) + add_test( NAME Test_Prefetch_Bad_Kart-tsan + COMMAND + env + dbgap|110600={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110600\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]} + dbgap|110601={\"\":500,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110601\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?jwt=e\"}]}]}]} + dbgap|110602={\"\":404,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110602\",\"locations\":[{\"link\":\"https://www.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi?debug=not-found\"}]}]}]} + dbgap|110603={\"\":400,\"result\":[{\"status\":200,\"files\":[{\"object\":\"dbgap|110603\",\"locations\":[{\"link\":\"https://trace.ncbi.nlm.nih.gov/Traces/sdlr/sdlr.cgi\"}]}]}]} + ./test-bad-kart.sh ${DIRTOTEST} prefetch-tsan + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + endif() add_test( NAME SlowTest_Prefetch_vdbcache COMMAND perl vdbcache.pl ${DIRTOTEST} prefetch 0 # 23