Python Coverage.py version differences, GitHub regression infrastructure. See #343, #348

.github/workflows/run_test_suite.yml

@@ -81,23 +81,18 @@ jobs:
       - name: make check
         run: |-
           set -x -o pipefail
-          # NOTE: There are two things going on in this hackery:
-          #       - So far "make check" exits with code 0 despite failures —
-          #         see issue #348 — so we need a more manual approach to detect
-          #         failing tests
-          #       - We compare the number of failing tests to the known status
-          #         quo — see issue #343 — so that
-          #         - we have a chance for a green CI while also
-          #         - we will notice when more of the existing tests start
-          #           to fail.
-          make check |& tee /dev/stderr \
-            | grep -F ' failed, ' | tee /dev/stderr \
-            | grep -F -q ', 1 failed, ' \
-            || { echo 'Number of tests expected to fail^^ does not match -- did you break an existing test?' >&2 ; false ; }
+          make check
 
       - name: Upload test log as an artifact
         uses: actions/upload-artifact@v4
         with:
           name: "lcov-${{ github.sha }}-${{ runner.os }}-test-log"  # .zip
           path: tests/test.log
           if-no-files-found: error
+
+      - name: Upload test directory shrapnel  as an artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: "lcov-${{ github.sha }}-${{ runner.os }}-shrapnel"  # .zip
+          path: tests
+          #if-no-files-found: error

bin/py2lcov

@@ -140,7 +140,7 @@ Example:
                         help="specify the test name for the TN: entry in LCOV .info file")
     parser.add_argument('-e', '--exclude', dest='excludePatterns', default='',
                         help="specify the exclude file patterns separated by ','")
-    parser.add_argument('-v', '--verbose', dest='verbose', default=False, action='store_true',
+    parser.add_argument('-v', '--verbose', dest='verbose', default=0, action='count',
                         help="print debug messages")
     parser.add_argument('--version-script', dest='version',
                         help="version extract callback")
@@ -195,7 +195,6 @@ Example:
         env["COVERAGE_FILE"] = f
         cmd = [args.cover_cmd, "xml", "-o", xml]
         try:
-            #x = subprocess.run(cmd, capture_output=True, shell=False, check=True, env=env)
             x = subprocess.run(cmd, shell=False, check=True, stdout=True, stderr=True, env=env)
         except subprocess.CalledProcessError as err:
             print("Error:  error during XML conversion of %s: %s" % (

bin/xml2lcov

@@ -80,7 +80,7 @@ Example:
                         help="specify the test name for the TN: entry in LCOV .info file")
     parser.add_argument('-e', '--exclude', dest='excludePatterns', default='',
                         help="specify the exclude file patterns separated by ','")
-    parser.add_argument('-v', '--verbose', dest='verbose', default=False, action='store_true',
+    parser.add_argument('-v', '--verbose', dest='verbose', default=0, action='count',
                         help="print debug messages")
     parser.add_argument('--version-script', dest='version',
                         help="version extract callback")

bin/xml2lcovutil.py

@@ -152,9 +152,14 @@ def process_xml_file(self, xml_file):
                 for source in root[0]:
                     # keep track of number of times we use each source_path to find
                     #  some file.  Unused source paths are likely a problem.
-                    source_paths.append([source.text, 0])
                     if self._args.verbose:
-                        print("source: " + source.text)
+                        print("source: '%s'" %(source.text))
+                    # unclear why the Coverage.py version on GitHub node
+                    # generates empty sources
+                    if source.text == None:
+                        print("skipping empty source (???)")
+                        continue
+                    source_paths.append([source.text, 0])
             else:
                 print("Error: parse xml fail: no 'sources' in %s" %(xml_file))
                 sys.exit(1)
@@ -174,17 +179,24 @@ def process_xml_file(self, xml_file):
             # name="." means current directory
             # name=".folder1.folder2" means external module or directory
             # name="abc" means internal module or directory
-            isExternal = (package.attrib['name'].startswith('.') and package.attrib['name'] != '.')
+            pname = package.attrib['name']
+            if self._args.verbose:
+                print("package: '%s'" % (pname))
+            isExternal = (pname.startswith('.') and pname != '.')
             #pdb.set_trace()
             for classes in package:
                 for fileNode in classes:
-                    if self._args.excludePatterns and any([fnmatch.fnmatchcase(fileNode.attrib['filename'], ef) for ef in self._excludePatterns]):
+                    name = fileNode.attrib['filename']
+                    if self._args.excludePatterns and any([fnmatch.fnmatchcase(name, ef) for ef in self._excludePatterns]):
                         if self._args.verbose:
-                            print("%s is excluded" % fileNode.attrib['filename'])
+                            print("%s is excluded" % name)
                         continue
-                    name = fileNode.attrib['filename']
+                    if self._args.verbose > 1:
+                        print("  file: %s" % (name))
                     if not isExternal:
                         for s in source_paths:
+                            if self._args.verbose > 1:
+                                print("  check src_path (%s %d)" % (s[0], s[1]))
                             path = os.path.join(s[0], name)
                             if os.path.exists(path):
                                 name = path

tests/bin/test_run

@@ -157,3 +157,5 @@ fi
 [ ! -z "$ELAPSED" ] && t_detail "TIME" "${ELAPSED}ms" >>"$LOGFILE"
 [ ! -z "$RESIDENT" ] && t_detail "MEM" "${RESIDENT}kB" >>"$LOGFILE"
 t_detail "RESULT" "$RESULT" >> "$LOGFILE"
+
+exit $RC

tests/gendiffcov/errs/msgtest.sh

@@ -817,6 +817,9 @@ if [ 0 != $? ] ; then
         exit 1
     fi
 fi
+if [ -d mycache ] ; then
+    find mycache -type f -exec chmod ugo+r {} \;
+fi
 
 echo "Tests passed"
 

tests/lcov/errs/errs.sh

@@ -305,6 +305,7 @@ if [ 0 != ${PIPESTATUS[0]} ] ; then
         exit $status
     fi
 fi
+chmod ugo+rx emptyDir
 
 # data consistency errors:
 #  - function marked 'hit' but no contained lines are hit

tests/py2lcov/py2lcov.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set +x
+set -x
 
 CLEAN_ONLY=0
 COVER=
@@ -154,7 +154,7 @@ if [ 0 != $? ] ; then
 fi
 
 # some corner cases:
-COVERAGE_FILE=./functions.dat $CMD  run --branch ./test.py
+COVERAGE_FILE=./functions.dat $CMD  run --branch ./test.py -v -v
 if [ 0 != $? ] ; then
     echo "coverage functions failed"
     if [ 0 == $KEEP_GOING ] ; then
@@ -182,19 +182,19 @@ done
 # look for expected location and function hit counts:
 for d in \
     'FN functions.info' \
-    'FNL:0,10,12' \
-    'FNA:0,0,unusedFunc' \
-    'FNL:1,2,7' \
-    'FNA:1,1,enter' \
-    'FNL:0,10,18' \
-    'FNA:0,0,main.localfunc' \
-    'FNL:1,12,16' \
-    'FNA:1,0,main.localfunc.nested1' \
-    'FNL:2,13,14' \
-    'FNA:2,0,main.localfunc.nested1.nested2' \
-    'FNL:3,5,18' \
+    'FNL:[0-9],10,12' \
+    'FNA:[0-9],0,unusedFunc' \
+    'FNL:[0-9],2,7' \
+    'FNA:[0-9],1,enter' \
+    'FNL:[0-9],10,18' \
+    'FNA:[0-9],0,main.localfunc' \
+    'FNL:[0-9],12,16' \
+    'FNA:[0-9],0,main.localfunc.nested1' \
+    'FNL:[0-9],13,14' \
+    'FNA:[0-9],0,main.localfunc.nested1.nested2' \
+    'FNL:[0-9],5,18' \
     ; do
-    grep $d functions.info
+    grep -E $d functions.info
     if [ 0 != $? ] ; then
         echo "did not find expected function data $d"
         if [ 0 == $KEEP_GOING ] ; then