Merge pull request #209 from truenas/truenas/zfs-2.2.3-staging

[truenas/zfs-2.2-release]: sync with upstream zfs-2.2.3-staging

.github/codeql-cpp.yml

@@ -0,0 +1,4 @@
+name: "Custom CodeQL Analysis"
+
+queries:
+  - uses: ./.github/codeql/custom-queries/cpp/deprecatedFunctionUsage.ql

.github/codeql-python.yml

@@ -0,0 +1,4 @@
+name: "Custom CodeQL Analysis"
+
+paths-ignore:
+  - tests

.github/codeql/custom-queries/cpp/deprecatedFunctionUsage.ql

@@ -0,0 +1,59 @@
+/**
+ * @name Deprecated function usage detection
+ * @description Detects functions whose usage is banned from the OpenZFS
+ *              codebase due to QA concerns.
+ * @kind problem
+ * @severity error
+ * @id cpp/deprecated-function-usage
+*/
+
+import cpp
+
+predicate isDeprecatedFunction(Function f) {
+  f.getName() = "strtok" or
+  f.getName() = "__xpg_basename" or
+  f.getName() = "basename" or
+  f.getName() = "dirname" or
+  f.getName() = "bcopy" or
+  f.getName() = "bcmp" or
+  f.getName() = "bzero" or
+  f.getName() = "asctime" or
+  f.getName() = "asctime_r" or
+  f.getName() = "gmtime" or
+  f.getName() = "localtime" or
+  f.getName() = "strncpy"
+
+}
+
+string getReplacementMessage(Function f) {
+  if f.getName() = "strtok" then
+    result = "Use strtok_r(3) instead!"
+  else if f.getName() = "__xpg_basename" then
+    result = "basename(3) is underspecified. Use zfs_basename() instead!"
+  else if f.getName() = "basename" then
+    result = "basename(3) is underspecified. Use zfs_basename() instead!"
+  else if f.getName() = "dirname" then
+    result = "dirname(3) is underspecified. Use zfs_dirnamelen() instead!"
+  else if f.getName() = "bcopy" then
+    result = "bcopy(3) is deprecated. Use memcpy(3)/memmove(3) instead!"
+  else if f.getName() = "bcmp" then
+    result = "bcmp(3) is deprecated. Use memcmp(3) instead!"
+  else if f.getName() = "bzero" then
+    result = "bzero(3) is deprecated. Use memset(3) instead!"
+  else if f.getName() = "asctime" then
+    result = "Use strftime(3) instead!"
+  else if f.getName() = "asctime_r" then
+    result = "Use strftime(3) instead!"
+  else if f.getName() = "gmtime" then
+    result = "gmtime(3) isn't thread-safe. Use gmtime_r(3) instead!"
+  else if f.getName() = "localtime" then
+    result = "localtime(3) isn't thread-safe. Use localtime_r(3) instead!"
+  else
+    result = "strncpy(3) is deprecated. Use strlcpy(3) instead!"
+}
+
+from FunctionCall fc, Function f
+where
+  fc.getTarget() = f and
+  isDeprecatedFunction(f)
+select fc, getReplacementMessage(f)

.github/codeql/custom-queries/cpp/qlpack.yml

@@ -0,0 +1,4 @@
+name: openzfs-cpp-queries
+version: 0.0.0
+libraryPathDependencies: codeql-cpp
+suites: openzfs-cpp-suite

.github/workflows/README.md

@@ -4,44 +4,54 @@
 ```mermaid
 flowchart TB
 subgraph CleanUp and Summary
-  Part1-20.04-->CleanUp+nice+Summary
-  Part2-20.04-->CleanUp+nice+Summary
-  PartN-20.04-->CleanUp+nice+Summary
-  Part1-22.04-->CleanUp+nice+Summary
-  Part2-22.04-->CleanUp+nice+Summary
-  PartN-22.04-->CleanUp+nice+Summary
+  CleanUp+Summary
 end
 
 subgraph Functional Testings
+  sanity-checks-20.04
+  zloop-checks-20.04
   functional-testing-20.04-->Part1-20.04
   functional-testing-20.04-->Part2-20.04
-  functional-testing-20.04-->PartN-20.04
+  functional-testing-20.04-->Part3-20.04
+  functional-testing-20.04-->Part4-20.04
   functional-testing-22.04-->Part1-22.04
   functional-testing-22.04-->Part2-22.04
-  functional-testing-22.04-->PartN-22.04
-end
-
-subgraph Sanity and zloop Testings
-  sanity-checks-20.04-->functional-testing-20.04
-  sanity-checks-22.04-->functional-testing-22.04
-  zloop-checks-20.04-->functional
-  zloop-checks-22.04-->functional
+  functional-testing-22.04-->Part3-22.04
+  functional-testing-22.04-->Part4-22.04
+  sanity-checks-22.04
+  zloop-checks-22.04
 end
 
 subgraph Code Checking + Building
+  Build-Ubuntu-20.04
   codeql.yml
   checkstyle.yml
+  Build-Ubuntu-22.04
+end
+
   Build-Ubuntu-20.04-->sanity-checks-20.04
-  Build-Ubuntu-22.04-->sanity-checks-22.04
   Build-Ubuntu-20.04-->zloop-checks-20.04
+  Build-Ubuntu-20.04-->functional-testing-20.04
+  Build-Ubuntu-22.04-->sanity-checks-22.04
   Build-Ubuntu-22.04-->zloop-checks-22.04
-end
+  Build-Ubuntu-22.04-->functional-testing-22.04
+
+  sanity-checks-20.04-->CleanUp+Summary
+  Part1-20.04-->CleanUp+Summary
+  Part2-20.04-->CleanUp+Summary
+  Part3-20.04-->CleanUp+Summary
+  Part4-20.04-->CleanUp+Summary
+  Part1-22.04-->CleanUp+Summary
+  Part2-22.04-->CleanUp+Summary
+  Part3-22.04-->CleanUp+Summary
+  Part4-22.04-->CleanUp+Summary
+  sanity-checks-22.04-->CleanUp+Summary
 ```
 
 
 1) build zfs modules for Ubuntu 20.04 and 22.04 (~15m)
 2) 2x zloop test (~10m) + 2x sanity test (~25m)
-3) functional testings in parts 1..5 (each ~1h)
+3) 4x functional testings in parts 1..4 (each ~1h)
 4) cleanup and create summary
    - content of summary depends on the results of the steps
 

.github/workflows/checkstyle.yaml

@@ -8,7 +8,7 @@ jobs:
   checkstyle:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
@@ -52,7 +52,7 @@ jobs:
       if: failure() && steps.CheckABI.outcome == 'failure'
       run: |
         find -name *.abi | tar -cf abi_files.tar -T -
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: failure() && steps.CheckABI.outcome == 'failure'
       with:
         name: New ABI files (use only if you're sure about interface changes)

.github/workflows/codeql.yml

@@ -24,11 +24,12 @@ jobs:
         echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV
 
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:
+        config-file: .github/codeql-${{ matrix.language }}.yml
         languages: ${{ matrix.language }}
 
     - name: Autobuild

.github/workflows/scripts/generate-summary.sh

@@ -87,7 +87,7 @@ function summarize_f() {
   output "\n## $headline\n"
   rm -rf testfiles
   for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2/part$i.tar"
+    tarfile="$2-part$i/part$i.tar"
     check_tarfile "$tarfile"
     check_logfile "testfiles/log"
   done

.github/workflows/scripts/setup-dependencies.sh

@@ -55,29 +55,24 @@ function mod_install() {
   cat /proc/spl/kstat/zfs/chksum_bench
   echo "::endgroup::"
 
-  echo "::group::Reclaim and report disk space"
-  # remove 4GiB of images
-  sudo systemd-run docker system prune --force --all --volumes
+  echo "::group::Optimize storage for ZFS testings"
+  # remove swap and umount fast storage
+  # 89GiB -> rootfs + bootfs with ~80MB/s -> don't care
+  # 64GiB -> /mnt with 420MB/s -> new testing ssd
+  sudo swapoff -a
 
-  # remove unused software
-  sudo systemd-run --wait rm -rf \
-    "$AGENT_TOOLSDIRECTORY" \
-    /opt/* \
-    /usr/local/* \
-    /usr/share/az* \
-    /usr/share/dotnet \
-    /usr/share/gradle* \
-    /usr/share/miniconda \
-    /usr/share/swift \
-    /var/lib/gems \
-    /var/lib/mysql \
-    /var/lib/snapd
-
-  # trim the cleaned space
-  sudo fstrim /
+  # this one is fast and mounted @ /mnt
+  # -> we reformat with ext4 + move it to /var/tmp
+  DEV="/dev/disk/azure/resource-part1"
+  sudo umount /mnt
+  sudo mkfs.ext4 -O ^has_journal -F $DEV
+  sudo mount -o noatime,barrier=0 $DEV /var/tmp
+  sudo chmod 1777 /var/tmp
 
   # disk usage afterwards
-  df -h /
+  sudo df -h /
+  sudo df -h /var/tmp
+  sudo fstrim -a
   echo "::endgroup::"
 }
 

.github/workflows/zfs-linux-tests.yml

@@ -13,10 +13,10 @@ jobs:
   zloop:
     runs-on: ubuntu-${{ inputs.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: modules-${{ inputs.os }}
     - name: Install modules
@@ -34,7 +34,7 @@ jobs:
       if: failure()
       run: |
         sudo chmod +r -R /var/tmp/zloop/
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: Zpool-logs-${{ inputs.os }}
@@ -43,7 +43,7 @@ jobs:
           !/var/tmp/zloop/*/vdev/
         retention-days: 14
         if-no-files-found: ignore
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: Zpool-files-${{ inputs.os }}
@@ -55,10 +55,10 @@ jobs:
   sanity:
     runs-on: ubuntu-${{ inputs.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: modules-${{ inputs.os }}
     - name: Install modules
@@ -77,7 +77,7 @@ jobs:
         RESPATH="/var/tmp/test_results"
         mv -f $RESPATH/current $RESPATH/testfiles
         tar cf $RESPATH/sanity.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: success() || failure()
       with:
         name: Logs-${{ inputs.os }}-sanity
@@ -91,10 +91,10 @@ jobs:
       matrix:
         tests: [ part1, part2, part3, part4 ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: modules-${{ inputs.os }}
     - name: Install modules
@@ -116,9 +116,9 @@ jobs:
         RESPATH="/var/tmp/test_results"
         mv -f $RESPATH/current $RESPATH/testfiles
         tar cf $RESPATH/${{ matrix.tests }}.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: success() || failure()
       with:
-        name: Logs-${{ inputs.os }}-functional
+        name: Logs-${{ inputs.os }}-functional-${{ matrix.tests }}
         path: /var/tmp/test_results/${{ matrix.tests }}.tar
         if-no-files-found: ignore

.github/workflows/zfs-linux.yml

@@ -14,14 +14,14 @@ jobs:
         os: [22.04]
     runs-on: ubuntu-${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Build modules
       run: .github/workflows/scripts/setup-dependencies.sh build
     - name: Prepare modules upload
       run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: modules-${{ matrix.os }}
         path: modules-${{ matrix.os }}.tgz
@@ -44,7 +44,7 @@ jobs:
     runs-on: ubuntu-22.04
     needs: testings
     steps:
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
     - name: Generating summary
       run: |
         tar xzf modules-22.04/modules-22.04.tgz .github tests
@@ -58,7 +58,7 @@ jobs:
       run: .github/workflows/scripts/generate-summary.sh 3
     - name: Summary for errors #4
       run: .github/workflows/scripts/generate-summary.sh 4
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: Summary Files
         path: Summary/

META

@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.2.2
+Version:       2.2.3
 Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 6.6
+Linux-Maximum: 6.7
 Linux-Minimum: 3.10

cmd/zdb/zdb.c

@@ -2360,7 +2360,7 @@ static void
 snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
     const blkptr_t *bp)
 {
-	abd_t *pabd;
+	static abd_t *pabd = NULL;
 	void *buf;
 	zio_t *zio;
 	zfs_zstdhdr_t zstd_hdr;
@@ -2391,7 +2391,8 @@ snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
 		return;
 	}
 
-	pabd = abd_alloc_for_io(SPA_MAXBLOCKSIZE, B_FALSE);
+	if (!pabd)
+		pabd = abd_alloc_for_io(SPA_MAXBLOCKSIZE, B_FALSE);
 	zio = zio_root(spa, NULL, NULL, 0);
 
 	/* Decrypt but don't decompress so we can read the compression header */
@@ -8490,6 +8491,14 @@ zdb_decompress_block(abd_t *pabd, void *buf, void *lbuf, uint64_t lsize,
 	*cfuncp++ = ZIO_COMPRESS_LZ4;
 	*cfuncp++ = ZIO_COMPRESS_LZJB;
 	mask |= ZIO_COMPRESS_MASK(LZ4) | ZIO_COMPRESS_MASK(LZJB);
+	/*
+	 * Every gzip level has the same decompressor, no need to
+	 * run it 9 times per bruteforce attempt.
+	 */
+	mask |= ZIO_COMPRESS_MASK(GZIP_2) | ZIO_COMPRESS_MASK(GZIP_3);
+	mask |= ZIO_COMPRESS_MASK(GZIP_4) | ZIO_COMPRESS_MASK(GZIP_5);
+	mask |= ZIO_COMPRESS_MASK(GZIP_6) | ZIO_COMPRESS_MASK(GZIP_7);
+	mask |= ZIO_COMPRESS_MASK(GZIP_8) | ZIO_COMPRESS_MASK(GZIP_9);
 	for (int c = 0; c < ZIO_COMPRESS_FUNCTIONS; c++)
 		if (((1ULL << c) & mask) == 0)
 			*cfuncp++ = c;