Add release testing

Add github action to be run when a release branch is created.
Release testing includes:
 - Regular PR testing
 - Fuzz testing for 1 hour
 - Checking that the version number has been updated correctly
 - Uploading the sdist and bdist builds as artifacts

Signed-off-by: Øyvind Rønningstad <[email protected]>

.github/workflows/run-tests.yaml

@@ -3,7 +3,7 @@ name: Run tests
 on: [pull_request]
 
 jobs:
-  test:
+  merge-test:
     runs-on: ubuntu-latest
     name: Run tests
 
@@ -17,18 +17,18 @@ jobs:
         pip3 install -U setuptools
         pip3 install -U -r scripts/requirements.txt
 
-    - name: Install cddl-gen package
+    - name: Generate and install cddl-gen package
       run: |
         python3 setup.py sdist bdist_wheel
         pip3 install dist/cddl-gen-*.tar.gz
         pip3 uninstall -y cddl-gen
         pip3 install dist/cddl_gen-*.whl
 
     - name: Run pycodestyle
-      working-directory:
       run: |
         pycodestyle cddl_gen/cddl_gen.py --max-line-length=100 --ignore=W191,E101,W503
         pycodestyle tests/scripts/run_tests.py --max-line-length=100 --ignore=W503,E501,E402
+        pycodestyle tests/scripts/release_test.py --max-line-length=100
         pycodestyle cddl_gen/__init__.py --max-line-length=100
         pycodestyle setup.py --max-line-length=100
 
@@ -75,3 +75,54 @@ jobs:
         export ZEPHYR_BASE=$(pwd)/zephyr
         export ZEPHYR_TOOLCHAIN_VARIANT=host
         $ZEPHYR_BASE/scripts/twister -i -T . -W --platform native_posix --platform native_posix_64 -t cbor_encode
+
+  release-test:
+    runs-on: ubuntu-latest
+    name: Run tests
+    needs: merge-test
+    if: "startswith(github.head_ref, 'release/')"
+
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: |
+        pip3 install -U pip
+        pip3 install -U setuptools
+        pip3 install -U -r scripts/requirements-base.txt
+
+    - name: Generate and install cddl-gen package
+      run: |
+        python3 setup.py sdist bdist_wheel
+        pip3 install dist/cddl-gen-*.tar.gz
+        pip3 uninstall -y cddl-gen
+        pip3 install dist/cddl_gen-*.whl
+
+    - name: Run python release tests
+      working-directory: tests/scripts
+      run: |
+        echo -n ${{ github.head_ref }} > HEAD_REF
+        python3 -m unittest release_test
+        rm HEAD_REF
+
+    - name: Install packages
+      run: |
+        sudo apt update
+        sudo apt install -y gcc-multilib afl++
+
+    - name: Run manifest12 fuzz tests
+      working-directory: tests/fuzz
+      run: |
+        ./test-afl.sh 3200 32 manifest12
+
+    - name: Run pet fuzz tests
+      working-directory: tests/fuzz
+      run: |
+        ./test-afl.sh 400 32 pet
+
+    - name: Upload release files
+      uses: actions/upload-artifact@v2
+      with:
+        name: cddl-gen-release
+        path: dist/*

tests/scripts/release_test.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2021 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+from sys import version
+from unittest import TestCase, main
+from pathlib import Path
+from subprocess import Popen, PIPE
+
+p_script_dir = Path(__file__).absolute().parents[0]
+p_root = p_script_dir.parents[1]
+p_VERSION = Path(p_root, "cddl_gen", "VERSION")
+p_RELEASE_NOTES = Path(p_root, "RELEASE_NOTES.md")
+p_HEAD_REF = Path(p_script_dir, "HEAD_REF")
+
+
+class ReleaseTest(TestCase):
+    def test_version_num(self):
+        current_branch = Popen(['git', 'branch', '--show-current'],
+                               stdout=PIPE).communicate()[0].decode("utf-8").strip()
+        if not current_branch:
+            current_branch = p_HEAD_REF.read_text()
+        self.assertRegex(
+            current_branch, r"release/\d+\.\d+\.\d+",
+            "Release tests must be run on a release branch on the form 'release/x.y.z'.")
+
+        version_number = current_branch.replace("release/", "")
+        self.assertRegex(
+            version_number, r'\d+\.\d+\.(?!99)\d+',
+            "Releases cannot have the x.y.99 development bugfix release number.")
+        self.assertEqual(
+            version_number, p_VERSION.read_text(),
+            f"{p_VERSION} has not been updated to the correct version number.")
+        self.assertEqual(
+            p_RELEASE_NOTES.read_text().splitlines()[0], r"# cddl-gen v. " + version_number,
+            f"{p_RELEASE_NOTES} has not been updated with the correct version number.")
+
+        tags_stdout, _ = Popen(['git', 'tag'], stdout=PIPE).communicate()
+        tags = tags_stdout.decode("utf-8").strip().splitlines()
+        self.assertNotIn(
+            version_number, tags,
+            "Version number already exists as a tag. Has the version number been updated?")
+
+
+if __name__ == "__main__":
+    main()

tests/test.sh

@@ -13,6 +13,8 @@ pycodestyle ../setup.py --max-line-length=100
 [[ $? -ne 0 ]] && exit 1
 pycodestyle scripts/run_tests.py --max-line-length=100 --ignore=W503,E501,E402
 [[ $? -ne 0 ]] && exit 1
+pycodestyle scripts/release_test.py --max-line-length=100
+[[ $? -ne 0 ]] && exit 1
 
 pushd "scripts"
 python3 -m unittest run_tests