CI installation refactoring.

Signed-off-by: fruffy <[email protected]>

.github/workflows/ci-lint.yaml

@@ -35,12 +35,16 @@ jobs:
           tools/ci-build.sh
 
       - name: Run cpplint on C/C++ files.
-        run: cmake --build build --target cpplint
+        run: poetry run cmake --build build --target cpplint
 
       - name: Run clang-format on C/C++ files.
-        run: cmake --build build --target clang-format
+        run: |
+          poetry run clang-format --version
+          poetry run cmake --build build --target clang-format
 
       - name: Run black and isort on Python files.
         run: |
-          cmake --build build --target black
-          cmake --build build --target isort
+          poetry run black --version
+          poetry run cmake --build build --target black
+          poetry run isort --version
+          poetry run cmake --build build --target isort

.github/workflows/ci-p4tools.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest -R "testgen|smith" --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest -R "testgen|smith" --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-ptf-kernels-weekly.yml

@@ -119,4 +119,4 @@ jobs:
 
       - name: Install VM and run PTF tests for eBPF backend
         run: |
-          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c ./test.sh
+          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c poetry run ./test.sh

.github/workflows/ci-ptf.yml

@@ -53,5 +53,5 @@ jobs:
           tools/ci-build.sh
 
       - name: Run PTF tests for eBPF backend (Ubuntu 20.04)
-        run: sudo -E ./test.sh
+        run: sudo -E env PATH="$PATH" poetry run ./test.sh
         working-directory: ./backends/ebpf/tests

.github/workflows/ci-static-build-test.yml

@@ -48,6 +48,6 @@ jobs:
       - shell: bash
         name: Build (Ubuntu 20.04)
         run: |
-          sudo -E tools/ci-build.sh
+          tools/ci-build.sh
           ./tools/ci-check-static.sh ./build/p4c-bm2-ss ./build/p4c-dpdk ./build/p4c-ebpf \
               ./build/p4c-pna-p4tc ./build/p4c-ubpf ./build/p4test ./build/p4testgen

.github/workflows/ci-test-debian.yml

@@ -14,6 +14,36 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+
+  # Build with GCC and test P4C on Ubuntu 24.04.
+  test-ubuntu24:
+    runs-on: ubuntu-24.04
+    env:
+      CTEST_PARALLEL_LEVEL: 4
+      IMAGE_TYPE: test
+      BUILD_GENERATOR: Ninja
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: test-${{ runner.os }}-gcc
+          max-size: 1000M
+
+      - name: Build (Ubuntu 24.04, GCC)
+        run: |
+          tools/ci-build.sh
+
+      - name: Run tests (Ubuntu 24.04)
+        # Need to use sudo for the eBPF kernel tests.
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+        working-directory: ./build
+
   # Build with GCC and test P4C on Ubuntu 22.04.
   test-ubuntu22:
     runs-on: ubuntu-22.04
@@ -39,7 +69,8 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
 
   # Build with GCC and test Tofino backend on Ubuntu 22.04.
@@ -112,6 +143,7 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
         if: matrix.unity == 'ON' && matrix.gtest == 'ON'

.github/workflows/ci-test-fedora.yml

@@ -35,7 +35,9 @@ jobs:
           submodules: recursive
 
       - name: Install dependencies (Fedora Linux)
-        run: tools/install_fedora_deps.sh
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          tools/install_fedora_deps.sh
 
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1
@@ -45,10 +47,12 @@ jobs:
 
       - name: Build p4c (Fedora Linux)
         run: |
+          export PATH="$HOME/.local/bin:$PATH"
           ./bootstrap.sh -DCMAKE_BUILD_TYPE=Release -DCMAKE_UNITY_BUILD=ON --build-generator "Ninja"
           cmake --build build -- -j $(nproc)
 
       - name: Run p4c tests (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"; ctest --output-on-failure --schedule-random
+          export PATH="$HOME/.local/bin:$PATH"
+          poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-test-mac.yml

@@ -46,6 +46,7 @@ jobs:
 
       - name: Install dependencies (MacOS)
         run: |
+          source ~/.bash_profile
           tools/install_mac_deps.sh
 
       - name: Build (MacOS)
@@ -58,7 +59,7 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build
 
   # Build and test p4c on MacOS 13 on x86.
@@ -93,6 +94,7 @@ jobs:
 
       - name: Install dependencies (MacOS)
         run: |
+          source ~/.bash_profile
           tools/install_mac_deps.sh
 
       - name: Build (MacOS)
@@ -105,5 +107,5 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly.yml

@@ -39,4 +39,4 @@ jobs:
         # this is needed to create network namespaces for the ebpf tests.
       - name: Run tests (Ubuntu 18.04)
         run: |
-          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random
+          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c  poetry run --output-on-failure --schedule-random

.github/workflows/ci-ubuntu-20-sanitizer-nightly.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-validation-nightly.yml

@@ -38,5 +38,5 @@ jobs:
 
       - name: Validate
         run: |
-          ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
+          poetry run ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
         working-directory: ./build

.gitignore

@@ -52,3 +52,6 @@ docs/doxygen/awesome_css/
 
 # thirdparty
 backends/tofino/third_party/
+
+# Poetry
+poetry.lock

Dockerfile

@@ -46,6 +46,7 @@ ARG BUILD_AUTO_VAR_INIT_PATTERN=OFF
 # - Disable leaks detector as p4c uses GC.
 ENV UBSAN_OPTIONS=print_stacktrace=1
 ENV ASAN_OPTIONS=print_stacktrace=1:detect_leaks=0
+ENV PATH="/root/.local/bin:$PATH"
 
 # Delegate the build to tools/ci-build.
 COPY . /p4c/

pyproject.toml

@@ -51,3 +51,52 @@ disable = [
   "missing-function-docstring",
   "too-few-public-methods",
 ]
+
+[tool.poetry]
+name = "p4c"
+version = "1.2.5.1"
+description = ""
+authors = ["Your Name <[email protected]>"]
+license = "Apache"
+readme = "README.md"
+package-mode = false
+
+[tool.poetry.dependencies]
+# Fixme: This restriction is because of pyinstaller
+python = "<3.14,>=3.8"
+# General testing utilties.
+pyroute2 = "0.7.3"
+ply = "3.11"
+scapy = "2.5.0"
+# Utilities for BMv2 and PTF tests.
+ptf = "0.10.0"
+googleapis-common-protos = "1.53.0"
+grpcio = [
+  {version = "1.67.0", python = ">3.6"},
+  {version = "1.48.2", python = "<=3.6"}
+]
+thrift = "0.21.0 "
+protobuf = [
+  {version = "3.20.2", python = ">3.6"},
+  {version = "3.19.2", python = "<=3.6"}
+]
+# Linters and formatters.
+clang-format = "18.1.0"
+black = [
+  {version = "24.3.0", python = ">3.6"},
+  {version = "22.8.0", python = "<=3.6"}
+]
+isort = [
+  {version = "5.13.2", python = ">3.6"},
+  {version = "5.10.1", python = "<=3.6"}
+]
+# FIXME: Remove and replace with local FetchContent version.
+p4runtime = "1.4.1"
+# FIXME: The Tofino back end should vendor these tools or use them locally.
+jsl= "0.2.4"
+pyinstaller= "6.11.0"
+jsonschema= "4.23.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tools/ci-build.sh

@@ -104,8 +104,10 @@ fi
 
 sudo apt-get update
 sudo apt-get install -y --no-install-recommends ${P4C_DEPS}
-sudo pip3 install --upgrade pip
-sudo pip3 install -r ${P4C_DIR}/requirements.txt
+# Set up poetry.
+sudo apt-get install -y python3-venv curl
+curl -sSL https://install.python-poetry.org | python3 -
+poetry install -C ${P4C_DIR}
 
 if [ "${BUILD_GENERATOR,,}" == "ninja" ] && [ ! $(command -v ninja) ]
 then
@@ -128,8 +130,6 @@ function build_bmv2() {
   # TODO: Remove this check once 18.04 is deprecated.
   if [[ "${DISTRIB_RELEASE}" == "18.04" ]] ; then
     P4C_RUNTIME_DEPS_BOOST="libboost-graph1.65.1 libboost-iostreams1.65.1"
-  else
-    P4C_RUNTIME_DEPS_BOOST="libboost-graph1.7* libboost-iostreams1.7*"
   fi
 
   P4C_RUNTIME_DEPS="cpp \
@@ -146,7 +146,7 @@ function build_bmv2() {
     P4C_RUNTIME_DEPS+=" gcc-9 g++-9"
     export CC=gcc-9
     export CXX=g++-9
-  else
+  elif [[ "${DISTRIB_RELEASE}" != "24.04" ]] ; then
    sudo apt-get install -y wget ca-certificates
     # Add the p4lang opensuse repository.
     echo "deb http://download.opensuse.org/repositories/home:/p4lang/xUbuntu_${DISTRIB_RELEASE}/ /" | sudo tee /etc/apt/sources.list.d/home:p4lang.list
@@ -155,11 +155,6 @@ function build_bmv2() {
   fi
 
   sudo apt-get update && sudo apt-get install -y --no-install-recommends ${P4C_RUNTIME_DEPS}
-
-  if [[ "${DISTRIB_RELEASE}" != "18.04" ]] ; then
-    # To run PTF nanomsg tests. Not available on 18.04.
-    sudo pip3 install nnpy
-  fi
 }
 
 if [[ "${ENABLE_BMV2}" == "ON" ]] ; then
@@ -292,12 +287,12 @@ fi
 if [ -e build ]; then /bin/rm -rf build; fi
 mkdir -p ${P4C_DIR}/build
 cd ${P4C_DIR}/build
-cmake ${CMAKE_FLAGS} -G "${BUILD_GENERATOR}" ..
+poetry run cmake ${CMAKE_FLAGS} -G "${BUILD_GENERATOR}" ..
 
 # If CMAKE_ONLY is active, only run CMake. Do not build.
 if [ "$CMAKE_ONLY" == "OFF" ]; then
-  cmake --build . -- -j $(nproc)
-  sudo cmake --install .
+  poetry run cmake --build . -- -j $(nproc)
+  sudo -E env PATH="$PATH" cmake --install .
   # Print ccache statistics after building
   ccache -p -s
 fi