Merge branch 'master' into master

.azure-pipelines/build-docker-sonic-vs-template.yml

@@ -47,6 +47,7 @@ jobs:
       path: $(Build.ArtifactStagingDirectory)/download
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      allowPartiallySucceededBuilds: true
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:

.azure-pipelines/build-swss-template.yml

@@ -60,15 +60,12 @@ jobs:
   - script: |
       set -ex
       sudo apt-get update
-      sudo apt-get install -y libhiredis0.14 libhiredis-dev
-      sudo apt-get install -y libzmq5 libzmq3-dev
-      sudo apt-get install -qq -y \
-          libhiredis-dev \
-          swig3.0
-      sudo apt-get install -y libdbus-1-3
-      sudo apt-get install -y libteam-dev \
-          libteam5 \
-          libteamdctl0
+      sudo apt-get install -y \
+        libhiredis-dev \
+        libzmq3-dev \
+        swig4.0 \
+        libdbus-1-dev \
+        libteam-dev
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -78,6 +75,7 @@ jobs:
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      allowPartiallySucceededBuilds: true
       path: $(Build.ArtifactStagingDirectory)/download
       patterns: |
         libswsscommon_1.0.0_*.deb

.azure-pipelines/build-template.yml

@@ -74,21 +74,18 @@ jobs:
       sudo apt-get update
       sudo apt-get install -qq -y \
         qtbase5-dev \
-        libdbus-glib-1-2 \
         libdbus-glib-1-dev \
         libpcsclite-dev \
         docbook-to-man \
         docbook-utils \
         aspell-en \
-        libhiredis0.14 \
         libhiredis-dev \
         libnl-3-dev \
         libnl-genl-3-dev \
         libnl-route-3-dev \
         libnl-nf-3-dev \
         libnl-cli-3-dev \
-        swig3.0 \
-        libpython2.7-dev \
+        swig4.0 \
         libzmq5 \
         libzmq3-dev \
         autoconf-archive
@@ -98,6 +95,7 @@ jobs:
       sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
       sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
       sudo service redis-server start
+      sudo mkdir -m 755 /var/run/sswsyncd
 
       sudo apt-get install -y rsyslog
       sudo service rsyslog start
@@ -143,6 +141,7 @@ jobs:
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      allowPartiallySucceededBuilds: true
       path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
   - script: |
@@ -160,7 +159,7 @@ jobs:
       if [ '${{ parameters.asan }}' == True ]; then
          extraflags='--enable-asan'
       fi
-      DEB_BUILD_OPTIONS=nocheck fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS=$extraflags CFLAGS="" CXXFLAGS="" binary-syncd-vs
+      DEB_BUILD_OPTIONS=nocheck DEB_CONFIGURE_EXTRA_FLAGS=$extraflags dpkg-buildpackage -us -uc -b -Psyncd,vs,nopython2 -j$(nproc)
       mv ../*.deb .
     displayName: "Compile sonic sairedis with coverage enabled"
   - script: |
@@ -182,10 +181,19 @@ jobs:
         set -ex
         # Add SYS_TIME capability for settimeofday ok in syncd test
         sudo setcap "cap_sys_time=eip" syncd/.libs/syncd_tests
+        # Add CAP_DAC_OVERRIDE capability for system directory creation in syncd unittest
+        sudo setcap "cap_dac_override,cap_ipc_lock,cap_ipc_owner,cap_sys_time=eip" unittest/syncd/.libs/tests
         make check
         gcovr --version
         find SAI/meta -name "*.gc*" | xargs rm -vf
-        gcovr -r ./ -e ".*/SAI/.*" -e ".+/json.hpp" -e "swss/.+" -e ".*/.libs/.*" -e ".*/debian/.*" --exclude-unreachable-branches -x --xml-pretty -o coverage.xml
+        gcov_dirs=$(find . -path "*.libs*gcda" | xargs dirname | sort -u |  cut -c"3-")
+        for dir in ${gcov_dirs}; do
+          source_dir=$(dirname $dir)
+          output_file="coverage-$source_dir.json"
+          gcovr --exclude-unreachable-branches --json-pretty -o $output_file --object-directory $source_dir $dir
+        done
+        gcovr -r ./ -e ".*/SAI/.*" -e ".+/json.hpp" -e "swss/.+" -e ".*/.libs/.*" -e ".*/debian/.*" --exclude-unreachable-branches --json-pretty -o coverage-all.json
+        gcovr -a "coverage-*.json" -x --xml-pretty -o coverage.xml
       displayName: "Run sonic sairedis unit tests"
   - publish: $(System.DefaultWorkingDirectory)/
     artifact: ${{ parameters.artifact_name }}
@@ -195,9 +203,9 @@ jobs:
           set -ex
           # Install .NET CORE
           curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-          sudo apt-add-repository https://packages.microsoft.com/debian/10/prod
+          sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
           sudo apt-get update
-          sudo apt-get install -y dotnet-sdk-5.0
+          sudo apt-get install -y dotnet-sdk-7.0
         displayName: "Install .NET CORE"
       - task: PublishCodeCoverageResults@1
         inputs:
@@ -219,6 +227,6 @@ jobs:
         contents: 'syslog-all.tgz'
         targetFolder: $(Build.ArtifactStagingDirectory)
   - publish: $(Build.ArtifactStagingDirectory)/
-    artifact: ${{ parameters.syslog_artifact_name }}
+    artifact: ${{ parameters.syslog_artifact_name }}@$(System.JobAttempt)
     displayName: "Publish syslog artifacts"
     condition: always()

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -3,19 +3,15 @@ FROM docker-sonic-vs
 ARG docker_container_name
 ARG need_dbg
 
-ADD ["debs", "/debs"]
+COPY ["debs", "/debs"]
 
-RUN dpkg --purge python-swsscommon python3-swsscommon swss libsairedis sonic-db-cli libswsscommon libsaimetadata libsaivs syncd-vs
+RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb \
+            /debs/python3-swsscommon_1.0.0_amd64.deb \
+            /debs/sonic-db-cli_1.0.0_amd64.deb \
+            /debs/libsaimetadata_1.0.0_amd64.deb \
+            /debs/libsairedis_1.0.0_amd64.deb \
+            /debs/libsaivs_1.0.0_amd64.deb \
+            /debs/syncd-vs_1.0.0_amd64.deb \
+            /debs/swss_1.0.0_amd64.deb
 
-RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb
-RUN dpkg -i /debs/python-swsscommon_1.0.0_amd64.deb
-RUN dpkg -i /debs/python3-swsscommon_1.0.0_amd64.deb
-RUN dpkg -i /debs/sonic-db-cli_1.0.0_amd64.deb
-
-RUN dpkg -i /debs/libsaimetadata_1.0.0_amd64.deb
-RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb
-RUN dpkg -i /debs/libsaivs_1.0.0_amd64.deb
-RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb
-RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/syncd-vs-dbg_1.0.0_amd64.deb ; fi
-
-RUN dpkg -i /debs/swss_1.0.0_amd64.deb
+RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/syncd-vs-dbgsym_1.0.0_amd64.deb ; fi

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -50,6 +50,7 @@ jobs:
       path: $(Build.ArtifactStagingDirectory)/download
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      allowPartiallySucceededBuilds: true
     displayName: "Download sonic swss common deb packages"
 
   - script: |
@@ -64,7 +65,7 @@ jobs:
       # install packages for vs test
       sudo apt-get install -y net-tools bridge-utils vlan
       sudo apt-get install -y python3-pip
-      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker==4.4.1 redis==3.3.4 flaky==3.7.0
+      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0
     displayName: "Install dependencies"
 
   - script: |
@@ -95,7 +96,8 @@ jobs:
       fi
       rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
-    continueOnError: ${{ parameters.asan }}
+    ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.asan, true)) }}:
+      continueOnError: true
 
   - task: PublishTestResults@2
     inputs:

.github/workflows/semgrep.yml

@@ -0,0 +1,22 @@
+name: Semgrep
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - master
+      - '201[7-9][0-1][0-9]'
+      - '202[0-9][0-1][0-9]'
+
+jobs:
+  semgrep:
+    if: github.repository_owner == 'sonic-net'
+    name: Semgrep
+    runs-on: ubuntu-latest
+    container:
+      image: returntocorp/semgrep
+    steps:
+      - uses: actions/checkout@v3
+      - run: semgrep ci
+        env:
+           SEMGREP_RULES: p/default

.gitignore

@@ -101,6 +101,8 @@ syncd/syncd_tests
 syncd/tests/tests
 tests/syncd
 tests/testclient
+tests/testclient_dash
+tests/testdash_gtest
 tests/tests
 tests/vssyncd
 unittest/lib/tests

azure-pipelines.yml

@@ -37,7 +37,7 @@ resources:
 parameters:
   - name: debian_version
     type: string
-    default: buster
+    default: bullseye
 variables:
   - name: BUILD_BRANCH
     ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:

configure.ac

@@ -10,8 +10,8 @@ AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_LIBTOOL
 AC_HEADER_STDC
-AM_PATH_PYTHON
 AM_PATH_PYTHON3
+AX_PYTHON3_DEVEL
 AX_CODE_COVERAGE
 AX_ADD_AM_MACRO_STATIC([])
 
@@ -102,6 +102,14 @@ AC_ARG_ENABLE(asan,
 	*) AC_MSG_ERROR(bad value ${enableval} for --enable-asan) ;;
 esac],[asan_enabled=false])
 
+AC_ARG_ENABLE(python2,
+[  --enable-python2     Generate Python 2 bindings],
+[case "${enableval}" in
+	yes) python2=true ;;
+	no)  python2=false ;;
+	*) AC_MSG_ERROR(bad value ${enableval} for --enable-python2) ;;
+esac],[python2=true])
+
 if test "x$asan_enabled" = "xtrue"; then
     CFLAGS_ASAN+=" -fsanitize=address"
     CFLAGS_ASAN+=" -DASAN_ENABLED"
@@ -112,13 +120,31 @@ if test "x$asan_enabled" = "xtrue"; then
     AC_SUBST(LDFLAGS_ASAN)
 fi
 
+if test "x$python2" = "xtrue"; then
+	m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+					  [python2 python2.7 python2.6 python2.5 python2.4 python2.3 python2.2 python2.1 python2.0])
+	AM_PATH_PYTHON
+	AX_PYTHON_DEVEL([< '3'])
+	PYTHON2_CPPFLAGS=$PYTHON_CPPFLAGS
+	PYTHON2_LIBS=$PYTHON_LIBS
+	AC_SUBST(PYTHON2_CPPFLAGS)
+	AC_SUBST(PYTHON2_LIBS)
+else
+	# The py-compile script either uses $PYTHON (which won't be set if we're not building python 2 bindings),
+	# or assumes a default of `python` (which may or may not exist as a symlink). Therefore, just assign
+	# PYTHON3 to PYTHON to make sure it always points to a valid binary.
+	PYTHON=$PYTHON3
+fi
+
 AM_CONDITIONAL(ASAN_ENABLED, test x$asan_enabled = xtrue)
+AM_CONDITIONAL(PYTHON2, test x$python2 = xtrue)
 
 AC_PATH_PROGS(SWIG, [swig3.0 swig])
 
 CXXFLAGS_COMMON=""
 CXXFLAGS_COMMON+=" -ansi"
 CXXFLAGS_COMMON+=" -fPIC"
+CXXFLAGS_COMMON+=" -pipe"
 CXXFLAGS_COMMON+=" -std=c++14"
 CXXFLAGS_COMMON+=" -Wall"
 CXXFLAGS_COMMON+=" -Wcast-align"
@@ -193,14 +219,32 @@ AC_MSG_CHECKING([SAI headers API version and library version check])
 AC_TRY_RUN([
 extern "C" {
 #include <sai.h>
+#include <cstdio>
 }
 int main() {
     sai_api_version_t version;
     if (SAI_STATUS_SUCCESS != sai_query_api_version(&version))
     {
         return 1;
     }
-    return (version != SAI_API_VERSION);
+/*
+since commit 8e1fb37 (v1.8.0) there is check enum binary backward compatibility
+so we are relaxing this check below, versions do not have to be equal but both
+need to be at least 2ebde24 (v1.9.0), this will make sure that enums are always
+ok, but since that commit some data structures changed and are not binary
+backwad compatible like next hop group api, acl_capability, structs are
+backward binary compatible from commit aed34c8, which closest tag version for
+it is commit 3ff228a (v1.12.0), so min version check should be set to
+(v1.12.0), but some production branches do not use that high SAI version yet,
+some use even version v1.7.0 which then it is impossible to provide even enum
+backward compatibility, check those links:
+https://github.com/opencomputeproject/SAI/pull/1297 (enums)
+https://github.com/opencomputeproject/SAI/pull/1795 (structs)
+*/
+    sai_api_version_t minversion = SAI_VERSION(1,9,0);
+    printf("min SAI version: %ld, sairedis SAI headers version: %ld, vendor lib SAI query api version: %ld\n",
+           minversion, SAI_API_VERSION, version);
+    return (version < minversion) || (SAI_API_VERSION < minversion);
 }],
 [AC_MSG_RESULT(ok)],
 [AC_MSG_RESULT(failed)

debian/compat

@@ -1 +1 @@
-10
+12