Merge branch 'master' into master

songqh2022 · Sep 20, 2024 · 2f2a105 · 2f2a105
2 parents 55f4a31 + 9b9d330
commit 2f2a105
Show file tree

Hide file tree

Showing 426 changed files with 12,236 additions and 8,566 deletions.
diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml
@@ -70,6 +70,21 @@ jobs:
       path: $(Build.ArtifactStagingDirectory)/download
       patterns: '**/target/${{ parameters.artifact_name }}.gz'
     displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-dash-api
+      ${{ if eq(parameters.arch, 'amd64') }}:
+        artifact: sonic-dash-api
+      ${{ else }}:
+        artifact: sonic-dash-api.${{ parameters.arch }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libdashapi*.deb
+    displayName: "Download dash api"
   - script: |
       set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)

diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml
@@ -33,6 +33,9 @@ parameters:
 - name: artifact_name
   type: string
 
+- name: syslog_artifact_name
+  type: string
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -65,7 +68,14 @@ jobs:
         libzmq3-dev \
         swig4.0 \
         libdbus-1-dev \
-        libteam-dev
+        libteam-dev \
+        rsyslog
+
+      sudo perl -i.bk -ne 'print if not /SONiCFileFormat|ActionFileDefaultTemplate/' /etc/rsyslog.conf
+      sudo sed -ie '/GLOBAL DIRECTIVES/{s/$/\n\$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-lst-lf%\\n"\n\$ActionFileDefaultTemplate SONiCFileFormat/}' /etc/rsyslog.conf
+
+      sudo service rsyslog start
+
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -109,14 +119,33 @@ jobs:
         target/debs/${{ parameters.debian_version }}/libnl-route*.deb
         target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
         target/debs/${{ parameters.debian_version }}/libyang_*.deb
+        target/debs/${{ parameters.debian_version }}/libprotobuf*.deb
+        target/debs/${{ parameters.debian_version }}/libprotoc*.deb
+        target/debs/${{ parameters.debian_version }}/protobuf-compiler*.deb
     displayName: "Download common libs"
-
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-dash-api
+      ${{ if eq(parameters.arch, 'amd64') }}:
+        artifact: sonic-dash-api
+      ${{ else }}:
+        artifact: sonic-dash-api.${{ parameters.arch }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libdashapi*.deb
+    displayName: "Download dash api test"
   - script: |
       set -ex
       # install libyang before install libswsscommon
       sudo dpkg -i $(find ./download -name libyang_*.deb)
       sudo dpkg -i $(find ./download -name *.deb)
       rm -rf download || true
+      cat /etc/apt/sources.list
+      dpkg --list |grep libnl
     workingDirectory: $(Build.ArtifactStagingDirectory)
     displayName: "Install libnl3, sonic swss common, and sairedis"
   - script: |
@@ -129,3 +158,21 @@ jobs:
   - publish: $(Build.ArtifactStagingDirectory)
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive swss debian packages"
+  - script: |
+      pwd
+      sudo chmod a+r /var/log/syslog*
+      tar -czf syslog-swss.tgz /var/log/syslog*
+      ls -al .
+      ls -al /var/log/
+    displayName: "Env info"
+    condition: always()
+  - task: CopyFiles@2
+    displayName: "Copy syslog to staging directory"
+    condition: always()
+    inputs:
+        contents: 'syslog-swss.tgz'
+        targetFolder: $(Build.ArtifactStagingDirectory)
+  - publish: $(Build.ArtifactStagingDirectory)/
+    artifact: ${{ parameters.syslog_artifact_name }}@$(System.JobAttempt)
+    displayName: "Publish syslog artifacts"
+    condition: always()
diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
@@ -98,7 +98,10 @@ jobs:
       sudo mkdir -m 755 /var/run/sswsyncd
 
       sudo apt-get install -y rsyslog
-      sudo service rsyslog start
+      sudo rsyslogd
+
+      cat /etc/apt/sources.list
+      dpkg --list | grep libnl
 
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
@@ -164,7 +167,8 @@ jobs:
     displayName: "Compile sonic sairedis with coverage enabled"
   - script: |
       sudo cp azsyslog.conf /etc/rsyslog.conf
-      sudo service rsyslog restart
+      sudo killall rsyslogd
+      sudo rsyslogd
     displayName: "Update rsyslog.conf"
   - ${{ if eq(parameters.run_unit_test, true) }}:
     - script: |

diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile
@@ -5,7 +5,12 @@ ARG need_dbg
 
 COPY ["debs", "/debs"]
 
-RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb \
+# Remove existing packages first before installing the new/current packages. This is to overcome limitations with
+# Docker's diff detection mechanism, where only the file size and the modification timestamp (which will remain the
+# same, even though contents have changed) are checked between the previous and current layer.
+RUN dpkg --purge libswsscommon python3-swsscommon sonic-db-cli libsaimetadata libsairedis libsaivs syncd-vs swss sonic-eventd libdashapi
+RUN dpkg -i /debs/libdashapi_1.0.0_amd64.deb \
+            /debs/libswsscommon_1.0.0_amd64.deb \
             /debs/python3-swsscommon_1.0.0_amd64.deb \
             /debs/sonic-db-cli_1.0.0_amd64.deb \
             /debs/libsaimetadata_1.0.0_amd64.deb \

diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml
@@ -10,6 +10,10 @@ parameters:
   type: string
   default: docker-sonic-vs
 
+- name: sonic_buildimage_ubuntu20_04
+  type: string
+  default: '$(BUILD_BRANCH)'
+
 - name: asan
   type: boolean
   default: false
@@ -52,20 +56,32 @@ jobs:
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       allowPartiallySucceededBuilds: true
     displayName: "Download sonic swss common deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-buildimage-ubuntu20.04
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/${{ parameters.sonic_buildimage_ubuntu20_04 }}'
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download sonic buildimage ubuntu20.04 deb packages"
 
   - script: |
       set -ex
       sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh
 
       sudo apt-get update
       sudo apt-get install -y libhiredis0.14 libyang0.16
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb
       sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
       sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 
       # install packages for vs test
       sudo apt-get install -y net-tools bridge-utils vlan
       sudo apt-get install -y python3-pip
-      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0
+      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0 requests==2.31.0
     displayName: "Install dependencies"
 
   - script: |
@@ -79,21 +95,11 @@ jobs:
         params='--graceful-stop'
       fi
       all_tests=$(ls test_*.py)
-      all_tests="${all_tests} p4rt"
-      test_set=()
-      # Run 20 tests as a set.
-      for test in ${all_tests}; do
-        test_set+=("${test}")
-        if [ ${#test_set[@]} -ge 20 ]; then
-          test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
-          test_set=()
-        fi
-      done
-      if [ ${#test_set[@]} -gt 0 ]; then
-        test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-        echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
-      fi
+      all_tests="${all_tests} p4rt dash"
+      RETRY=3
+      IMAGE_NAME=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
+      echo $all_tests | xargs -n 1 | xargs -P 8 -I TEST_MODULE sudo ./run-tests.sh "$IMAGE_NAME" "$params" "TEST_MODULE" "$RETRY"
+
       rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
     ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.asan, true)) }}:

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -69,6 +69,7 @@ jobs:
             autoconf-archive \
             uuid-dev \
             libjansson-dev \
+            nlohmann-json3-dev \
             python
 
     - if: matrix.language == 'cpp'

diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,7 @@
 *.so
 *.pyc
 *_wrap.cpp
+*.bak
 
 # Packaging Files #
 ###################
@@ -58,6 +59,10 @@ stamp-h1
 python/build/
 python/setup.py
 
+lib/sai_redis.cpp
+vslib/sai_vs.cpp
+proxylib/sai_proxy.cpp
+
 # Test Files #
 ##############
 meta/test-suite.log
@@ -111,7 +116,11 @@ unittest/meta/tests
 unittest/syncd/tests
 unittest/vslib/tests
 unittest/vslib/testslibsaivs
+unittest/proxylib/tests
+unittest/proxylib/testslibsaiproxy
 vslib/tests
+stub/tests
+unittest/stub/stub/testslibsaistub
 
 # Temporary files #
 ###################

diff --git a/Makefile.am b/Makefile.am
@@ -1,4 +1,4 @@
-SUBDIRS = meta lib vslib pyext
+SUBDIRS = meta lib vslib proxylib pyext
 
 if SYNCD
 SUBDIRS += syncd saiplayer saidump saidiscovery saisdkdump saiasiccmp tests unittest

diff --git a/SAI b/SAI
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -101,6 +101,42 @@ stages:
       syslog_artifact_name: sonic-sairedis.syslog.arm64
       debian_version: ${{ parameters.debian_version }}
 
+- stage: BuildBookworm
+  dependsOn: BuildArm
+  condition: succeeded('BuildArm')
+  jobs:
+  - template: .azure-pipelines/build-template.yml
+    parameters:
+      arch: amd64
+      sonic_slave: sonic-slave-bookworm
+      swss_common_artifact_name: sonic-swss-common-bookworm
+      artifact_name: sonic-sairedis-bookworm
+      syslog_artifact_name: sonic-sairedis-bookworm.syslog
+      run_unit_test: true
+      debian_version: ${{ parameters.debian_version }}
+
+  - template: .azure-pipelines/build-template.yml
+    parameters:
+      arch: armhf
+      timeout: 180
+      pool: sonicbld-armhf
+      sonic_slave: sonic-slave-bookworm-armhf
+      swss_common_artifact_name: sonic-swss-common-bookworm.armhf
+      artifact_name: sonic-sairedis-bookworm.armhf
+      syslog_artifact_name: sonic-sairedis-bookworm.syslog.armhf
+      debian_version: ${{ parameters.debian_version }}
+
+  - template: .azure-pipelines/build-template.yml
+    parameters:
+      arch: arm64
+      timeout: 180
+      pool: sonicbld-arm64
+      sonic_slave: sonic-slave-bookworm-arm64
+      swss_common_artifact_name: sonic-swss-common-bookworm.arm64
+      artifact_name: sonic-sairedis-bookworm.arm64
+      syslog_artifact_name: sonic-sairedis-bookworm.syslog.arm64
+      debian_version: ${{ parameters.debian_version }}
+
 - stage: BuildSwss
   dependsOn: Build
   condition: succeeded('Build')
@@ -111,6 +147,7 @@ stages:
       sonic_slave: sonic-slave-${{ parameters.debian_version }}
       swss_common_artifact_name: sonic-swss-common
       sairedis_artifact_name: sonic-sairedis
+      syslog_artifact_name: sonic-swss.syslog
       artifact_name: sonic-swss
       debian_version: ${{ parameters.debian_version }}
 

diff --git a/configure.ac b/configure.ac
@@ -42,6 +42,11 @@ AC_ARG_WITH(sai,
 esac])
 AM_CONDITIONAL(SAIVS,   test x$sai = xvs)
 
+AC_ARG_WITH(dashsai,
+    [ --with-dashsai         compiling against DASH SAI],
+    [AM_CONDITIONAL(DASHSAI,   true)],
+    [AM_CONDITIONAL(DASHSAI,   false)])
+
 AC_ARG_WITH(swss-common-inc,
 [  --with-swss-common-inc=DIR
                            prefix where swss-common includes are installed],
@@ -114,6 +119,7 @@ if test "x$asan_enabled" = "xtrue"; then
     CFLAGS_ASAN+=" -fsanitize=address"
     CFLAGS_ASAN+=" -DASAN_ENABLED"
     CFLAGS_ASAN+=" -ggdb -fno-omit-frame-pointer -U_FORTIFY_SOURCE"
+    CFLAGS_ASAN+=" -Wno-maybe-uninitialized"
     AC_SUBST(CFLAGS_ASAN)
 
     LDFLAGS_ASAN+=" -lasan"
@@ -208,12 +214,17 @@ CXXFLAGS="$SAVED_FLAGS"
 
 AC_SUBST(CXXFLAGS_COMMON)
 
+AC_ARG_WITH(extra-libsai-ldflags,
+[  --with-extra-libsai-ldflags=FLAGS
+                          extra libsai.so flags for vendor library],
+[AC_SUBST(EXTRA_LIBSAI_LDFLAGS, "${withval}")])
+
 AC_SUBST(SAIINC, "-I\$(top_srcdir)/SAI/inc -I\$(top_srcdir)/SAI/experimental -I\$(top_srcdir)/SAI/meta")
 
 AM_COND_IF([SYNCD], [
 AM_COND_IF([SAIVS], [], [
 SAVED_FLAGS="$CXXFLAGS"
-CXXFLAGS="-Xlinker --no-as-needed -lsai -I$srcdir/SAI/inc -I$srcdir/SAI/experimental -I$srcdir/SAI/meta"
+CXXFLAGS="-Xlinker --no-as-needed -lsai -I$srcdir/SAI/inc -I$srcdir/SAI/experimental -I$srcdir/SAI/meta $EXTRA_LIBSAI_LDFLAGS"
 AC_CHECK_FUNCS(sai_query_api_version, [
 AC_MSG_CHECKING([SAI headers API version and library version check])
 AC_TRY_RUN([
@@ -248,7 +259,8 @@ https://github.com/opencomputeproject/SAI/pull/1795 (structs)
 }],
 [AC_MSG_RESULT(ok)],
 [AC_MSG_RESULT(failed)
-AC_MSG_ERROR("SAI headers API version and library version mismatch")])])
+AC_MSG_ERROR("SAI headers API version and library version mismatch")])],
+[AC_MSG_ERROR("SAI library libsai.so does not have sai_query_api_version API which is required")])
 CXXFLAGS="$SAVED_FLAGS"
 ])])
 
@@ -259,6 +271,14 @@ AC_CHECK_FUNCS(sai_bulk_object_clear_stats sai_bulk_object_get_stats)
 CXXFLAGS="$SAVED_FLAGS"
 ])
 
+AC_DEFINE([SAIREDIS_GIT_REVISION],
+        "[m4_esyscmd([echo -n $(git rev-parse --short HEAD || echo 0000000)])]",
+        [sairedis git revision information])
+
+AC_DEFINE([SAI_GIT_REVISION],
+        "[m4_esyscmd([echo -n $(cd SAI && git rev-parse --short HEAD || echo 0000000)])]",
+        [SAI git revision information])
+
 AC_OUTPUT(Makefile
           meta/Makefile
           lib/Makefile
@@ -271,11 +291,13 @@ AC_OUTPUT(Makefile
           saidiscovery/Makefile
           saiasiccmp/Makefile
           tests/Makefile
+          proxylib/Makefile
           unittest/Makefile
           unittest/meta/Makefile
           unittest/lib/Makefile
           unittest/vslib/Makefile
           unittest/syncd/Makefile
+          unittest/proxylib/Makefile
           pyext/Makefile
           pyext/py2/Makefile
           pyext/py3/Makefile)
diff --git a/debian/control b/debian/control
@@ -7,7 +7,7 @@ Standards-Version: 1.0.0
 
 Package: syncd
 Architecture: any
-Build-Profiles: <syncd !rpc !vs>
+Build-Profiles: <syncd !vs>
 Depends: ${misc:Pre-Depends}
 Recommends: ${shlibs:Depends}
 Conflicts: syncd-rpc, syncd-vs