Merge branch 'main' into pulsar-e2e-tests

numaproj · Jan 13, 2025 · 15184ff · 15184ff
2 parents cd54c7b + ade70d8
commit 15184ff
Show file tree

Hide file tree

Showing 133 changed files with 9,224 additions and 3,030 deletions.
diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml
@@ -35,7 +35,7 @@ jobs:
       - name: Make checksums
         run: make checksums
       - name: store artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: binaries
           path: dist
@@ -65,7 +65,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: rust/numaflow-rs-linux-amd64
@@ -98,7 +98,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: rust/numaflow-rs-linux-arm64
@@ -127,19 +127,19 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Download Go binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries
           path: dist/
 
       - name: Download Rust amd64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: dist/
 
       - name: Download Rust arm64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: dist/

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Make checksums
         run: make checksums
       - name: store artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: binaries
           path: dist
@@ -56,7 +56,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/x86_64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-amd64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: rust/numaflow-rs-linux-amd64
@@ -82,7 +82,7 @@ jobs:
       - name: Rename binary
         run: cp -pv target/aarch64-unknown-linux-gnu/release/numaflow numaflow-rs-linux-arm64
       - name: Upload numaflow binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: rust/numaflow-rs-linux-arm64
@@ -111,19 +111,19 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Download Go binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries
           path: dist/
 
       - name: Download Rust amd64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-amd64
           path: dist/
 
       - name: Download Rust arm64 binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: numaflow-rs-linux-arm64
           path: dist/
@@ -171,7 +171,7 @@ jobs:
       - run: bom generate --image quay.io/numaproj/numaflow:$VERSION -o /tmp/numaflow.spdx
       # pack the boms into one file to make it easy to download
       - run: cd /tmp && tar -zcf sbom.tar.gz *.spdx
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: sbom.tar.gz
           path: /tmp/sbom.tar.gz
@@ -191,11 +191,11 @@ jobs:
             echo "VERSION=${GITHUB_REF##*/}" >> $GITHUB_ENV
           fi
       - name: Download binaries
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: binaries
           path: dist/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: sbom.tar.gz
           path: /tmp

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,25 @@
 # Changelog
 
+## v1.4.2 (2024-12-11)
+
+ * [c9dc38f4](https://github.com/numaproj/numaflow/commit/c9dc38f4cce2b5db598536a7539f2a35febcf1ca) Update manifests to v1.4.2
+ * [fea792b3](https://github.com/numaproj/numaflow/commit/fea792b36bd342adcdcdd96768b6fdd68921bfd2) fix: set max decode size of proto message (#2275)
+
+### Contributors
+
+ * Sidhant Kohli
+
+## v1.4.1 (2024-12-05)
+
+ * [346f2a73](https://github.com/numaproj/numaflow/commit/346f2a7321d158fa9ce9392cfdcc76d671d6f577) Update manifests to v1.4.1
+ * [1343e4d4](https://github.com/numaproj/numaflow/commit/1343e4d47934afcea324d4426df810dd9e99d9ab) feat: add sdk infomation metrics (#2208)
+ * [1abb5ede](https://github.com/numaproj/numaflow/commit/1abb5ede3577016b7c2a923755e1445146efdb05) fix: Fix Sink Config to respect Fallback (#2261)
+
+### Contributors
+
+ * Derek Wang
+ * Yashash H L
+
 ## v1.4.0 (2024-11-08)
 
  * [6892c115](https://github.com/numaproj/numaflow/commit/6892c11590ea482c186724e55837dbcfb2100ce3) Update manifests to v1.4.0
@@ -283,6 +303,15 @@
  * samhith-kakarla
  * xdevxy
 
+## v1.2.2 (2024-11-15)
+
+ * [61adf4e9](https://github.com/numaproj/numaflow/commit/61adf4e9805c2772d937a7513afcb3c14048127c) Update manifests to v1.2.2
+ * [623cc4e2](https://github.com/numaproj/numaflow/commit/623cc4e2aaa2d67d196cb972bd525a60544d2148) fix: update key len (#2223)
+
+### Contributors
+
+ * Sidhant Kohli
+
 ## v1.2.1 (2024-05-07)
 
  * [89ea33f1](https://github.com/numaproj/numaflow/commit/89ea33f1d69785f6f5f17f1d5854ac189003918a) Update manifests to v1.2.1

diff --git a/Dockerfile b/Dockerfile
@@ -69,8 +69,6 @@ COPY --from=base /bin/numaflow /bin/numaflow
 COPY --from=base /bin/numaflow-rs /bin/numaflow-rs
 COPY ui/build /ui/build
 
-COPY ./rust/serving/config config
-
 ENTRYPOINT [ "/bin/numaflow" ]
 
 ####################################################################################################
@@ -89,4 +87,4 @@ RUN chmod +x /bin/e2eapi
 ####################################################################################################
 FROM scratch AS e2eapi
 COPY --from=testbase /bin/e2eapi .
-ENTRYPOINT ["/e2eapi"]
+ENTRYPOINT ["/e2eapi"]
diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -21764,7 +21764,7 @@
           "description": "BackOff specifies the parameters for the backoff strategy, controlling how delays between retries should increase."
         },
         "onFailure": {
-          "description": "OnFailure specifies the action to take when a retry fails. The default action is to retry.",
+          "description": "OnFailure specifies the action to take when the specified retry strategy fails. The possible values are: 1. \"retry\": start another round of retrying the operation, 2. \"fallback\": re-route the operation to a fallback sink and 3. \"drop\": drop the operation and perform no further action. The default action is to retry.",
           "type": "string"
         }
       },

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -21751,7 +21751,7 @@
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Backoff"
         },
         "onFailure": {
-          "description": "OnFailure specifies the action to take when a retry fails. The default action is to retry.",
+          "description": "OnFailure specifies the action to take when the specified retry strategy fails. The possible values are: 1. \"retry\": start another round of retrying the operation, 2. \"fallback\": re-route the operation to a fallback sink and 3. \"drop\": drop the operation and perform no further action. The default action is to retry.",
           "type": "string"
         }
       }

diff --git a/config/advanced-install/namespaced-numaflow-server.yaml b/config/advanced-install/namespaced-numaflow-server.yaml
@@ -143,6 +143,77 @@ data:
     # example for local prometheus service
     # url: http://prometheus-operated.monitoring.svc.cluster.local:9090
     patterns:
+    - name: vertex_gauge
+      object: vertex
+      title: Vertex Pending Messages
+      description: This query is the total number of pending messages for the vertex
+      expr: |
+        sum($metric_name{$filters}) by ($dimension, period)
+      params:
+        - name: start_time
+          required: false
+        - name: end_time
+          required: false
+      metrics:
+        - metric_name: vertex_pending_messages
+          display_name: Vertex Pending Messages
+          # set "Units" or unset for default behaviour
+          # unit: Units
+          required_filters:
+            - namespace
+            - pipeline
+            - vertex
+          dimensions:
+            - name: pod
+              # expr: optional expression for prometheus query
+              # overrides the default expression
+              filters:
+                - name: pod
+                  required: false
+                - name: period
+                  required: false
+            - name: vertex
+              # expr: optional expression for prometheus query
+              # overrides the default expression
+              filters:
+                - name: period
+                  required: false
+
+    - name: mono_vertex_gauge
+      object: mono-vertex
+      title: Pending Messages Lag
+      description: This query is the total number of pending messages for the mono vertex
+      expr: |
+        sum($metric_name{$filters}) by ($dimension, period)
+      params:
+        - name: start_time
+          required: false
+        - name: end_time
+          required: false
+      metrics:
+        - metric_name: monovtx_pending
+          display_name: Mono Vertex Pending Messages
+          # set "Units" or unset for default behaviour
+          # unit: Units
+          required_filters:
+            - namespace
+            - mvtx_name
+          dimensions:
+            - name: pod
+              # expr: optional expression for prometheus query
+              # overrides the default expression
+              filters:
+                - name: pod
+                  required: false
+                - name: period
+                  required: false
+            - name: mono-vertex
+              # expr: optional expression for prometheus query
+              # overrides the default expression
+              filters:
+                - name: period
+                  required: false
+
     - name: mono_vertex_histogram
       object: mono-vertex
       title: Processing Time Latency
@@ -160,32 +231,34 @@ data:
           required: false
       metrics:
         - metric_name: monovtx_processing_time_bucket
+          display_name: Mono Vertex Processing Time Latency
+          # set "Units" or unset for default behaviour otherwise set "s" or "ms" for latency metrics
+          # Note: latency values are in μs
+          # unit: s
           required_filters:
             - namespace
             - mvtx_name
           dimensions:
+            - name: mono-vertex
             - name: pod
-              # expr: optional expression for prometheus query
-              # overrides the default expression
               filters:
                 - name: pod
                   required: false
+        - metric_name: monovtx_sink_time_bucket
+          display_name: Mono Vertex Sink Write Time Latency
+          # set "Units" or unset for default behaviour otherwise set "s" or "ms" for latency metrics
+          # Note: latency values are in μs
+          # unit: ms
+          required_filters:
+            - namespace
+            - mvtx_name
+          dimensions:
             - name: mono-vertex
-              # expr: optional expression for prometheus query
-              # overrides the default expression
-        # Add histogram metrics similar to the pattern above
-        #- metric_name: monovtx_sink_time_bucket
-        #  required_filters:
-        #    - namespace
-        #    - mvtx_name
-        #  dimensions:
-        #    - name: pod
-        #      #expr: optional
-        #      filters:
-        #        - name: pod
-        #          required: false
-        #    - name: mono-vertex
-        #      #expr: optional
+            - name: pod
+              filters:
+                - name: pod
+                  required: false
+
     - name: vertex_throughput
       object: vertex
       title: Vertex Throughput and Message Rates
@@ -200,14 +273,42 @@ data:
           required: false
       metrics:
         - metric_name: forwarder_data_read_total
+          display_name: Vertex Read Processing Rate
+          # set "Units" or unset for default behaviour
+          # unit: Units
           required_filters:
             - namespace
             - pipeline
             - vertex
           dimensions:
             - name: vertex
-              # expr: optional expression for prometheus query
-              # overrides the default expression
+            - name: pod
+              filters:
+                - name: pod
+                  required: false
+
+    - name: mono_vertex_throughput
+      object: mono-vertex
+      title: Mono-Vertex Throughput and Message Rates
+      description: This pattern measures the throughput of a mono-vertex in messages per second across different dimensions
+      expr: sum(rate($metric_name{$filters}[$duration])) by ($dimension)
+      params:
+        - name: duration
+          required: true
+        - name: start_time
+          required: false
+        - name: end_time
+          required: false
+      metrics:
+        - metric_name: monovtx_read_total
+          display_name: Mono Vertex Read Processing Rate
+          # set "Units" or unset for default behaviour
+          # unit: Units
+          required_filters:
+            - namespace
+            - mvtx_name
+          dimensions:
+            - name: mono-vertex
             - name: pod
               filters:
                 - name: pod