Merge branch 'fluent:master' into master

.github/workflows/call-build-linux-packages.yaml

@@ -54,7 +54,7 @@ jobs:
   call-build-capture-source:
     # Capture source tarball and generate checksum for it
     name: Extract any supporting metadata
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: ${{ inputs.environment }}
     permissions:
       contents: read
@@ -105,8 +105,8 @@ jobs:
   call-build-linux-packages:
     name: ${{ matrix.distro }} package build and stage to S3
     environment: ${{ inputs.environment }}
-    # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
-    runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
+    # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-22.04 usage.
+    runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-22.04' }}
     permissions:
       contents: read
     strategy:

.github/workflows/call-build-macos.yaml

@@ -74,8 +74,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - name: "Normal macOS-latest runner (Intel)"
-            runner: macos-12
           - name: "Apple Silicon macOS runner"
             runner: macos-14
 
@@ -121,8 +119,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - name: "Normal macOS-latest package (Intel)"
-            os: macos-12
           - name: "Apple Silicon macOS package"
             os: macos-14
 

.github/workflows/call-run-integration-test.yaml

@@ -192,7 +192,7 @@ jobs:
           bats-version: 1.9.0
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.10.0
+        uses: helm/kind-action@v1.12.0
         with:
           node_image: kindest/node:${{ matrix.k8s-release }}
           cluster_name: kind

.github/workflows/call-test-images.yaml

@@ -184,7 +184,7 @@ jobs:
           ref: ${{ inputs.ref }}
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.10.0
+        uses: helm/kind-action@v1.12.0
 
       - name: Set up Helm
         uses: azure/setup-helm@v4

.github/workflows/staging-release.yaml

@@ -172,7 +172,7 @@ jobs:
 
   staging-release-apt-packages:
     name: S3 - update APT packages bucket
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: release
     needs:
       - staging-release-version-check
@@ -327,7 +327,7 @@ jobs:
 
   staging-release-source-s3:
     name: S3 - update source bucket
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: release
     needs:
       - staging-release-version-check

.github/workflows/unit-tests.yaml

@@ -78,6 +78,7 @@ jobs:
           CC: ${{ matrix.compiler }}
           CXX: ${{ matrix.compiler }}
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-macos-unit-tests:
     # We chain this after Linux one as there are costs and restrictions associated
@@ -111,6 +112,7 @@ jobs:
           CC: gcc
           CXX: g++
           FLB_OPT: ${{ matrix.flb_option }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-aarch64-unit-tests:
     # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
@@ -167,6 +169,7 @@ jobs:
         env:
           CC: ${{ matrix.config.compiler }}
           CXX: ${{ matrix.config.compiler }}
+          CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }}
 
   run-qemu-ubuntu-unit-tests:
     # We chain this after Linux one as there are CPU time costs for QEMU emulation
@@ -179,6 +182,7 @@ jobs:
       matrix:
         arch:
           - s390x
+          - riscv64
     steps:
       - name: Checkout Fluent Bit code
         uses: actions/checkout@v4
@@ -188,19 +192,17 @@ jobs:
         id: build-and-test-on-qemu
         with:
           arch: ${{ matrix.arch }}
-          distro: ubuntu20.04
+          distro: ubuntu22.04
           shell: /bin/bash
           dockerRunArgs: |
             --volume "/var/lib/dbus/machine-id:/var/lib/dbus/machine-id"
             --volume "/etc/machine-id:/etc/machine-id"
           install: |
             apt-get update
-            apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common#libsystemd-dev
-            ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+            apt-get install -y gcc-12 g++-12 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common
 
-            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
-            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
-            update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90
+            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90
           run: |
             cd build
             export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
@@ -218,7 +220,6 @@ jobs:
             make -j $nparallel
             ctest -j $nparallel --build-run-dir . --output-on-failure
 
-
   # Required check looks at this so do not remove
   run-all-unit-tests:
     if: always()

CMakeLists.txt

@@ -6,9 +6,9 @@ cmake_policy(SET CMP0069 NEW)
 set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 
 # Fluent Bit Version
-set(FLB_VERSION_MAJOR  3)
-set(FLB_VERSION_MINOR  2)
-set(FLB_VERSION_PATCH  3)
+set(FLB_VERSION_MAJOR  4)
+set(FLB_VERSION_MINOR  0)
+set(FLB_VERSION_PATCH  0)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -321,6 +321,11 @@ if (FLB_SYSTEM_LINUX)
   include(cmake/s390x.cmake)
 endif ()
 
+# Build for Linux - riscv64 arch
+if (FLB_SYSTEM_LINUX)
+  include(cmake/riscv64.cmake)
+endif ()
+
 # Enable signed char support on Linux AARCH64 if specified
 if (FLB_LINUX_ON_AARCH64)
   set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")

CODEOWNERS

@@ -63,7 +63,7 @@
 /plugins/out_datadog     @nokute78 @edsiper
 /plugins/out_es          @pettitwesley @edsiper
 /plugins/out_pgsql       @sxd
-/plugins/out_stackdriver @braydonk @igorpeshansky @qingling128
+/plugins/out_stackdriver @braydonk @jefferbrecht @jeffluoo
 
 # AWS Plugins
 /plugins/out_s3               @pettitwesley
@@ -85,8 +85,8 @@
 
 # Google test code
 # --------------
-/tests/runtime/out_stackdriver.c @braydonk @igorpeshansky @qingling128
-/tests/runtime/data/stackdriver  @braydonk @igorpeshansky @qingling128
+/tests/runtime/out_stackdriver.c @braydonk @jefferbrecht @jeffluoo
+/tests/runtime/data/stackdriver  @braydonk @jefferbrecht @jeffluoo
 
 # Devcontainer
 /.devcontainer                  @patrick-stephens @niedbalski @edsiper

README.md

@@ -13,16 +13,14 @@
 
 ![](documentation/fluentbit_ecosystem.png)
 
-[Fluent Bit](http://fluentbit.io) is a fast Log Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project.
+[Fluent Bit](http://fluentbit.io) is a fast Log, Metrics and Traces Processor and Forwarder for Linux, Windows, Embedded Linux, MacOS and BSD family operating systems. It's part of the Graduated [Fluentd](http://fluentd.org) Ecosystem and a [CNCF](https://cncf.io) sub-project.
 
-Fluent Bit allows to collect log events or metrics from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point.
+Fluent Bit allows to collect different signal types such as logs, metrics and traces from different sources, process them and deliver them to different backends such as [Fluentd](http://fluentd.org), Elasticsearch, Splunk, DataDog, Kafka, New Relic, Azure services, AWS services, Google services, NATS, InfluxDB or any custom HTTP end-point.
 
 Fluent Bit comes with full SQL [Stream Processing](https://docs.fluentbit.io/manual/stream-processing/introduction) capabilities: data manipulation and analytics using SQL queries.
 
 Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures.
 
-
-
 ## Features
 
 - High Performance at low CPU and Memory footprint
@@ -48,7 +46,7 @@ Fluent Bit runs on x86_64, x86, arm32v7, and arm64v8 architectures.
 
 ## Fluent Bit in Production
 
-[Fluent Bit](https://fluentbit.io) is used widely in production environments. As of 2022, [Fluent Bit surpasses 3 Billion downloads](https://www.cncf.io/blog/2022/10/13/fluent-bit-surpasses-three-billion-downloads/) and continues to be deployed over **10 million times a day**. The following is a preview of who uses Fluent Bit heavily in production:
+Fluent Bit is a widely adopted solution in production environments. As of 2024, Fluent Bit has surpassed 15 billion downloads and continues to be deployed over 10 million times daily. Below is a preview of some of the organizations that rely heavily on Fluent Bit in their production systems:
 
 > If your company uses Fluent Bit and is not listed, feel free to open a GitHub issue and we will add the logo.
 

cmake/headers.cmake

@@ -37,6 +37,7 @@ include_directories(
 
   ${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
   ${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics
+  ${CMAKE_CURRENT_BINARY_DIR}/lib/cprofiles/include
   ${CMAKE_CURRENT_BINARY_DIR}/include
 
   ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/

cmake/libraries.cmake

@@ -20,8 +20,8 @@ set(FLB_PATH_LIB_ONIGMO      "lib/onigmo")
 set(FLB_PATH_LIB_MPACK       "lib/mpack-amalgamation-1.1.1")
 set(FLB_PATH_LIB_MINIZ       "lib/miniz")
 set(FLB_PATH_LIB_TUTF8E      "lib/tutf8e")
-set(FLB_PATH_LIB_CARES       "lib/c-ares-1.33.1")
+set(FLB_PATH_LIB_CARES       "lib/c-ares-1.34.4")
 set(FLB_PATH_LIB_SNAPPY      "lib/snappy-fef67ac")
 set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.4.0")
 set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")
-set(FLB_PATH_LIB_WASM_MICRO_RUNTIME  "lib/wasm-micro-runtime-WAMR-1.3.0")
+set(FLB_PATH_LIB_WASM_MICRO_RUNTIME  "lib/wasm-micro-runtime-WAMR-1.3.3")

cmake/riscv64.cmake

@@ -0,0 +1,8 @@
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv64)")
+  message(STATUS "Forcing characters to be signed, as on x86_64.")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")
+  if(FLB_LUAJIT)
+    message(WARNING "LuaJIT is disabled, this platform does not support built-in LuaJIT and system provided one neither.")
+    set(FLB_LUAJIT OFF)
+  endif()
+endif ()

dockerfiles/Dockerfile

@@ -13,7 +13,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=3.2.3
+ARG RELEASE_VERSION=3.2.4
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32

fluent-bit-3.2.3.bb → fluent-bit-4.0.0.bb

@@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
 SECTION = "net"
 
 PR = "r0"
-PV = "3.2.3"
+PV = "4.0.0"
 
 SRCREV = "v${PV}"
 SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"

include/fluent-bit/calyptia/calyptia_constants.h

@@ -36,8 +36,10 @@
 #define CALYPTIA_ENDPOINT_TRACE   "/v1/traces/%s"
 
 #define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets"
-#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini&config_format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML "/v1/fleets/%s/config?format=yaml&config_format=yaml"
 #define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files"
+#define CALYPTIA_ENDPOINT_FLEET_BY_NAME "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true"
 
 /* Storage */
 #define CALYPTIA_SESSION_FILE     "session.CALYPTIA"

include/fluent-bit/flb_cfl_ra_key.h

@@ -0,0 +1,74 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2024 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_CFL_RA_KEY_H
+#define FLB_CFL_RA_KEY_H
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_sds.h>
+#include <fluent-bit/flb_regex.h>
+#include <fluent-bit/record_accessor/flb_ra_parser.h>
+#include <monkey/mk_core.h>
+#include <cfl/cfl.h>
+
+#include <stdbool.h>
+
+enum cfl_ra_types {
+    FLB_CFL_RA_BOOL = 0,
+    FLB_CFL_RA_INT,
+    FLB_CFL_RA_FLOAT,
+    FLB_CFL_RA_STRING,
+    FLB_CFL_RA_NULL
+};
+
+/* condition value types */
+typedef union {
+    bool boolean;
+    int64_t i64;
+    double f64;
+    flb_sds_t string;
+} cfl_ra_val;
+
+/* Represent any value object */
+struct flb_cfl_ra_value {
+    int type;
+    struct cfl_variant v;
+    cfl_ra_val val;
+};
+
+struct flb_cfl_ra_value *flb_cfl_ra_key_to_value(flb_sds_t ckey,
+                                                   struct cfl_variant vobj,
+                                                   struct mk_list *subkeys);
+void flb_cfl_ra_key_value_destroy(struct flb_cfl_ra_value *v);
+
+int flb_cfl_ra_key_value_get(flb_sds_t ckey, struct cfl_variant vobj,
+                             struct mk_list *subkeys,
+                             cfl_sds_t *start_key,
+                             cfl_sds_t *out_key, struct cfl_variant **out_val);
+
+int flb_cfl_ra_key_strcmp(flb_sds_t ckey, struct cfl_variant vobj,
+                          struct mk_list *subkeys, char *str, int len);
+int flb_cfl_ra_key_regex_match(flb_sds_t ckey, struct cfl_variant vobj,
+                               struct mk_list *subkeys, struct flb_regex *regex,
+                               struct flb_regex_search *result);
+int flb_cfl_ra_key_value_append(struct flb_ra_parser *rp, struct cfl_variant *vobj,
+                                struct cfl_variant *in_val);
+int flb_cfl_ra_key_value_update(struct flb_ra_parser *rp,  struct cfl_variant *vobj,
+                                cfl_sds_t in_key, struct cfl_variant *in_val);
+#endif