Release/0.5.0

.github/workflows/ci-cd.yml

@@ -102,10 +102,48 @@ jobs:
         id: extract_version
         run: echo "VERSION_NAME=${GITHUB_REF#refs/heads/release/}" >> $GITHUB_ENV
 
+      - name: Install and setup NASM on Windows
+        if: matrix.target == 'x86_64-pc-windows-msvc'
+        shell: pwsh
+        run: |
+          choco install nasm
+          $nasmPath = "C:\Program Files\NASM"
+          $env:PATH += ";$nasmPath"
+          echo "$nasmPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "NASM_PATH=$nasmPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          refreshenv
+          nasm -v
+
+      - name: Install vcpkg on Windows
+        if: matrix.target == 'x86_64-pc-windows-msvc'
+        shell: pwsh
+        run: |
+          git clone https://github.com/Microsoft/vcpkg.git C:\vcpkg
+          C:\vcpkg\bootstrap-vcpkg.bat
+          C:\vcpkg\vcpkg integrate install
+          echo "VCPKG_ROOT=C:\vcpkg" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "C:\vcpkg" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+      - name: Install CMake on Windows
+        if: matrix.target == 'x86_64-pc-windows-msvc'
+        shell: pwsh
+        run: |
+          choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+          refreshenv
+          cmake --version
+          echo "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
+      - name: Install dependencies with vcpkg on Windows
+        if: matrix.target == 'x86_64-pc-windows-msvc'
+        shell: pwsh
+        run: |
+          C:\vcpkg\vcpkg install openssl:x64-windows-static-md zlib:x64-windows-static-md
+
       - name: Build binaries
         working-directory: cli
         env:
           SVM_TARGET_PLATFORM: ${{ matrix.svm_target_platform }}
+          CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake
         shell: bash
         run: |
           set -eo pipefail
@@ -118,10 +156,19 @@ jobs:
 
           [[ "$target" == *windows* ]] && exe=".exe"
 
+          if [[ "$target" == *windows* ]]; then
+            export PATH="$PATH:/c/Program Files/NASM:/c/vcpkg"
+            export NASM="$NASM_PATH/nasm.exe"
+            echo "NASM location: $NASM"
+            "$NASM" -v
+            echo "CMAKE_TOOLCHAIN_FILE: $CMAKE_TOOLCHAIN_FILE"
+            ls -l "$CMAKE_TOOLCHAIN_FILE" || echo "Toolchain file not found!"
+          fi
+
           if [[ "${{ env.BUILD_TYPE }}" == "release" ]]; then
-            cargo build --release --target "$target" "${flags[@]}"
+            RUST_BACKTRACE=1 CMAKE_TOOLCHAIN_FILE="$CMAKE_TOOLCHAIN_FILE" cargo build --release --target "$target" "${flags[@]}" -vv
           else
-            cargo build --target "$target" "${flags[@]}"
+            RUST_BACKTRACE=1 CMAKE_TOOLCHAIN_FILE="$CMAKE_TOOLCHAIN_FILE" cargo build --target "$target" "${flags[@]}" -vv
           fi
 
       - name: Archive binaries

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rindexer_cli"
-version = "0.3.1"
+version = "0.5.0"
 edition = "2021"
 resolver = "2"
 

core/Cargo.toml

@@ -55,14 +55,19 @@ aws-sdk-sns = "1.37.0"
 lapin = "2.5.0"
 deadpool = { version = "0.12", features = ["rt_tokio_1"] }
 deadpool-lapin = "0.12"
-rdkafka = { version = "0.36", features = ["tokio"] }
 teloxide = "0.12"
 serenity = { version = "0.12", features = ["client", "framework"] }
 
 # build
 jemallocator = { version = "0.5.0", optional = true }
 jemalloc-ctl = { version = "0.5.0", optional = true }
 
+[target.'cfg(not(windows))'.dependencies]
+rdkafka = { version = "0.36", features = ["tokio"] }
+
+[target.'cfg(windows)'.dependencies]
+rdkafka = { version = "0.36", features = ["tokio", "cmake-build"] }
+
 [profile.release]
 lto = "fat"
 codegen-units = 1

core/src/streams/kafka.rs

@@ -1,5 +1,6 @@
 use std::time::Duration;
 
+#[cfg(not(windows))]
 use rdkafka::{
     config::ClientConfig,
     message::{Header, OwnedHeaders},
@@ -14,40 +15,50 @@ use crate::{manifest::stream::KafkaStreamConfig, streams::STREAM_MESSAGE_ID_KEY}
 #[derive(Error, Debug)]
 pub enum KafkaError {
     #[error("Kafka error: {0}")]
-    RdkafkaError(#[from] rdkafka::error::KafkaError),
+    RdkafkaError(String),
 
     #[error("Could not parse message: {0}")]
     CouldNotParseMessage(#[from] serde_json::Error),
 }
 
 #[derive(Clone)]
 pub struct Kafka {
+    #[cfg(not(windows))]
     producer: FutureProducer,
 }
 
 impl Kafka {
     pub async fn new(config: &KafkaStreamConfig) -> Result<Self, KafkaError> {
-        let servers_list = config.brokers.join(",");
-        let mut client_config = ClientConfig::new();
+        #[cfg(not(windows))]
+        {
+            let servers_list = config.brokers.join(",");
+            let mut client_config = ClientConfig::new();
 
-        client_config
-            .set("bootstrap.servers", &servers_list)
-            .set("security.protocol", &config.security_protocol)
-            .set("acks", &config.acks);
+            client_config
+                .set("bootstrap.servers", &servers_list)
+                .set("security.protocol", &config.security_protocol)
+                .set("acks", &config.acks);
 
-        if let Some(ref sasl_mechanisms) = config.sasl_mechanisms {
-            client_config.set("sasl.mechanisms", sasl_mechanisms);
-        }
-        if let Some(ref sasl_username) = config.sasl_username {
-            client_config.set("sasl.username", sasl_username);
-        }
-        if let Some(ref sasl_password) = config.sasl_password {
-            client_config.set("sasl.password", sasl_password);
-        }
+            if let Some(ref sasl_mechanisms) = config.sasl_mechanisms {
+                client_config.set("sasl.mechanisms", sasl_mechanisms);
+            }
+            if let Some(ref sasl_username) = config.sasl_username {
+                client_config.set("sasl.username", sasl_username);
+            }
+            if let Some(ref sasl_password) = config.sasl_password {
+                client_config.set("sasl.password", sasl_password);
+            }
 
-        let producer: FutureProducer = client_config.create().map_err(KafkaError::RdkafkaError)?;
+            let producer: FutureProducer = client_config.create()
+                .map_err(|e| KafkaError::RdkafkaError(e.to_string()))?;
 
-        Ok(Self { producer })
+            Ok(Self { producer })
+        }
+
+        #[cfg(windows)]
+        {
+            panic!("Kafka is not supported on Windows")
+        }
     }
 
     pub async fn publish(
@@ -57,23 +68,31 @@ impl Kafka {
         key: &Option<String>,
         message: &Value,
     ) -> Result<(), KafkaError> {
-        let message_body = serde_json::to_vec(message)?;
-
-        let record = if key.is_some() {
-            FutureRecord::to(topic).key(key.as_ref().unwrap()).payload(&message_body).headers(
-                OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }),
-            )
-        } else {
-            FutureRecord::to(topic).payload(&message_body).headers(
-                OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }),
-            )
-        };
-
-        self.producer
-            .send(record, Timeout::After(Duration::from_secs(0)))
-            .await
-            .map_err(|(e, _)| KafkaError::RdkafkaError(e))?;
-
-        Ok(())
+        #[cfg(not(windows))]
+        {
+            let message_body = serde_json::to_vec(message)?;
+
+            let record = if key.is_some() {
+                FutureRecord::to(topic).key(key.as_ref().unwrap()).payload(&message_body).headers(
+                    OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }),
+                )
+            } else {
+                FutureRecord::to(topic).payload(&message_body).headers(
+                    OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }),
+                )
+            };
+
+            self.producer
+                .send(record, Timeout::After(Duration::from_secs(0)))
+                .await
+                .map_err(|(e, _)| KafkaError::RdkafkaError(e.to_string()))?;
+
+            Ok(())
+        }
+
+        #[cfg(windows)]
+        {
+            panic!("Kafka is not supported on Windows")
+        }
     }
-}
+}

documentation/docs/pages/docs/changelog.mdx

@@ -26,6 +26,27 @@
 
 all release branches are deployed through `release/VERSION_NUMBER` branches
 
+## 0.5.0-beta - 6th August 2024
+
+github branch - https://github.com/joshstevens19/rindexer/tree/release/0.5.0
+
+- linux binary - https://rindexer.xyz/releases/linux-amd64/0.5.0/rindexer_linux-amd64.tar.gz
+- mac apple silicon binary - https://rindexer.xyz/releases/darwin-arm64/0.5.0/rindexer_darwin-arm64.tar.gz
+- mac apple intel binary - https://rindexer.xyz/releases/darwin-amd64/0.5.0/rindexer_darwin-amd64.tar.gz
+- windows binary - https://rindexer/releases.xyz/win32-amd64/0.5.0/rindexer_win32-amd64.zip
+
+### Features
+-------------------------------------------------
+
+- feat: support chatbots on telegram - https://rindexer.xyz/docs/start-building/chatbots/telegram
+- feat: support chatbots on discord - https://rindexer.xyz/docs/start-building/chatbots/discord
+- feat: support chatbots on slack - https://rindexer.xyz/docs/start-building/chatbots/slack
+- feat: support streams with kafka - https://rindexer.xyz/docs/start-building/streams/kafka
+- feat: support streams with rabbitmq - https://rindexer.xyz/docs/start-building/streams/rabbitmq
+- feat: support streams with webhooks - https://rindexer.xyz/docs/start-building/streams/webhooks
+- feat: support streams with sns/sqs - https://rindexer.xyz/docs/start-building/streams/sns
+- feat: create .gitignore file for new projects
+
 ## 0.4.0-beta - 30th July 2024
 
 github branch - https://github.com/joshstevens19/rindexer/tree/release/0.4.0

documentation/docs/pages/docs/start-building/streams/kafka.mdx

@@ -1,5 +1,10 @@
 # Kafka
 
+:::warn
+Kafka streams do not work with windows from the CLI installation, it will panic if you try to use it with windows.
+If you are on windows and want to use kafka streams you should use the docker image.
+:::
+
 :::info
 rindexer streams can be used without any other storage providers. It can also be used with storage providers.
 :::