diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index dcfa5712..d2851705 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -102,10 +102,48 @@ jobs: id: extract_version run: echo "VERSION_NAME=${GITHUB_REF#refs/heads/release/}" >> $GITHUB_ENV + - name: Install and setup NASM on Windows + if: matrix.target == 'x86_64-pc-windows-msvc' + shell: pwsh + run: | + choco install nasm + $nasmPath = "C:\Program Files\NASM" + $env:PATH += ";$nasmPath" + echo "$nasmPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "NASM_PATH=$nasmPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + refreshenv + nasm -v + + - name: Install vcpkg on Windows + if: matrix.target == 'x86_64-pc-windows-msvc' + shell: pwsh + run: | + git clone https://github.com/Microsoft/vcpkg.git C:\vcpkg + C:\vcpkg\bootstrap-vcpkg.bat + C:\vcpkg\vcpkg integrate install + echo "VCPKG_ROOT=C:\vcpkg" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "C:\vcpkg" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install CMake on Windows + if: matrix.target == 'x86_64-pc-windows-msvc' + shell: pwsh + run: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + refreshenv + cmake --version + echo "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Install dependencies with vcpkg on Windows + if: matrix.target == 'x86_64-pc-windows-msvc' + shell: pwsh + run: | + C:\vcpkg\vcpkg install openssl:x64-windows-static-md zlib:x64-windows-static-md + - name: Build binaries working-directory: cli env: SVM_TARGET_PLATFORM: ${{ matrix.svm_target_platform }} + CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake shell: bash run: | set -eo pipefail @@ -118,10 +156,19 @@ jobs: [[ "$target" == *windows* ]] && exe=".exe" + if [[ "$target" == *windows* ]]; then + export PATH="$PATH:/c/Program Files/NASM:/c/vcpkg" + export NASM="$NASM_PATH/nasm.exe" + echo "NASM location: $NASM" + "$NASM" -v + echo "CMAKE_TOOLCHAIN_FILE: $CMAKE_TOOLCHAIN_FILE" + ls -l "$CMAKE_TOOLCHAIN_FILE" || echo "Toolchain file not found!" + fi + if [[ "${{ env.BUILD_TYPE }}" == "release" ]]; then - cargo build --release --target "$target" "${flags[@]}" + RUST_BACKTRACE=1 CMAKE_TOOLCHAIN_FILE="$CMAKE_TOOLCHAIN_FILE" cargo build --release --target "$target" "${flags[@]}" -vv else - cargo build --target "$target" "${flags[@]}" + RUST_BACKTRACE=1 CMAKE_TOOLCHAIN_FILE="$CMAKE_TOOLCHAIN_FILE" cargo build --target "$target" "${flags[@]}" -vv fi - name: Archive binaries diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4401e056..1b2b720f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rindexer_cli" -version = "0.3.1" +version = "0.5.0" edition = "2021" resolver = "2" diff --git a/core/Cargo.toml b/core/Cargo.toml index 40da6ffd..369495f2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -55,7 +55,6 @@ aws-sdk-sns = "1.37.0" lapin = "2.5.0" deadpool = { version = "0.12", features = ["rt_tokio_1"] } deadpool-lapin = "0.12" -rdkafka = { version = "0.36", features = ["tokio"] } teloxide = "0.12" serenity = { version = "0.12", features = ["client", "framework"] } @@ -63,6 +62,12 @@ serenity = { version = "0.12", features = ["client", "framework"] } jemallocator = { version = "0.5.0", optional = true } jemalloc-ctl = { version = "0.5.0", optional = true } +[target.'cfg(not(windows))'.dependencies] +rdkafka = { version = "0.36", features = ["tokio"] } + +[target.'cfg(windows)'.dependencies] +rdkafka = { version = "0.36", features = ["tokio", "cmake-build"] } + [profile.release] lto = "fat" codegen-units = 1 diff --git a/core/src/streams/kafka.rs b/core/src/streams/kafka.rs index a5a553e1..3f9b5334 100644 --- a/core/src/streams/kafka.rs +++ b/core/src/streams/kafka.rs @@ -1,5 +1,6 @@ use std::time::Duration; +#[cfg(not(windows))] use rdkafka::{ config::ClientConfig, message::{Header, OwnedHeaders}, @@ -14,7 +15,7 @@ use crate::{manifest::stream::KafkaStreamConfig, streams::STREAM_MESSAGE_ID_KEY} #[derive(Error, Debug)] pub enum KafkaError { #[error("Kafka error: {0}")] - RdkafkaError(#[from] rdkafka::error::KafkaError), + RdkafkaError(String), #[error("Could not parse message: {0}")] CouldNotParseMessage(#[from] serde_json::Error), @@ -22,32 +23,42 @@ pub enum KafkaError { #[derive(Clone)] pub struct Kafka { + #[cfg(not(windows))] producer: FutureProducer, } impl Kafka { pub async fn new(config: &KafkaStreamConfig) -> Result { - let servers_list = config.brokers.join(","); - let mut client_config = ClientConfig::new(); + #[cfg(not(windows))] + { + let servers_list = config.brokers.join(","); + let mut client_config = ClientConfig::new(); - client_config - .set("bootstrap.servers", &servers_list) - .set("security.protocol", &config.security_protocol) - .set("acks", &config.acks); + client_config + .set("bootstrap.servers", &servers_list) + .set("security.protocol", &config.security_protocol) + .set("acks", &config.acks); - if let Some(ref sasl_mechanisms) = config.sasl_mechanisms { - client_config.set("sasl.mechanisms", sasl_mechanisms); - } - if let Some(ref sasl_username) = config.sasl_username { - client_config.set("sasl.username", sasl_username); - } - if let Some(ref sasl_password) = config.sasl_password { - client_config.set("sasl.password", sasl_password); - } + if let Some(ref sasl_mechanisms) = config.sasl_mechanisms { + client_config.set("sasl.mechanisms", sasl_mechanisms); + } + if let Some(ref sasl_username) = config.sasl_username { + client_config.set("sasl.username", sasl_username); + } + if let Some(ref sasl_password) = config.sasl_password { + client_config.set("sasl.password", sasl_password); + } - let producer: FutureProducer = client_config.create().map_err(KafkaError::RdkafkaError)?; + let producer: FutureProducer = client_config.create() + .map_err(|e| KafkaError::RdkafkaError(e.to_string()))?; - Ok(Self { producer }) + Ok(Self { producer }) + } + + #[cfg(windows)] + { + panic!("Kafka is not supported on Windows") + } } pub async fn publish( @@ -57,23 +68,31 @@ impl Kafka { key: &Option, message: &Value, ) -> Result<(), KafkaError> { - let message_body = serde_json::to_vec(message)?; - - let record = if key.is_some() { - FutureRecord::to(topic).key(key.as_ref().unwrap()).payload(&message_body).headers( - OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }), - ) - } else { - FutureRecord::to(topic).payload(&message_body).headers( - OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }), - ) - }; - - self.producer - .send(record, Timeout::After(Duration::from_secs(0))) - .await - .map_err(|(e, _)| KafkaError::RdkafkaError(e))?; - - Ok(()) + #[cfg(not(windows))] + { + let message_body = serde_json::to_vec(message)?; + + let record = if key.is_some() { + FutureRecord::to(topic).key(key.as_ref().unwrap()).payload(&message_body).headers( + OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }), + ) + } else { + FutureRecord::to(topic).payload(&message_body).headers( + OwnedHeaders::new().insert(Header { key: STREAM_MESSAGE_ID_KEY, value: Some(id) }), + ) + }; + + self.producer + .send(record, Timeout::After(Duration::from_secs(0))) + .await + .map_err(|(e, _)| KafkaError::RdkafkaError(e.to_string()))?; + + Ok(()) + } + + #[cfg(windows)] + { + panic!("Kafka is not supported on Windows") + } } -} +} \ No newline at end of file diff --git a/documentation/docs/pages/docs/changelog.mdx b/documentation/docs/pages/docs/changelog.mdx index ba5fa5d6..6f6d7168 100644 --- a/documentation/docs/pages/docs/changelog.mdx +++ b/documentation/docs/pages/docs/changelog.mdx @@ -26,6 +26,27 @@ all release branches are deployed through `release/VERSION_NUMBER` branches +## 0.5.0-beta - 6th August 2024 + +github branch - https://github.com/joshstevens19/rindexer/tree/release/0.5.0 + +- linux binary - https://rindexer.xyz/releases/linux-amd64/0.5.0/rindexer_linux-amd64.tar.gz +- mac apple silicon binary - https://rindexer.xyz/releases/darwin-arm64/0.5.0/rindexer_darwin-arm64.tar.gz +- mac apple intel binary - https://rindexer.xyz/releases/darwin-amd64/0.5.0/rindexer_darwin-amd64.tar.gz +- windows binary - https://rindexer/releases.xyz/win32-amd64/0.5.0/rindexer_win32-amd64.zip + +### Features +------------------------------------------------- + +- feat: support chatbots on telegram - https://rindexer.xyz/docs/start-building/chatbots/telegram +- feat: support chatbots on discord - https://rindexer.xyz/docs/start-building/chatbots/discord +- feat: support chatbots on slack - https://rindexer.xyz/docs/start-building/chatbots/slack +- feat: support streams with kafka - https://rindexer.xyz/docs/start-building/streams/kafka +- feat: support streams with rabbitmq - https://rindexer.xyz/docs/start-building/streams/rabbitmq +- feat: support streams with webhooks - https://rindexer.xyz/docs/start-building/streams/webhooks +- feat: support streams with sns/sqs - https://rindexer.xyz/docs/start-building/streams/sns +- feat: create .gitignore file for new projects + ## 0.4.0-beta - 30th July 2024 github branch - https://github.com/joshstevens19/rindexer/tree/release/0.4.0 diff --git a/documentation/docs/pages/docs/start-building/streams/kafka.mdx b/documentation/docs/pages/docs/start-building/streams/kafka.mdx index af033a8f..a5a85425 100644 --- a/documentation/docs/pages/docs/start-building/streams/kafka.mdx +++ b/documentation/docs/pages/docs/start-building/streams/kafka.mdx @@ -1,5 +1,10 @@ # Kafka +:::warn +Kafka streams do not work with windows from the CLI installation, it will panic if you try to use it with windows. +If you are on windows and want to use kafka streams you should use the docker image. +::: + :::info rindexer streams can be used without any other storage providers. It can also be used with storage providers. ::: diff --git a/documentation/docs/public/releases/darwin-amd64/0.5.0/rindexer_darwin-amd64.tar.gz b/documentation/docs/public/releases/darwin-amd64/0.5.0/rindexer_darwin-amd64.tar.gz new file mode 100644 index 00000000..0552ca3c Binary files /dev/null and b/documentation/docs/public/releases/darwin-amd64/0.5.0/rindexer_darwin-amd64.tar.gz differ diff --git a/documentation/docs/public/releases/darwin-amd64/rindexer_darwin-amd64.tar.gz b/documentation/docs/public/releases/darwin-amd64/rindexer_darwin-amd64.tar.gz index 5d10271f..f53c9cf7 100644 Binary files a/documentation/docs/public/releases/darwin-amd64/rindexer_darwin-amd64.tar.gz and b/documentation/docs/public/releases/darwin-amd64/rindexer_darwin-amd64.tar.gz differ diff --git a/documentation/docs/public/releases/darwin-arm64/0.5.0/rindexer_darwin-arm64.tar.gz b/documentation/docs/public/releases/darwin-arm64/0.5.0/rindexer_darwin-arm64.tar.gz new file mode 100644 index 00000000..7ea78805 Binary files /dev/null and b/documentation/docs/public/releases/darwin-arm64/0.5.0/rindexer_darwin-arm64.tar.gz differ diff --git a/documentation/docs/public/releases/darwin-arm64/rindexer_darwin-arm64.tar.gz b/documentation/docs/public/releases/darwin-arm64/rindexer_darwin-arm64.tar.gz index 6bc143ba..172f03c5 100644 Binary files a/documentation/docs/public/releases/darwin-arm64/rindexer_darwin-arm64.tar.gz and b/documentation/docs/public/releases/darwin-arm64/rindexer_darwin-arm64.tar.gz differ diff --git a/documentation/docs/public/releases/linux-amd64/0.5.0/rindexer_linux-amd64.tar.gz b/documentation/docs/public/releases/linux-amd64/0.5.0/rindexer_linux-amd64.tar.gz new file mode 100644 index 00000000..c9f362e2 Binary files /dev/null and b/documentation/docs/public/releases/linux-amd64/0.5.0/rindexer_linux-amd64.tar.gz differ diff --git a/documentation/docs/public/releases/linux-amd64/rindexer_linux-amd64.tar.gz b/documentation/docs/public/releases/linux-amd64/rindexer_linux-amd64.tar.gz index 509f75a7..c9f362e2 100644 Binary files a/documentation/docs/public/releases/linux-amd64/rindexer_linux-amd64.tar.gz and b/documentation/docs/public/releases/linux-amd64/rindexer_linux-amd64.tar.gz differ diff --git a/documentation/docs/public/releases/win32-amd64/0.5.0/rindexer_win32-amd64.zip b/documentation/docs/public/releases/win32-amd64/0.5.0/rindexer_win32-amd64.zip new file mode 100644 index 00000000..80e40c95 Binary files /dev/null and b/documentation/docs/public/releases/win32-amd64/0.5.0/rindexer_win32-amd64.zip differ diff --git a/documentation/docs/public/releases/win32-amd64/rindexer_win32-amd64.zip b/documentation/docs/public/releases/win32-amd64/rindexer_win32-amd64.zip index 3a5385ae..80e40c95 100644 Binary files a/documentation/docs/public/releases/win32-amd64/rindexer_win32-amd64.zip and b/documentation/docs/public/releases/win32-amd64/rindexer_win32-amd64.zip differ