Merge branch 'main' of github.com:xmtp/libxmtp into insipx/v2-streaming

.github/workflows/release-kotlin-bindings.yml

@@ -0,0 +1,113 @@
+name: Release Kotlin Bindings
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-linux:
+    runs-on: warp-ubuntu-latest-x64-8x
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - x86_64-linux-android
+          - i686-linux-android
+          - armv7-linux-androideabi
+          - aarch64-linux-android
+        include:
+          - target: x86_64-linux-android
+            output_target: x86_64
+          - target: i686-linux-android
+            output_target: x86
+          - target: armv7-linux-androideabi
+            output_target: armeabi-v7a
+          - target: aarch64-linux-android
+            output_target: arm64-v8a
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: "Cache"
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            ./target/${{ matrix.target }}
+          key: ${{ matrix.target }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
+
+      - name: Install rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+          target: ${{ matrix.target }}
+
+      # Install latest cross to mitigate unwind linking issue on android builds.
+      # See https://github.com/cross-rs/cross/issues/1222
+      - name: Install rust cross
+        run: |
+          cargo install cross --git https://github.com/cross-rs/cross
+
+      - name: Build target
+        uses: actions-rs/cargo@v1
+        env: 
+          CROSS_NO_WARNINGS: "0"
+        with:
+          use-cross: true
+          command: build
+          args: --release --target ${{ matrix.target }} --manifest-path bindings_ffi/Cargo.toml --target-dir bindings_ffi/target
+
+      - name: Prepare JNI libs
+        run: |
+          mkdir -p bindings_ffi/jniLibs/${{ matrix.output_target }}/ && \
+          cp bindings_ffi/target/${{ matrix.target }}/release/libxmtpv3.so bindings_ffi/jniLibs/${{ matrix.output_target }}/libuniffi_xmtpv3.so
+     
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.output_target }}
+          path: bindings_ffi/jniLibs/${{ matrix.output_target }}/libuniffi_xmtpv3.so
+          retention-days: 1
+
+  package-kotlin:
+      needs: [build-linux]
+      runs-on: warp-ubuntu-latest-x64-8x
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v4
+
+        - name: Download artifacts
+          uses: actions/download-artifact@v3
+          with:
+            path: bindings_ffi/jniLibs
+
+        - name: Build archive
+          working-directory: bindings_ffi
+          run: |
+            zip -r LibXMTPKotlinFFI.zip jniLibs
+
+        - name: Get short SHA
+          id: slug
+          run: echo "::set-output name=sha7::$(echo ${GITHUB_SHA} | cut -c1-7)"
+
+        - name: Create Release
+          id: create_release
+          uses: actions/create-release@v1
+          env:
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          with:
+            tag_name: kotlin-bindings-${{ steps.slug.outputs.sha7 }}
+            release_name: Kotlin-Bindings-${{ steps.slug.outputs.sha7 }}
+            draft: false
+            prerelease: true
+
+        - name: Upload Release Asset
+          uses: actions/upload-release-asset@v1
+          env:
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          with:
+            upload_url: ${{ steps.create_release.outputs.upload_url }}
+            asset_path: ./bindings_ffi/LibXMTPKotlinFFI.zip
+            asset_name: LibXMTPKotlinFFI.zip
+            asset_content_type: application/zip

.github/workflows/validation-server-publish.yml

@@ -48,3 +48,13 @@ jobs:
           variable-name: validation_service_image
           variable-value: "ghcr.io/xmtp/mls-validation-service@${{ steps.push.outputs.digest }}"
           variable-value-required-prefix: "ghcr.io/xmtp/mls-validation-service@sha256:"
+
+      - name: Deploy (production)
+        uses: xmtp-labs/terraform-deployer@v1
+        with:
+          terraform-token: ${{ secrets.TERRAFORM_TOKEN }}
+          terraform-org: xmtp
+          terraform-workspace: production
+          variable-name: validation_service_image
+          variable-value: "ghcr.io/xmtp/mls-validation-service@${{ steps.push.outputs.digest }}"
+          variable-value-required-prefix: "ghcr.io/xmtp/mls-validation-service@sha256:"

.gitignore

@@ -10,6 +10,7 @@ wasm-pack.log
 # generated
 bindings_swift/Sources
 bindings_swift/xmtp-rust-swift.zip
+bindings_ffi/jniLibs
 
 # Logs
 logs

bindings_ffi/gen_kotlin.sh

@@ -11,3 +11,7 @@ bindings_ffi/target/release/ffi-uniffi-bindgen generate \
 popd > /dev/null
 make libxmtp-version
 cp libxmtp-version.txt src/uniffi/$PROJECT_NAME/
+
+# Assumes libxmtp is in a peer directory of xmtp-android
+cp src/uniffi/xmtpv3/xmtpv3.kt ../../xmtp-android/library/src/main/java/xmtpv3.kt
+cp src/uniffi/xmtpv3/libxmtp-version.txt ../../xmtp-android/library/src/main/java/libxmtp-version.txt

bindings_ffi/src/mls.rs

@@ -242,6 +242,20 @@ impl FfiXmtpClient {
         })
     }
 
+    pub fn group(&self, group_id: Vec<u8>) -> Result<FfiGroup, GenericError> {
+        let convo = self.inner_client.group(group_id)?;
+        Ok(FfiGroup {
+            inner_client: self.inner_client.clone(),
+            group_id: convo.group_id,
+            created_at_ns: convo.created_at_ns,
+        })
+    }
+
+    pub fn message(&self, message_id: Vec<u8>) -> Result<FfiMessage, GenericError> {
+        let message = self.inner_client.message(message_id)?;
+        Ok(message.into())
+    }
+
     pub async fn can_message(
         &self,
         account_addresses: Vec<String>,
@@ -264,6 +278,13 @@ impl FfiXmtpClient {
     pub async fn db_reconnect(&self) -> Result<(), GenericError> {
         Ok(self.inner_client.reconnect_db()?)
     }
+
+    pub async fn find_inbox_id(&self, address: String) -> Result<Option<String>, GenericError> {
+        let inner = self.inner_client.as_ref();
+
+        let result = inner.find_inbox_id_from_address(address).await?;
+        Ok(result)
+    }
 }
 
 #[uniffi::export(async_runtime = "tokio")]
@@ -313,6 +334,7 @@ pub struct FfiConversations {
 pub enum GroupPermissions {
     AllMembers,
     AdminOnly,
+    CustomPolicy,
 }
 
 impl From<PreconfiguredPolicies> for GroupPermissions {
@@ -1009,7 +1031,11 @@ pub struct FfiGroupPermissions {
 #[uniffi::export]
 impl FfiGroupPermissions {
     pub fn policy_type(&self) -> Result<GroupPermissions, GenericError> {
-        Ok(self.inner.preconfigured_policy()?.into())
+        if let Ok(preconfigured_policy) = self.inner.preconfigured_policy() {
+            Ok(preconfigured_policy.into())
+        } else {
+            Ok(GroupPermissions::CustomPolicy)
+        }
     }
 }
 
@@ -1575,7 +1601,7 @@ mod tests {
             .unwrap();
         assert_eq!(bo_messages2.len(), second_msg_check);
 
-        // TODO: message_callbacks should eventually come through here, why does this 
+        // TODO: message_callbacks should eventually come through here, why does this
         // not work?
         // tokio::time::sleep(tokio::time::Duration::from_millis(10000)).await;
         // assert_eq!(message_callbacks.message_count(), second_msg_check as u32);
@@ -1698,15 +1724,11 @@ mod tests {
             .stream(Box::new(stream_callback.clone()))
             .await
             .unwrap();
+        tokio::time::sleep(tokio::time::Duration::from_millis(250)).await;
 
-        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
         group.send("hello".as_bytes().to_vec()).await.unwrap();
-        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
         group.send("goodbye".as_bytes().to_vec()).await.unwrap();
-        // Because of the event loop, I need to make the test give control
-        // back to the stream before it can process each message. Using sleep to do that.
-        // I think this will work fine in practice
-        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+        tokio::time::sleep(tokio::time::Duration::from_millis(250)).await;
         assert_eq!(stream_callback.message_count(), 2);
 
         stream_closer.end();
@@ -1824,7 +1846,6 @@ mod tests {
 
     // TODO: Test current fails 50% of the time with db locking messages
     #[tokio::test(flavor = "multi_thread", worker_threads = 5)]
-    #[ignore]
     async fn test_stream_groups_gets_callback_when_streaming_messages() {
         let alix = new_test_client().await;
         let bo = new_test_client().await;
@@ -1860,7 +1881,7 @@ mod tests {
 
         alix_group.send("hello1".as_bytes().to_vec()).await.unwrap();
         tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
-        
+
         assert_eq!(group_callbacks.message_count(), 1);
         assert_eq!(message_callbacks.message_count(), 1);
 

bindings_node/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @xmtp/mls-client-bindings-node
 
+## 0.0.7
+
+- Improved streaming welcomes
+- Improved DB retries
+- Changed encoding of the MLS database to `bincode` for performance
+- Added `find_inbox_id_by_address` to client
+- Added `find_group_by_id` and `find_message_by_id` to conversations
+
 ## 0.0.6
 
 - Fixed some group syncing issues

bindings_node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xmtp/mls-client-bindings-node",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "repository": {
     "type": "git",
     "url": "git+https://[email protected]/xmtp/libxmtp.git",

bindings_node/src/conversations.rs

@@ -87,6 +87,34 @@ impl NapiConversations {
     Ok(out)
   }
 
+  #[napi]
+  pub fn find_group_by_id(&self, group_id: String) -> Result<NapiGroup> {
+    let group_id = hex::decode(group_id).map_err(|e| Error::from_reason(format!("{}", e)))?;
+
+    let group = self
+      .inner_client
+      .group(group_id)
+      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+
+    Ok(NapiGroup::new(
+      self.inner_client.clone(),
+      group.group_id,
+      group.created_at_ns,
+    ))
+  }
+
+  #[napi]
+  pub fn find_message_by_id(&self, message_id: String) -> Result<NapiMessage> {
+    let message_id = hex::decode(message_id).map_err(|e| Error::from_reason(format!("{}", e)))?;
+
+    let message = self
+      .inner_client
+      .message(message_id)
+      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+
+    Ok(NapiMessage::from(message))
+  }
+
   #[napi]
   pub async fn process_streamed_welcome_message(
     &self,

bindings_node/src/mls_client.rs

@@ -269,4 +269,15 @@ impl NapiClient {
 
     Ok(())
   }
+
+  #[napi]
+  pub async fn find_inbox_id_by_address(&self, address: String) -> Result<Option<String>> {
+    let inbox_id = self
+      .inner_client
+      .find_inbox_id_from_address(address)
+      .await
+      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+
+    Ok(inbox_id)
+  }
 }

bindings_node/test/Client.test.ts

@@ -24,4 +24,11 @@ describe('Client', () => {
     const canMessage = await client.canMessage([user.account.address])
     expect(canMessage).toEqual({ [user.account.address.toLowerCase()]: true })
   })
+
+  it('should find an inbox ID from an address', async () => {
+    const user = createUser()
+    const client = await createRegisteredClient(user)
+    const inboxId = await client.findInboxIdByAddress(user.account.address)
+    expect(inboxId).toBe(client.inboxId())
+  })
 })

bindings_node/test/Conversations.test.ts

@@ -55,6 +55,37 @@ describe('Conversations', () => {
     expect(group2[0].id).toBe(group.id)
   })
 
+  it('should find a group by ID', async () => {
+    const user1 = createUser()
+    const user2 = createUser()
+    const client1 = await createRegisteredClient(user1)
+    const client2 = await createRegisteredClient(user2)
+    const group = await client1
+      .conversations()
+      .createGroup([user2.account.address])
+    expect(group).toBeDefined()
+    expect(group.id()).toBeDefined()
+    const foundGroup = client1.conversations().findGroupById(group.id())
+    expect(foundGroup).toBeDefined()
+    expect(foundGroup!.id()).toBe(group.id())
+  })
+
+  it('should find a message by ID', async () => {
+    const user1 = createUser()
+    const user2 = createUser()
+    const client1 = await createRegisteredClient(user1)
+    await createRegisteredClient(user2)
+    const group = await client1
+      .conversations()
+      .createGroup([user2.account.address])
+    const messageId = await group.send(encodeTextMessage('gm!'))
+    expect(messageId).toBeDefined()
+
+    const message = client1.conversations().findMessageById(messageId)
+    expect(message).toBeDefined()
+    expect(message!.id).toBe(messageId)
+  })
+
   it('should create a new group with options', async () => {
     const user1 = createUser()
     const user2 = createUser()

xmtp_mls/src/api/mod.rs

@@ -3,7 +3,10 @@ pub mod mls;
 #[cfg(test)]
 pub mod test_utils;
 
-use crate::{retry::Retry, XmtpApi};
+use crate::{
+    retry::{Retry, RetryableError},
+    XmtpApi,
+};
 use thiserror::Error;
 use xmtp_id::associations::DeserializationError as AssociationDeserializationError;
 use xmtp_proto::api_client::Error as ApiError;
@@ -19,6 +22,12 @@ pub enum WrappedApiError {
     AssociationDeserialization(#[from] AssociationDeserializationError),
 }
 
+impl RetryableError for WrappedApiError {
+    fn is_retryable(&self) -> bool {
+        matches!(self, Self::Api(_))
+    }
+}
+
 #[derive(Debug)]
 pub struct ApiClientWrapper<ApiClient> {
     api_client: ApiClient,