feat: implement livekit (#47)

* feat: implement livekit

* clean

* wip (windows)

* add voicechat and android support

* doc

* clippy and format

* fix: set correct linker on android, fail on error (#46)

* chore: remove unused asset (#48)

* remove unused `use`

* update CI

---------

Co-authored-by: Mateo Miccino <[email protected]>

.github/workflows/ci.yml

@@ -96,6 +96,9 @@ jobs:
       - name: install ffmpeg deps (linux)
         run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
         if: runner.os == 'linux'
+      - name: install livekit deps (linux)
+        run: sudo apt update -y; sudo apt install -y libssl-dev libx11-dev libgl1-mesa-dev libxext-dev
+        if: runner.os == 'linux'
 
       - name: cargo xtask install
         working-directory: rust/xtask
@@ -157,6 +160,9 @@ jobs:
       - name: install ffmpeg deps (linux)
         run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
         if: runner.os == 'linux'
+      - name: install livekit deps (linux)
+        run: sudo apt update -y; sudo apt install -y libssl-dev libx11-dev libgl1-mesa-dev libxext-dev
+        if: runner.os == 'linux'
 
       # => MacOS 
       - name: install ffmpeg deps (macOs)

godot/default_bus_layout.tres

@@ -1,4 +1,15 @@
-[gd_resource type="AudioBusLayout" format=3 uid="uid://ctjmxlxnfk873"]
+[gd_resource type="AudioBusLayout" load_steps=2 format=3 uid="uid://ctjmxlxnfk873"]
+
+[sub_resource type="AudioEffectCapture" id="AudioEffectCapture_umb32"]
+resource_name = "Capture"
 
 [resource]
 bus/0/volume_db = 0.0672607
+bus/1/name = &"Capture"
+bus/1/solo = false
+bus/1/mute = true
+bus/1/bypass_fx = false
+bus/1/volume_db = 0.0
+bus/1/send = &"Master"
+bus/1/effect/0/effect = SubResource("AudioEffectCapture_umb32")
+bus/1/effect/0/enabled = true

godot/export_presets.cfg

@@ -360,7 +360,7 @@ permissions/receive_boot_completed=false
 permissions/receive_mms=false
 permissions/receive_sms=false
 permissions/receive_wap_push=false
-permissions/record_audio=false
+permissions/record_audio=true
 permissions/reorder_tasks=false
 permissions/restart_packages=false
 permissions/send_respond_via_message=false

godot/project.godot

@@ -16,6 +16,11 @@ run/main_scene="res://src/main.tscn"
 config/features=PackedStringArray("4.1")
 config/icon="res://decentraland_logo.png"
 
+[audio]
+
+driver/enable_input=true
+driver/mix_rate=48000
+
 [autoload]
 
 Global="*res://src/global.gd"

godot/src/decentraland_components/avatar.gd

@@ -404,3 +404,26 @@ func set_running():
 func set_idle():
 	if animation_player.current_animation != "Idle":
 		animation_player.play("Idle")
+
+
+var audio_stream_player: AudioStreamPlayer = null
+var audio_stream_player_gen: AudioStreamGenerator = null
+
+
+func spawn_voice_channel(sample_rate, num_channels, samples_per_channel):
+	printt("init voice chat ", sample_rate, num_channels, samples_per_channel)
+	audio_stream_player = AudioStreamPlayer.new()
+	audio_stream_player_gen = AudioStreamGenerator.new()
+
+	audio_stream_player.set_stream(audio_stream_player_gen)
+	audio_stream_player_gen.mix_rate = sample_rate
+	add_child(audio_stream_player)
+	audio_stream_player.play()
+
+
+func push_voice_frame(frame):
+#	print("voice chat ", frame)
+	if not audio_stream_player.playing:
+		audio_stream_player.play()
+
+	audio_stream_player.get_stream_playback().push_buffer(frame)

godot/src/main.gd

@@ -6,14 +6,19 @@ func _ready():
 
 
 func start():
-	var resolution_manager = ResolutionManager.new()
-	resolution_manager.refresh_window_options()
-
-	resolution_manager.change_window_size(get_window(), get_viewport(), Global.config.window_size)
-	resolution_manager.change_resolution(get_window(), get_viewport(), Global.config.resolution)
-	resolution_manager.change_ui_scale(get_window(), Global.config.ui_scale)
-	resolution_manager.center_window(get_window())
-	resolution_manager.apply_fps_limit()
+	if not OS.has_feature("Server"):
+		print("Running from platform")
+		var resolution_manager = ResolutionManager.new()
+		resolution_manager.refresh_window_options()
+		resolution_manager.change_window_size(
+			get_window(), get_viewport(), Global.config.window_size
+		)
+		resolution_manager.change_resolution(get_window(), get_viewport(), Global.config.resolution)
+		resolution_manager.change_ui_scale(get_window(), Global.config.ui_scale)
+		resolution_manager.center_window(get_window())
+		resolution_manager.apply_fps_limit()
+	else:
+		print("Running from Server")
 
 	if Global.is_mobile:
 		var screen_size = DisplayServer.screen_get_size()

godot/src/ui/components/advance_settings/advance_settings.tscn

@@ -97,7 +97,7 @@ layout_mode = 2
 size_flags_horizontal = 3
 tooltip_text = "Select scene to load"
 focus_mode = 0
-item_count = 7
+item_count = 8
 popup/item_0/text = "mannakia.dcl.eth"
 popup/item_0/id = 0
 popup/item_1/text = "http://127.0.0.1:8000"
@@ -112,6 +112,8 @@ popup/item_5/text = "https://sdk-team-cdn.decentraland.org/ipfs/streaming-world-
 popup/item_5/id = 5
 popup/item_6/text = "https://peer.decentraland.org"
 popup/item_6/id = 6
+popup/item_7/text = "shibu.dcl.eth"
+popup/item_7/id = 7
 
 [node name="HSeparator4" type="HSeparator" parent="VBoxContainer/ColorRect_Background/HBoxContainer/VBoxContainer_General"]
 layout_mode = 2

godot/src/ui/explorer.tscn

@@ -1,4 +1,4 @@
-[gd_scene load_steps=16 format=3 uid="uid://deq5v42fmh0y7"]
+[gd_scene load_steps=17 format=3 uid="uid://deq5v42fmh0y7"]
 
 [ext_resource type="Script" path="res://src/ui/explorer.gd" id="1_5n8xk"]
 [ext_resource type="Texture2D" uid="uid://by286h7kaeqr3" path="res://assets/empty-scenes/FloorBaseGrass_01/Floor_Grass01.png.png" id="2_7jksa"]
@@ -8,6 +8,7 @@
 [ext_resource type="PackedScene" uid="uid://bl6h58asl377" path="res://src/ui/components/chat/chat.tscn" id="9_4ktln"]
 [ext_resource type="PackedScene" uid="uid://c6a0rjrc13kel" path="res://src/ui/components/line_edit_command/line_edit_command.tscn" id="9_5u55i"]
 [ext_resource type="PackedScene" uid="uid://dmr0fcamx7t56" path="res://src/mobile/joystick/virtual_joystick.tscn" id="9_lxw33"]
+[ext_resource type="PackedScene" uid="uid://wgrmvh6h51w3" path="res://src/ui/voice_chat.tscn" id="10_l3sp6"]
 [ext_resource type="PackedScene" uid="uid://mc4jrvowdpxp" path="res://src/ui/components/pointer_tooltip/pointer_tooltip.tscn" id="11_qjs00"]
 
 [sub_resource type="ConcavePolygonShape3D" id="ConcavePolygonShape3D_xs7js"]
@@ -26,7 +27,7 @@ texture_filter = 0
 
 [sub_resource type="Theme" id="Theme_1ufu0"]
 
-[sub_resource type="ButtonGroup" id="ButtonGroup_3vqs6"]
+[sub_resource type="ButtonGroup" id="ButtonGroup_gkn7s"]
 resource_name = "Tabs"
 
 [node name="explorer" type="Node3D"]
@@ -171,12 +172,25 @@ layout_mode = 2
 [node name="Control_Menu" parent="UI" instance=ExtResource("5_mso44")]
 visible = false
 layout_mode = 1
-group = SubResource("ButtonGroup_3vqs6")
+group = SubResource("ButtonGroup_gkn7s")
 
 [node name="Timer_BroadcastPosition" type="Timer" parent="."]
 wait_time = 0.1
 autostart = true
 
+[node name="voice_chat" parent="." instance=ExtResource("10_l3sp6")]
+anchors_preset = 7
+anchor_left = 0.5
+anchor_top = 1.0
+anchor_right = 0.5
+anchor_bottom = 1.0
+offset_left = -100.0
+offset_top = -48.0
+offset_right = 100.0
+offset_bottom = -8.0
+grow_horizontal = 2
+grow_vertical = 0
+
 [connection signal="gui_input" from="UI" to="." method="_on_ui_gui_input"]
 [connection signal="timeout" from="UI/Timer_FPSLabel" to="." method="_on_timer_timeout"]
 [connection signal="request_open_map" from="UI/Control_Minimap" to="." method="_on_control_minimap_request_open_map"]

godot/src/ui/voice_chat.gd

@@ -0,0 +1,44 @@
+extends Control
+
+var recording: bool = false
+var _effect_capture: AudioEffectCapture
+var _prev_frame_recording = false
+
+@onready var button_record = $Button
+@onready var audio_stream_player = $AudioStreamPlayer
+
+
+func _ready():
+	var devices = AudioServer.get_input_device_list()
+	print(AudioServer.get_input_device_list())
+#	AudioServer.input_device = devices[1]
+
+	var idx = AudioServer.get_bus_index("Capture")
+	_effect_capture = AudioServer.get_bus_effect(idx, 0)
+
+	audio_stream_player.stream = AudioStreamMicrophone.new()
+	audio_stream_player.bus = "Capture"
+
+
+func _process(delta):
+	if recording:
+		var stereo_data: PackedVector2Array = _effect_capture.get_buffer(
+			_effect_capture.get_frames_available()
+		)
+		if stereo_data.size() > 0:
+			Global.comms.broadcast_voice(stereo_data)
+
+	_prev_frame_recording = recording
+
+
+func _on_button_pressed():
+	if recording:
+		button_record.text = "Enable mic"
+		recording = false
+		_effect_capture.clear_buffer()
+		audio_stream_player.stop()
+	else:
+		button_record.text = "Stop mic"
+		recording = true
+		audio_stream_player.play()
+		_effect_capture.clear_buffer()

godot/src/ui/voice_chat.tscn

@@ -0,0 +1,21 @@
+[gd_scene load_steps=2 format=3 uid="uid://wgrmvh6h51w3"]
+
+[ext_resource type="Script" path="res://src/ui/voice_chat.gd" id="1_qu3n3"]
+
+[node name="voice_chat" type="Control"]
+layout_mode = 3
+anchors_preset = 0
+script = ExtResource("1_qu3n3")
+
+[node name="Button" type="Button" parent="."]
+layout_mode = 1
+anchors_preset = 15
+anchor_right = 1.0
+anchor_bottom = 1.0
+grow_horizontal = 2
+grow_vertical = 2
+text = "Enable Mic"
+
+[node name="AudioStreamPlayer" type="AudioStreamPlayer" parent="."]
+
+[connection signal="pressed" from="Button" to="." method="_on_button_pressed"]

rust/decentraland-godot-lib/.cargo/config.toml

@@ -0,0 +1,11 @@
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-C", "target-feature=+crt-static"]
+
+[target.aarch64-pc-windows-msvc]
+rustflags = ["-C", "target-feature=+crt-static"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-args=-ObjC"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-args=-ObjC"]

rust/decentraland-godot-lib/Cargo.toml

@@ -48,5 +48,12 @@ bytes = "1.4.0"
 tokio-tungstenite = "*"
 futures-util = "*"
 
+livekit = { git = "https://github.com/robtfm/client-sdk-rust", branch="h264-false", features=["rustls-tls-webpki-roots"] }
+
+[target.'cfg(target_os = "android")'.dependencies]
+jni = { version = "0.21.1", features = ["invocation"] }
+paranoid-android = "0.2.1"
+
 [build-dependencies]
+webrtc-sys-build = "0.2.0"
 prost-build = "0.11.8"

rust/decentraland-godot-lib/android-build.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 # Tested with NDK 25.2.9519653
 ANDROID_NDK=~/Android/Sdk/ndk/25.2.9519653
 export FFMPEG_DIR=~/Documents/github/ffmpeg-kit/prebuilt/android-arm64/ffmpeg
@@ -11,17 +13,24 @@ export TARGET_AR=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar
 export RUSTY_V8_MIRROR=https://github.com/leanmendoza/rusty_v8/releases/download
 export CARGO_FFMPEG_SYS_DISABLE_SIZE_T_IS_USIZE=1
 
+export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android33-clang
+
 # Store the original content of Cargo.toml
 cargo_file_path="Cargo.toml"
 original_content=$(cat $cargo_file_path)
 ffmpeg_dep='ffmpeg-next = { git = "https://github.com/decentraland/rust-ffmpeg/", branch="audioline-and-mobile-fix" }'
 ffmpeg_dep_android='ffmpeg-next = { git = "https://github.com/decentraland/rust-ffmpeg/", branch="audioline-and-mobile-fix", features=["fix_usize_size_t"] }'
 sed -i "s|$ffmpeg_dep|$ffmpeg_dep_android|g" "$cargo_file_path"
 
-(GN_ARGS=use_custom_libcxx=false RUST_BACKTRACE=full cargo build --target aarch64-linux-android -vv --verbose --release) || true
+(ANDROID_NDK_HOME=$ANDROID_NDK GN_ARGS=use_custom_libcxx=false RUST_BACKTRACE=full cargo build --target aarch64-linux-android -vv --verbose --release) || true
 
 # Revert Cargo.toml back to its original content
 echo "$original_content" > $cargo_file_path
 
-mkdir ../../godot/lib/android/ || true
+mkdir -p ../../godot/lib/android/
 cp target/aarch64-linux-android/release/libdecentraland_godot_lib.so ../../godot/lib/android/libdecentraland_godot_lib.so
+cp target/aarch64-linux-android/release/libdecentraland_godot_lib.so ../../godot/android/build/libs/debug/arm64-v8a/libdecentraland_godot_lib.so
+
+# Dependencies 
+# - from web-rtc: libwebrtc.jar 
+# - from ffmpeg: libavcodec, libavfilter, libavdevice, libavformat, libavutil, libswresample, libswscale

rust/decentraland-godot-lib/build.rs

@@ -316,6 +316,11 @@ fn main() -> io::Result<()> {
 
     std::env::set_var("PROTOC", protoc_path);
     prost_build::compile_protos(&proto_files, &["src/dcl/components/proto/"])?;
+
+    if env::var("CARGO_CFG_TARGET_OS").unwrap() == "android" {
+        webrtc_sys_build::configure_jni_symbols().unwrap();
+    }
+
     Ok(())
 }
 

rust/decentraland-godot-lib/builds.md

@@ -40,4 +40,18 @@ This only buils for arm64. `fribidi` and `libass` should be compiled but I got e
 
 1. Once the build is done, you need to modify the `android-build.sh` the line:
     - `export FFMPEG_DIR=$FFMPEG_FOLDER/prebuilt/apple-ios-arm64/ffmpeg` 
-    - Replace $FFMPEG_FOLDER with your path where you clone ffmpeg_kit
+    - Replace $FFMPEG_FOLDER with your path where you clone ffmpeg_kit
+
+# Run
+## Android
+1. Open the editor and go to `Project` and click `Install Android Build Template` (only requires onces and you need to install export templates)
+2. Add the next line in `godot/android/build/src/com/godot/game/GodotApp.java` after `public class GodotApp extends FullScreenGodotApp {`:
+```
+	// This block calls the JNI_OnLoad, needed for livekit 
+	static {
+		System.loadLibrary("decentraland_godot_lib");
+	}
+```
+3. Ensure the dependencies are copied: 
+- from web-rtc: libwebrtc.jar 
+- from ffmpeg: libavcodec, libavfilter, libavdevice, libavformat, libavutil, libswresample, libswscale

rust/decentraland-godot-lib/src/avatars/avatar_scene.rs

@@ -105,8 +105,8 @@ impl AvatarScene {
 }
 
 impl AvatarScene {
-    const FROM_ENTITY_ID: u16 = 10;
-    const MAX_ENTITY_ID: u16 = 200;
+    const FROM_ENTITY_ID: u16 = 32;
+    const MAX_ENTITY_ID: u16 = 256;
     // const AVATAR_COMPONENTS: &[SceneComponentId] = &[SceneComponentId::AVATAR_ATTACH];
 
     // This function is not optimized, it will iterate over all the entities but this happens only when add an player
@@ -192,4 +192,44 @@ impl AvatarScene {
             &[profile.to_godot_dictionary(base_url).to_variant()],
         );
     }
+
+    pub fn spawn_voice_channel(
+        &mut self,
+        alias: u32,
+        sample_rate: u32,
+        num_channels: u32,
+        samples_per_channel: u32,
+    ) {
+        let entity_id = if let Some(entity_id) = self.avatar_entity.get(&alias) {
+            *entity_id
+        } else {
+            // TODO: handle this condition
+            return;
+        };
+
+        let (sample_rate, num_channels, samples_per_channel) = (
+            sample_rate.to_variant(),
+            num_channels.to_variant(),
+            samples_per_channel.to_variant(),
+        );
+
+        self.avatar_godot_scene.get_mut(&entity_id).unwrap().call(
+            "spawn_voice_channel".into(),
+            &[sample_rate, num_channels, samples_per_channel],
+        );
+    }
+
+    pub fn push_voice_frame(&mut self, alias: u32, frame: PackedVector2Array) {
+        let entity_id = if let Some(entity_id) = self.avatar_entity.get(&alias) {
+            *entity_id
+        } else {
+            // TODO: handle this condition
+            return;
+        };
+
+        self.avatar_godot_scene
+            .get_mut(&entity_id)
+            .unwrap()
+            .call("push_voice_frame".into(), &[frame.to_variant()]);
+    }
 }