feat/virtual background mediapipe

README.md

@@ -110,6 +110,31 @@ android {
 
 If necessary, in the same `build.gradle` you will need to increase `minSdkVersion` of `defaultConfig` up to `23` (currently default Flutter generator set it to `16`).
 
+### Virtual Background (Work in Progress - Only Android)
+The Virtual Background feature allows users to set a background image for their video stream during video calls. This feature is powered by the WebRTC plugin and enables users to replace their real background with a custom image.
+
+#### How to Use
+
+- Enable virtual background
+
+```dart
+final ByteData data = await rootBundle.load(
+  Assets.images.virtualBackgroundSimple.path,
+);
+final Uint8List virtualBackgroundImage = data.buffer.asUint8List();
+
+rtc.Helper.enableVirtualBackground(
+  backgroundImage: virtualBackgroundImage,
+  thresholdConfidence: 0.6,
+);
+```
+
+- Disable virtual background
+
+```dart
+rtc.Helper.disableVirtualBackground();
+```
+
 ### Important reminder
 When you compile the release apk, you need to add the following operations,
 [Setup Proguard Rules](https://github.com/flutter-webrtc/flutter-webrtc/commit/d32dab13b5a0bed80dd9d0f98990f107b9b514f4)

android/build.gradle

@@ -11,6 +11,7 @@ buildscript {
     dependencies {
         classpath 'com.android.tools.build:gradle:7.4.2'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath 'de.undercouch:gradle-download-task:4.1.2'
     }
 }
 
@@ -23,6 +24,7 @@ rootProject.allprojects {
 
 apply plugin: 'com.android.library'
 apply plugin: 'kotlin-android'
+apply plugin: 'de.undercouch.download'
 
 android {
     if (project.android.hasProperty("namespace")) {
@@ -50,10 +52,24 @@ android {
     }
 }
 
+// import DownloadModels task
+project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets'
+project.ext.TEST_ASSETS_DIR = projectDir.toString() + '/src/androidTest/assets'
+
+// Download default models; if you wish to use your own models then
+// place them in the "assets" directory and comment out this line.
+apply from: 'download_models.gradle'
+
 dependencies {
-     implementation 'io.github.webrtc-sdk:android:114.5735.02'
+    // implementation 'io.github.webrtc-sdk:android:114.5735.02'
     implementation 'com.twilio:audioswitch:1.1.8'
-    implementation 'androidx.annotation:annotation:1.1.0'
+    implementation 'androidx.annotation:annotation:1.6.0'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
-    // implementation files('libwebrtc.aar')
+    implementation files('libwebrtc.aar')
+
+    // ML Kit
+    implementation 'com.google.mediapipe:tasks-vision:0.10.2'
+
+    implementation 'com.google.android.gms:play-services-tflite-java:16.1.0'
+    implementation 'com.google.android.gms:play-services-tflite-gpu:16.2.0'
 }

android/download_models.gradle

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *             http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//task downloadModelFile(type: Download) {
+//    src 'https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/1/deeplab_v3.tflite'
+//    dest project.ext.ASSET_DIR + '/deeplabv3.tflite'
+//    overwrite false
+//}
+
+//task downloadHairSegmenterModelFile(type: Download) {
+//    src 'https://storage.googleapis.com/mediapipe-models/image_segmenter/hair_segmenter/float32/1/hair_segmenter.tflite'
+//    dest project.ext.ASSET_DIR + '/hair_segmenter.tflite'
+//    overwrite false
+//}
+//
+//task downloadSelfieMulticlassSegmenterModelFile(type: Download) {
+//    src 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/latest/selfie_multiclass_256x256.tflite'
+//    dest project.ext.ASSET_DIR + '/selfie_multiclass.tflite'
+//    overwrite false
+//}
+
+task downloadSelfieSegmenterModelFile(type: Download) {
+    src 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite'
+    dest project.ext.ASSET_DIR + '/selfie_segmenter.tflite'
+    overwrite false
+}
+
+preBuild.dependsOn downloadSelfieSegmenterModelFile