initial version of codelab for image gen

codelabs/image_generation_basic/.gitignore

@@ -0,0 +1,15 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

codelabs/image_generation_basic/android/finish/app/.gitignore

@@ -0,0 +1 @@
+/build

codelabs/image_generation_basic/android/finish/app/build.gradle

@@ -0,0 +1,52 @@
+plugins {
+    id 'com.android.application'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'com.google.mediapipe.examples.imagegeneration'
+    compileSdk 33
+
+    defaultConfig {
+        applicationId "com.google.mediapipe.examples.imagegeneration"
+        minSdk 24
+        targetSdk 33
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    buildFeatures {
+        viewBinding true
+    }
+}
+
+// Downloads the TFLite and Task files used for plugins
+project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets'
+
+dependencies {
+
+    implementation 'androidx.core:core-ktx:1.8.0'
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'com.google.android.material:material:1.5.0'
+    implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
+    implementation 'androidx.activity:activity-ktx:1.7.2'
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.5'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
+    implementation 'com.google.mediapipe:tasks-vision-image-generator:0.10.5.2'
+}

codelabs/image_generation_basic/android/finish/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

codelabs/image_generation_basic/android/finish/app/src/main/AndroidManifest.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools" >
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:largeHeap="true"
+        android:theme="@style/Theme.ImageGeneration"
+        tools:targetApi="31" >
+        <!-- Some devices, like the Pixel 6, may need to actively declare the use of native libs -->
+        <uses-native-library android:name="libOpenCL.so" android:required="false" />
+        <uses-native-library android:name="libOpenCL-car.so" android:required="false"/>
+        <uses-native-library android:name="libOpenCL-pixel.so" android:required="false" />
+
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:keepScreenOn="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

codelabs/image_generation_basic/android/finish/app/src/main/java/com/google/mediapipe/examples/imagegeneration/ImageGenerationHelper.kt

@@ -0,0 +1,74 @@
+package com.google.mediapipe.examples.imagegeneration
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.Canvas
+import android.graphics.Color
+import android.graphics.Paint
+import com.google.mediapipe.framework.image.BitmapExtractor
+import com.google.mediapipe.framework.image.MPImage
+import com.google.mediapipe.tasks.core.BaseOptions
+import com.google.mediapipe.tasks.core.Delegate
+import com.google.mediapipe.tasks.vision.imagegenerator.ImageGenerator
+import com.google.mediapipe.tasks.vision.imagegenerator.ImageGenerator.ConditionOptions
+import com.google.mediapipe.tasks.vision.imagegenerator.ImageGenerator.ConditionOptions.ConditionType
+import com.google.mediapipe.tasks.vision.imagegenerator.ImageGenerator.ConditionOptions.EdgeConditionOptions
+import com.google.mediapipe.tasks.vision.imagegenerator.ImageGenerator.ConditionOptions.FaceConditionOptions
+import com.google.mediapipe.tasks.vision.imagegenerator.ImageGenerator.ImageGeneratorOptions
+
+
+class ImageGenerationHelper(
+    val context: Context
+) {
+
+    lateinit var imageGenerator: ImageGenerator
+
+    // Setup image generation model with output size, iteration
+    fun initializeImageGenerator(modelPath: String) {
+        val options = ImageGeneratorOptions.builder()
+            .setImageGeneratorModelDirectory(modelPath)
+            .build()
+
+        imageGenerator = ImageGenerator.createFromOptions(context, options)
+    }
+
+    // Set input prompt, iteration, seed
+    fun setInput(prompt: String, iteration: Int, seed: Int) {
+        imageGenerator.setInputs(prompt, iteration, seed)
+    }
+
+
+    fun generate(prompt: String, iteration: Int, seed: Int): Bitmap {
+        val result = imageGenerator.generate(prompt, iteration, seed)
+        val bitmap = BitmapExtractor.extract(result?.generatedImage())
+        return bitmap
+    }
+
+    fun execute(showResult: Boolean): Bitmap {
+        // execute image generation model
+        val result = imageGenerator.execute(showResult)
+
+        if (result == null || result.generatedImage() == null) {
+            return Bitmap.createBitmap(512, 512, Bitmap.Config.ARGB_8888)
+                .apply {
+                    val canvas = Canvas(this)
+                    val paint = Paint()
+                    paint.color = Color.WHITE
+                    canvas.drawPaint(paint)
+                }
+        }
+
+        val bitmap =
+            BitmapExtractor.extract(result.generatedImage())
+
+        return bitmap
+    }
+
+    fun close() {
+        try {
+            imageGenerator.close()
+        } catch (e: Exception) {
+            e.printStackTrace()
+        }
+    }
+}

codelabs/image_generation_basic/android/finish/app/src/main/java/com/google/mediapipe/examples/imagegeneration/MainActivity.kt

@@ -0,0 +1,175 @@
+package com.google.mediapipe.examples.imagegeneration
+
+import android.os.Bundle
+import android.view.View
+import android.view.inputmethod.InputMethodManager
+import android.widget.Toast
+import androidx.activity.viewModels
+import androidx.appcompat.app.AppCompatActivity
+import androidx.core.widget.doOnTextChanged
+import androidx.lifecycle.Lifecycle
+import androidx.lifecycle.lifecycleScope
+import androidx.lifecycle.repeatOnLifecycle
+import com.google.mediapipe.examples.imagegeneration.databinding.ActivityMainBinding
+import kotlinx.coroutines.launch
+import java.util.*
+
+class MainActivity : AppCompatActivity() {
+    companion object {
+        private const val DEFAULT_DISPLAY_ITERATION = 5
+        private const val DEFAULT_ITERATION = 20
+        private const val DEFAULT_SEED = 0
+        private val DEFAULT_PROMPT = R.string.default_prompt_diffusion
+        private val DEFAULT_DISPLAY_OPTIONS = R.id.radio_final // FINAL
+    }
+
+    private lateinit var binding: ActivityMainBinding
+    private val viewModel: DiffusionViewModel by viewModels()
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        binding = ActivityMainBinding.inflate(layoutInflater)
+        setContentView(binding.root)
+        viewModel.createImageGenerationHelper(this)
+
+        lifecycleScope.launch {
+            repeatOnLifecycle(Lifecycle.State.STARTED) {
+                // Update UI
+                viewModel.uiState.collect { uiState ->
+                    binding.llInitializeSection.visibility =
+                        if (uiState.initialized) android.view.View.GONE else android.view.View.VISIBLE
+                    binding.llGenerateSection.visibility =
+                        if (uiState.initialized) android.view.View.VISIBLE else android.view.View.GONE
+                    binding.llDisplayIteration.visibility =
+                        if (uiState.displayOptions == DisplayOptions.ITERATION) android.view.View.VISIBLE else android.view.View.GONE
+
+                    // Button initialize is enabled when (the display option is final or iteration and display iteration is not null) and is not initializing
+                    binding.btnInitialize.isEnabled =
+                        (uiState.displayOptions == DisplayOptions.FINAL || (uiState.displayOptions == DisplayOptions.ITERATION && uiState.displayIteration != null)) && !uiState.isInitializing
+
+                    if (uiState.isGenerating) {
+                        binding.btnGenerate.isEnabled = false
+                        binding.btnGenerate.text = uiState.generatingMessage
+                        binding.tvDisclaimer.visibility = View.VISIBLE
+                    } else {
+                        binding.btnGenerate.text = "Generate"
+                        if (uiState.initialized) {
+                            binding.btnGenerate.isEnabled =
+                                uiState.prompt.isNotEmpty() && uiState.iteration != null && uiState.seed != null
+                        } else {
+                            binding.btnGenerate.isEnabled = false
+                        }
+                    }
+                    binding.imgOutput.setImageBitmap(uiState.outputBitmap)
+                    showError(uiState.error)
+                    showGenerateTime(uiState.generateTime)
+                    showInitializedTime(uiState.initializedTime)
+                }
+            }
+        }
+
+        handleListener()
+        setDefaultValue()
+    }
+
+    private fun handleListener() {
+        binding.btnInitialize.setOnClickListener {
+            viewModel.initializeImageGenerator()
+            closeSoftKeyboard()
+        }
+        binding.btnGenerate.setOnClickListener {
+            viewModel.generateImage()
+            closeSoftKeyboard()
+        }
+        binding.btnSeedRandom.setOnClickListener {
+            randomSeed()
+            closeSoftKeyboard()
+        }
+
+        binding.radioDisplayOptions.setOnCheckedChangeListener { group, checkedId ->
+            when (checkedId) {
+                R.id.radio_iteration -> {
+                    viewModel.updateDisplayOptions(DisplayOptions.ITERATION)
+                }
+
+                R.id.radio_final -> {
+                    viewModel.updateDisplayOptions(DisplayOptions.FINAL)
+                }
+            }
+        }
+
+        binding.edtDisplayIteration.doOnTextChanged { text, _, _, _ ->
+            viewModel.updateDisplayIteration(text.toString().toIntOrNull())
+        }
+        binding.edtPrompt.doOnTextChanged { text, _, _, _ ->
+            viewModel.updatePrompt(text.toString())
+        }
+        binding.edtIterations.doOnTextChanged { text, _, _, _ ->
+            viewModel.updateIteration(text.toString().toIntOrNull())
+        }
+        binding.edtSeed.doOnTextChanged { text, _, _, _ ->
+            viewModel.updateSeed(text.toString().toIntOrNull())
+        }
+    }
+
+    private fun showError(message: String?) {
+        if (message.isNullOrEmpty()) return
+        runOnUiThread {
+            Toast.makeText(this, message, Toast.LENGTH_SHORT).show()
+        }
+        // prevent showing error message twice
+        viewModel.clearError()
+    }
+
+    private fun showGenerateTime(time: Long?) {
+        if (time == null) return
+        runOnUiThread {
+            Toast.makeText(
+                this,
+                "Generation time: ${time / 1000.0} seconds",
+                Toast.LENGTH_SHORT
+            ).show()
+        }
+        // prevent showing generate time twice
+        viewModel.clearGenerateTime()
+    }
+
+    private fun showInitializedTime(time: Long?) {
+        if (time == null) return
+        runOnUiThread {
+            Toast.makeText(
+                this,
+                "Initialized time: ${time / 1000.0} seconds",
+                Toast.LENGTH_SHORT
+            ).show()
+        }
+        // prevent showing initialized time twice
+        viewModel.clearInitializedTime()
+    }
+
+    private fun closeSoftKeyboard() {
+        val imm = getSystemService(INPUT_METHOD_SERVICE) as InputMethodManager
+        imm.hideSoftInputFromWindow(binding.root.windowToken, 0)
+    }
+
+    private fun setDefaultValue() {
+        with(binding) {
+            edtPrompt.setText(getString(DEFAULT_PROMPT))
+            edtIterations.setText(DEFAULT_ITERATION.toString())
+            edtSeed.setText(DEFAULT_SEED.toString())
+            radioDisplayOptions.check(DEFAULT_DISPLAY_OPTIONS)
+            edtDisplayIteration.setText(DEFAULT_DISPLAY_ITERATION.toString())
+        }
+    }
+
+    private fun randomSeed() {
+        val random = Random()
+        val seed = Math.abs(random.nextInt())
+        binding.edtSeed.setText(seed.toString())
+    }
+
+    override fun onDestroy() {
+        super.onDestroy()
+        viewModel.closeGenerator()
+    }
+}