ios hand landmarker add choose delegate (#364)

google-ai-edge · Apr 9, 2024 · 17b3662 · 17b3662
1 parent 57d4a16
commit 17b3662
Show file tree

Hide file tree

Showing 10 changed files with 144 additions and 24 deletions.
diff --git a/examples/hand_landmarker/ios/HandLandmarker/Base.lproj/Main.storyboard b/examples/hand_landmarker/ios/HandLandmarker/Base.lproj/Main.storyboard
@@ -1,10 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="21701" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="52l-qN-uG0">
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="32700.99.1234" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="52l-qN-uG0">
     <device id="retina6_12" orientation="portrait" appearance="light"/>
     <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="21679"/>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="22684"/>
         <capability name="Safe area layout guides" minToolsVersion="9.0"/>
         <capability name="System colors in document resources" minToolsVersion="11.0"/>
+        <capability name="UIMenu" message="Requires Xcode 11 or later." minToolsVersion="11.0" requiredIntegratedClassName="UICommandDiff"/>
         <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
     </dependencies>
     <scenes>
@@ -259,13 +261,52 @@
                                         <nil key="textColor"/>
                                         <nil key="highlightedColor"/>
                                     </label>
+                                    <view contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="BXC-Hu-zPM">
+                                        <rect key="frame" x="0.0" y="264.66666666666669" width="393" height="32"/>
+                                        <subviews>
+                                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="Delegate" textAlignment="natural" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="b1W-W1-j6b">
+                                                <rect key="frame" x="16" y="7.3333333333333144" width="58" height="17"/>
+                                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                                <nil key="textColor"/>
+                                                <nil key="highlightedColor"/>
+                                            </label>
+                                            <button opaque="NO" contentMode="scaleToFill" showsMenuAsPrimaryAction="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="q7u-sb-fuE">
+                                                <rect key="frame" x="353" y="0.0" width="24" height="32"/>
+                                                <color key="backgroundColor" red="0.0" green="0.49803921569999998" blue="0.5450980392" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                                <constraints>
+                                                    <constraint firstAttribute="height" constant="32" id="3sd-Jh-T4B"/>
+                                                </constraints>
+                                                <color key="tintColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                                <menu key="menu" id="snT-yZ-2bm">
+                                                    <children>
+                                                        <command title="item1" id="DRl-Ie-q88"/>
+                                                        <command title="item2" id="pnH-ZP-rML"/>
+                                                    </children>
+                                                </menu>
+                                                <buttonConfiguration key="configuration" style="plain"/>
+                                                <userDefinedRuntimeAttributes>
+                                                    <userDefinedRuntimeAttribute type="string" keyPath="layer.cornerRadius" value="8"/>
+                                                </userDefinedRuntimeAttributes>
+                                            </button>
+                                        </subviews>
+                                        <color key="backgroundColor" white="0.0" alpha="0.0" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <constraints>
+                                            <constraint firstItem="q7u-sb-fuE" firstAttribute="top" secondItem="BXC-Hu-zPM" secondAttribute="top" id="5t8-J4-YHi"/>
+                                            <constraint firstAttribute="trailing" secondItem="q7u-sb-fuE" secondAttribute="trailing" constant="16" id="O9a-WY-Rkw"/>
+                                            <constraint firstAttribute="bottom" secondItem="q7u-sb-fuE" secondAttribute="bottom" id="TGb-h0-JLt"/>
+                                            <constraint firstItem="b1W-W1-j6b" firstAttribute="centerY" secondItem="q7u-sb-fuE" secondAttribute="centerY" id="big-3J-Xyt"/>
+                                            <constraint firstItem="q7u-sb-fuE" firstAttribute="leading" relation="greaterThanOrEqual" secondItem="b1W-W1-j6b" secondAttribute="trailing" id="svf-8H-i83"/>
+                                            <constraint firstItem="b1W-W1-j6b" firstAttribute="leading" secondItem="BXC-Hu-zPM" secondAttribute="leading" constant="16" id="vvr-hh-dUC"/>
+                                        </constraints>
+                                    </view>
                                 </subviews>
                                 <color key="backgroundColor" white="0.0" alpha="0.0" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
                                 <constraints>
                                     <constraint firstItem="tad-jn-O5U" firstAttribute="top" secondItem="DDi-33-JAf" secondAttribute="bottom" constant="12" id="3f3-bg-DVN"/>
                                     <constraint firstItem="ME8-eS-OTB" firstAttribute="centerY" secondItem="tad-jn-O5U" secondAttribute="centerY" id="4FL-XQ-Him"/>
                                     <constraint firstItem="1fr-hY-4II" firstAttribute="trailing" secondItem="0nH-Zx-p69" secondAttribute="trailing" id="5T6-MD-Uko"/>
                                     <constraint firstAttribute="trailing" secondItem="0nH-Zx-p69" secondAttribute="trailing" constant="16" id="5xr-fC-LfQ"/>
+                                    <constraint firstItem="BXC-Hu-zPM" firstAttribute="top" secondItem="tad-jn-O5U" secondAttribute="bottom" constant="12" id="6Ke-IO-58S"/>
                                     <constraint firstItem="jCg-YX-6ga" firstAttribute="leading" secondItem="iaZ-s2-My0" secondAttribute="leading" id="6kr-Ui-TIb"/>
                                     <constraint firstItem="jCg-YX-6ga" firstAttribute="centerY" secondItem="tad-jn-O5U" secondAttribute="centerY" id="9dd-nl-UN1"/>
                                     <constraint firstItem="0nH-Zx-p69" firstAttribute="leading" secondItem="s05-yK-gXh" secondAttribute="trailing" constant="4" id="Bda-ye-Je2"/>
@@ -294,8 +335,10 @@
                                     <constraint firstItem="d2C-9O-82c" firstAttribute="top" secondItem="0Hl-aV-l7w" secondAttribute="bottom" constant="12" id="mcI-24-pk6"/>
                                     <constraint firstItem="iaZ-s2-My0" firstAttribute="centerY" secondItem="0nH-Zx-p69" secondAttribute="centerY" id="o81-hG-imF"/>
                                     <constraint firstItem="iaZ-s2-My0" firstAttribute="top" secondItem="d2C-9O-82c" secondAttribute="bottom" constant="26" id="pPe-bR-0qO"/>
+                                    <constraint firstItem="BXC-Hu-zPM" firstAttribute="leading" secondItem="AOI-Gh-SLK" secondAttribute="leading" id="rSn-Uj-5Py"/>
                                     <constraint firstItem="0Hl-aV-l7w" firstAttribute="top" secondItem="AOI-Gh-SLK" secondAttribute="top" id="rl8-eu-Q06"/>
                                     <constraint firstItem="tad-jn-O5U" firstAttribute="leading" secondItem="ME8-eS-OTB" secondAttribute="trailing" constant="4" id="txK-t9-38i"/>
+                                    <constraint firstAttribute="trailing" secondItem="BXC-Hu-zPM" secondAttribute="trailing" id="uAA-RP-lV7"/>
                                     <constraint firstItem="dzb-pb-2Oi" firstAttribute="leading" secondItem="iaZ-s2-My0" secondAttribute="leading" id="uPl-xe-HGE"/>
                                     <constraint firstItem="1fr-hY-4II" firstAttribute="centerY" secondItem="06h-cY-2bs" secondAttribute="centerY" id="whL-o0-qmm"/>
                                     <constraint firstAttribute="trailing" secondItem="tZo-oH-ToE" secondAttribute="trailing" id="xSe-Ss-8WB"/>
@@ -315,6 +358,7 @@
                     </view>
                     <navigationItem key="navigationItem" id="BIS-1q-gQH"/>
                     <connections>
+                        <outlet property="chooseDelegateButton" destination="q7u-sb-fuE" id="Bm1-LB-AQx"/>
                         <outlet property="inferenceTimeLabel" destination="cRO-ho-FXh" id="E3e-km-74r"/>
                         <outlet property="inferenceTimeNameLabel" destination="d2C-9O-82c" id="O7p-Pr-lsE"/>
                         <outlet property="minHandDetectionConfidenceStepper" destination="0nH-Zx-p69" id="UPk-Tc-SO0"/>

diff --git a/examples/hand_landmarker/ios/HandLandmarker/Configurations/DefaultConstants.swift b/examples/hand_landmarker/ios/HandLandmarker/Configurations/DefaultConstants.swift
@@ -14,6 +14,7 @@
 
 import Foundation
 import UIKit
+import MediaPipeTasksVision
 
 // MARK: Define default constants
 struct DefaultConstants {
@@ -29,4 +30,40 @@ struct DefaultConstants {
   static var minHandPresenceConfidence: Float = 0.5
   static var minTrackingConfidence: Float = 0.5
   static let modelPath: String? = Bundle.main.path(forResource: "hand_landmarker", ofType: "task")
+  static let delegate: HandLandmarkerDelegate = .CPU
+}
+
+// MARK: ImageClassifierDelegate
+enum HandLandmarkerDelegate: CaseIterable {
+  case GPU
+  case CPU
+
+  var name: String {
+    switch self {
+    case .GPU:
+      return "GPU"
+    case .CPU:
+      return "CPU"
+    }
+  }
+
+  var delegate: Delegate {
+    switch self {
+    case .GPU:
+      return .GPU
+    case .CPU:
+      return .CPU
+    }
+  }
+
+  init?(name: String) {
+    switch name {
+    case HandLandmarkerDelegate.CPU.name:
+      self = HandLandmarkerDelegate.CPU
+    case HandLandmarkerDelegate.GPU.name:
+      self = HandLandmarkerDelegate.GPU
+    default:
+      return nil
+    }
+  }
 }
diff --git a/...les/hand_landmarker/ios/HandLandmarker/Configurations/InferenceConfigurationManager.swift b/...les/hand_landmarker/ios/HandLandmarker/Configurations/InferenceConfigurationManager.swift
@@ -38,6 +38,10 @@ class InferenceConfigurationManager: NSObject {
     didSet { postConfigChangedNotification() }
   }
 
+  var delegate: HandLandmarkerDelegate = DefaultConstants.delegate {
+    didSet { postConfigChangedNotification() }
+  }
+
   static let sharedInstance = InferenceConfigurationManager()
 
   static let notificationName = Notification.Name.init(rawValue: "com.google.mediapipe.inferenceConfigChanged")

diff --git a/examples/hand_landmarker/ios/HandLandmarker/Services/HandLandmarkerService.swift b/examples/hand_landmarker/ios/HandLandmarker/Services/HandLandmarkerService.swift
@@ -49,21 +49,24 @@ class HandLandmarkerService: NSObject {
   private var minHandPresenceConfidence: Float
   private var minTrackingConfidence: Float
   var modelPath: String
+  private var delegate: HandLandmarkerDelegate
 
   // MARK: - Custom Initializer
   private init?(modelPath: String?,
                 runningMode:RunningMode,
                 numHands: Int,
                 minHandDetectionConfidence: Float,
                 minHandPresenceConfidence: Float,
-                minTrackingConfidence: Float) {
+                minTrackingConfidence: Float,
+                delegate: HandLandmarkerDelegate) {
     guard let modelPath = modelPath else { return nil }
     self.modelPath = modelPath
     self.runningMode = runningMode
     self.numHands = numHands
     self.minHandDetectionConfidence = minHandDetectionConfidence
     self.minHandPresenceConfidence = minHandPresenceConfidence
     self.minTrackingConfidence = minTrackingConfidence
+    self.delegate = delegate
     super.init()
 
     createHandLandmarker()
@@ -77,6 +80,7 @@ class HandLandmarkerService: NSObject {
     handLandmarkerOptions.minHandPresenceConfidence = minHandPresenceConfidence
     handLandmarkerOptions.minTrackingConfidence = minTrackingConfidence
     handLandmarkerOptions.baseOptions.modelAssetPath = modelPath
+    handLandmarkerOptions.baseOptions.delegate = delegate.delegate
     if runningMode == .liveStream {
       handLandmarkerOptions.handLandmarkerLiveStreamDelegate = self
     }
@@ -95,14 +99,16 @@ class HandLandmarkerService: NSObject {
     minHandDetectionConfidence: Float,
     minHandPresenceConfidence: Float,
     minTrackingConfidence: Float,
-    videoDelegate: HandLandmarkerServiceVideoDelegate?) -> HandLandmarkerService? {
+    videoDelegate: HandLandmarkerServiceVideoDelegate?,
+    delegate: HandLandmarkerDelegate) -> HandLandmarkerService? {
     let handLandmarkerService = HandLandmarkerService(
       modelPath: modelPath,
       runningMode: .video,
       numHands: numHands,
       minHandDetectionConfidence: minHandDetectionConfidence,
       minHandPresenceConfidence: minHandPresenceConfidence,
-      minTrackingConfidence: minTrackingConfidence)
+      minTrackingConfidence: minTrackingConfidence,
+      delegate: delegate)
     handLandmarkerService?.videoDelegate = videoDelegate
     return handLandmarkerService
   }
@@ -113,14 +119,16 @@ class HandLandmarkerService: NSObject {
     minHandDetectionConfidence: Float,
     minHandPresenceConfidence: Float,
     minTrackingConfidence: Float,
-    liveStreamDelegate: HandLandmarkerServiceLiveStreamDelegate?) -> HandLandmarkerService? {
+    liveStreamDelegate: HandLandmarkerServiceLiveStreamDelegate?,
+    delegate: HandLandmarkerDelegate) -> HandLandmarkerService? {
     let handLandmarkerService = HandLandmarkerService(
       modelPath: modelPath,
       runningMode: .liveStream,
       numHands: numHands,
       minHandDetectionConfidence: minHandDetectionConfidence,
       minHandPresenceConfidence: minHandPresenceConfidence,
-      minTrackingConfidence: minTrackingConfidence)
+      minTrackingConfidence: minTrackingConfidence,
+      delegate: delegate)
     handLandmarkerService?.liveStreamDelegate = liveStreamDelegate
 
     return handLandmarkerService
@@ -131,14 +139,16 @@ class HandLandmarkerService: NSObject {
     numHands: Int,
     minHandDetectionConfidence: Float,
     minHandPresenceConfidence: Float,
-    minTrackingConfidence: Float) -> HandLandmarkerService? {
+    minTrackingConfidence: Float,
+    delegate: HandLandmarkerDelegate) -> HandLandmarkerService? {
     let handLandmarkerService = HandLandmarkerService(
       modelPath: modelPath,
       runningMode: .image,
       numHands: numHands,
       minHandDetectionConfidence: minHandDetectionConfidence,
       minHandPresenceConfidence: minHandPresenceConfidence,
-      minTrackingConfidence: minTrackingConfidence)
+      minTrackingConfidence: minTrackingConfidence,
+      delegate: delegate)
 
     return handLandmarkerService
   }

diff --git a/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/BottomSheetViewController.swift b/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/BottomSheetViewController.swift
@@ -44,6 +44,7 @@ class BottomSheetViewController: UIViewController {
   @IBOutlet weak var minTrackingConfidenceValueLabel: UILabel!
 
   @IBOutlet weak var toggleBottomSheetButton: UIButton!
+  @IBOutlet weak var chooseDelegateButton: UIButton!
 
   // MARK: Instance Variables
   var isUIEnabled: Bool = false {
@@ -77,8 +78,29 @@ class BottomSheetViewController: UIViewController {
 
     minTrackingConfidenceStepper.value = Double(InferenceConfigurationManager.sharedInstance.minTrackingConfidence)
     minTrackingConfidenceValueLabel.text = "\(InferenceConfigurationManager.sharedInstance.minTrackingConfidence)"
+
+    // Chose delegate option
+    let selectedDelegateAction = {(action: UIAction) in
+      self.updateDelegate(title: action.title)
+    }
+    let delegateActions: [UIAction] = HandLandmarkerDelegate.allCases.compactMap { delegate in
+      return UIAction(
+        title: delegate.name,
+        state: (InferenceConfigurationManager.sharedInstance.delegate == delegate) ? .on : .off,
+        handler: selectedDelegateAction
+      )
+    }
+
+    chooseDelegateButton.menu = UIMenu(children: delegateActions)
+    chooseDelegateButton.showsMenuAsPrimaryAction = true
+    chooseDelegateButton.changesSelectionAsPrimaryAction = true
   }
-
+
+  private func updateDelegate(title: String) {
+    guard let delegate = HandLandmarkerDelegate(name: title) else { return }
+    InferenceConfigurationManager.sharedInstance.delegate = delegate
+  }
+
   private func enableOrDisableClicks() {
     numHandsStepper.isEnabled = isUIEnabled
     minHandDetectionConfidenceStepper.isEnabled = isUIEnabled

diff --git a/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/CameraViewController.swift b/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/CameraViewController.swift
@@ -155,7 +155,8 @@ class CameraViewController: UIViewController {
         minHandDetectionConfidence: InferenceConfigurationManager.sharedInstance.minHandDetectionConfidence,
         minHandPresenceConfidence: InferenceConfigurationManager.sharedInstance.minHandPresenceConfidence,
         minTrackingConfidence: InferenceConfigurationManager.sharedInstance.minTrackingConfidence,
-        liveStreamDelegate: self)
+        liveStreamDelegate: self,
+        delegate: InferenceConfigurationManager.sharedInstance.delegate)
   }
 
   private func clearhandLandmarkerServiceOnSessionInterruption() {

diff --git a/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/MediaLibraryViewController.swift b/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/MediaLibraryViewController.swift
@@ -300,15 +300,17 @@ extension MediaLibraryViewController: UIImagePickerControllerDelegate, UINavigat
         numHands: InferenceConfigurationManager.sharedInstance.numHands,
         minHandDetectionConfidence: InferenceConfigurationManager.sharedInstance.minHandDetectionConfidence,
         minHandPresenceConfidence: InferenceConfigurationManager.sharedInstance.minHandPresenceConfidence,
-        minTrackingConfidence: InferenceConfigurationManager.sharedInstance.minTrackingConfidence)
+        minTrackingConfidence: InferenceConfigurationManager.sharedInstance.minTrackingConfidence,
+        delegate: InferenceConfigurationManager.sharedInstance.delegate)
       case .video:
       handLandmarkerService = HandLandmarkerService.videoHandLandmarkerService(
         modelPath: InferenceConfigurationManager.sharedInstance.modelPath,
         numHands: InferenceConfigurationManager.sharedInstance.numHands,
         minHandDetectionConfidence: InferenceConfigurationManager.sharedInstance.minHandDetectionConfidence,
         minHandPresenceConfidence: InferenceConfigurationManager.sharedInstance.minHandPresenceConfidence,
         minTrackingConfidence: InferenceConfigurationManager.sharedInstance.minTrackingConfidence,
-        videoDelegate: self)
+        videoDelegate: self,
+        delegate: InferenceConfigurationManager.sharedInstance.delegate)
       default:
         break;
     }

diff --git a/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/RootViewController.swift b/examples/hand_landmarker/ios/HandLandmarker/ViewContoller/RootViewController.swift
@@ -35,9 +35,9 @@ class RootViewController: UIViewController {
 
   // MARK: Constants
   private struct Constants {
-    static let inferenceBottomHeight = 260.0
+    static let inferenceBottomHeight = 304.0
     static let expandButtonHeight = 41.0
-    static let expandButtonTopSpace = 10.0
+    static let expandButtonTopSpace = 0.0
     static let mediaLibraryViewControllerStoryBoardId = "MEDIA_LIBRARY_VIEW_CONTROLLER"
     static let cameraViewControllerStoryBoardId = "CAMERA_VIEW_CONTROLLER"
     static let storyBoardName = "Main"

diff --git a/examples/hand_landmarker/ios/Podfile b/examples/hand_landmarker/ios/Podfile
@@ -5,7 +5,7 @@ target 'HandLandmarker' do
   # Comment the next line if you don't want to use dynamic frameworks
   use_frameworks!
 
-  pod 'MediaPipeTasksVision', '0.10.5'
+  pod 'MediaPipeTasksVision', '0.10.12'
 
 end