fix: support more objective-c objects (#323)

* Update _macos.py * Update _macos.py * Update _macos.py * add bug report option * Update _macos.py * undo formatting * refactor * split * remove debug line * Update _macos.py * Update _macos.py * Update _macos.py * Update _macos.py * reorganize * Update _macos.py * Update _macos.py * nonetype * Update _macos.py
OpenAdaptAI · Jul 17, 2023 · be0774f · be0774f
1 parent 0d6a155
commit be0774f
Showing 1 changed file with 71 additions and 19 deletions.
diff --git a/openadapt/window/_macos.py b/openadapt/window/_macos.py
@@ -6,6 +6,9 @@
 import AppKit
 import ApplicationServices
 import Quartz
+import Foundation
+import re
+import plistlib
 
 
 def get_active_window_state():
@@ -14,8 +17,8 @@ def get_active_window_state():
     meta = get_active_window_meta()
     data = get_window_data(meta)
     title_parts = [
-        meta['kCGWindowOwnerName'],
-        meta['kCGWindowName'],
+        meta["kCGWindowOwnerName"],
+        meta["kCGWindowName"],
     ]
     title_parts = [part for part in title_parts if part]
     title = " ".join(title_parts)
@@ -47,14 +50,12 @@ def get_active_window_state():
 def get_active_window_meta():
     windows = Quartz.CGWindowListCopyWindowInfo(
         (
-            Quartz.kCGWindowListExcludeDesktopElements |
-            Quartz.kCGWindowListOptionOnScreenOnly
+            Quartz.kCGWindowListExcludeDesktopElements
+            | Quartz.kCGWindowListOptionOnScreenOnly
         ),
         Quartz.kCGNullWindowID,
     )
-    active_windows_info = [
-        win for win in windows if win['kCGWindowLayer'] == 0
-    ]
+    active_windows_info = [win for win in windows if win["kCGWindowLayer"] == 0 and win["kCGWindowOwnerName"] != "Window Server"]
     active_window_info = active_windows_info[0]
     return active_window_info
 
@@ -63,7 +64,7 @@ def get_active_window(window_meta):
     pid = window_meta["kCGWindowOwnerPID"]
     app_ref = ApplicationServices.AXUIElementCreateApplication(pid)
     error_code, window = ApplicationServices.AXUIElementCopyAttributeValue(
-        app_ref, 'AXFocusedWindow', None
+        app_ref, "AXFocusedWindow", None
     )
     if error_code:
         logger.error("Error getting focused window")
@@ -98,13 +99,14 @@ def dump_state(element, elements=None):
                 state[k] = _state
         return state
     else:
-        error_code, attr_names = (
-            ApplicationServices.AXUIElementCopyAttributeNames(element, None)
+        error_code, attr_names = ApplicationServices.AXUIElementCopyAttributeNames(
+            element, None
         )
         if attr_names:
             state = {}
             for attr_name in attr_names:
-
+                if attr_name is None:
+                    continue
                 # don't traverse back up
                 # for WindowEvents:
                 if "parent" in attr_name.lower():
@@ -113,14 +115,19 @@ def dump_state(element, elements=None):
                 if attr_name in ("AXTopLevelUIElement", "AXWindow"):
                     continue
 
-                error_code, attr_val = (
-                    ApplicationServices.AXUIElementCopyAttributeValue(
-                        element, attr_name, None,
-                    )
+                (
+                    error_code,
+                    attr_val,
+                ) = ApplicationServices.AXUIElementCopyAttributeValue(
+                    element,
+                    attr_name,
+                    None,
                 )
 
                 # for ActionEvents
-                if attr_name == "AXRole" and "application" in attr_val.lower():
+                if attr_val is not None and (
+                    attr_name == "AXRole" and "application" in attr_val.lower()
+                ):
                     continue
 
                 _state = dump_state(attr_val, elements)
@@ -135,14 +142,59 @@ def dump_state(element, elements=None):
 def deepconvert_objc(object):
     """Convert all contents of an ObjC object to Python primitives."""
     value = object
+    strings = (
+        str,
+        AppKit.NSString,
+        ApplicationServices.AXTextMarkerRangeRef,
+        ApplicationServices.AXUIElementRef,
+        ApplicationServices.AXTextMarkerRef,
+        Quartz.CGPathRef,
+    )
+
     if isinstance(object, AppKit.NSNumber):
         value = int(object)
     elif isinstance(object, AppKit.NSArray) or isinstance(object, list):
         value = [deepconvert_objc(x) for x in object]
     elif isinstance(object, AppKit.NSDictionary) or isinstance(object, dict):
-        value = dict(object)
-        for k, v in value.items():
-            value[k] = deepconvert_objc(v)
+        value = {deepconvert_objc(k): deepconvert_objc(v) for k, v in object.items()}
+    elif isinstance(object, strings):
+        value = str(object)
+    # handle core-foundation class AXValueRef
+    elif isinstance(object, ApplicationServices.AXValueRef):
+        # convert to dict - note: this object is not iterable
+        # TODO: access directly, e.g. via ApplicationServices.AXUIElementCopyAttributeValue
+        rep = repr(object)
+        x_value = re.search(r"x:([\d.]+)", rep)
+        y_value = re.search(r"y:([\d.]+)", rep)
+        w_value = re.search(r"w:([\d.]+)", rep)
+        h_value = re.search(r"h:([\d.]+)", rep)
+        type_value = re.search(r"type\s?=\s?(\w+)", rep)
+        value = {
+            "x": float(x_value.group(1)) if x_value else None,
+            "y": float(y_value.group(1)) if y_value else None,
+            "w": float(w_value.group(1)) if w_value else None,
+            "h": float(h_value.group(1)) if h_value else None,
+            "type": type_value.group(1) if type_value else None,
+        }
+    elif isinstance(object, Foundation.NSURL):
+        value = str(object.absoluteString())
+    elif isinstance(object, Foundation.__NSCFAttributedString):
+        value = str(object.string())
+    elif isinstance(object, Foundation.__NSCFData):
+        value = {
+            deepconvert_objc(k): deepconvert_objc(v)
+            for k, v in plistlib.loads(object).items()
+        }
+    elif isinstance(object, plistlib.UID):
+        value = object.data
+    else:
+        if object and not (isinstance(object, bool) or isinstance(object, int)):
+            logger.warning(
+                f"Unknown type: {type(object)} - "
+                f"Please report this on GitHub: "
+                f"https://github.com/MLDSAI/OpenAdapt/issues/new?assignees=&labels=bug&projects=&template=bug_form.yml&title=%5BBug%5D%3A+"
+            )
+            logger.warning(f"{object=}")
     if value:
         value = atomacos._converter.Converter().convert_value(value)
     return value