Merge pull request #66 from tanjeffreyz/dev

Added support for windowed mode, detached and expanded minimaps

bot.py

@@ -5,12 +5,11 @@
 import threading
 import time
 import cv2
-import mss
-import mss.windows
 import utils
 import inspect
 import components
 import numpy as np
+from PIL import ImageGrab
 from os.path import splitext, basename
 from routine import Routine
 from components import Point
@@ -64,30 +63,30 @@ def _main(self):
         model = detection.load_model()
         print('\n[~] Initialized detection algorithm.')
 
-        mss.windows.CAPTUREBLT = 0
-        with mss.mss() as sct:
-            self.ready = True
-            config.listener.enabled = True
-            while True:
-                if config.enabled and len(config.routine) > 0:
-                    self.buff.main()
-
-                    # Highlight the current Point
-                    config.gui.view.routine.select(config.routine.index)
-                    config.gui.view.details.display_info(config.routine.index)
-
-                    # Execute next Point in the routine
-                    element = config.routine[config.routine.index]
-                    if self.rune_active and isinstance(element, Point) \
-                            and element.location == self.rune_closest_pos:
-                        self._solve_rune(model, sct)
-                    element.execute()
-                    config.routine.step()
-                else:
-                    time.sleep(0.01)
+        # mss.windows.CAPTUREBLT = 0
+        # with mss.mss() as sct:
+        self.ready = True
+        config.listener.enabled = True
+        while True:
+            if config.enabled and len(config.routine) > 0:
+                self.buff.main()
+
+                # Highlight the current Point
+                config.gui.view.routine.select(config.routine.index)
+                config.gui.view.details.display_info(config.routine.index)
+
+                # Execute next Point in the routine
+                element = config.routine[config.routine.index]
+                if self.rune_active and isinstance(element, Point) \
+                        and element.location == self.rune_closest_pos:
+                    self._solve_rune(model)
+                element.execute()
+                config.routine.step()
+            else:
+                time.sleep(0.01)
 
     @utils.run_if_enabled
-    def _solve_rune(self, model, sct):
+    def _solve_rune(self, model):
         """
         Moves to the position of the rune and solves the arrow-key puzzle.
         :param model:   The TensorFlow model to classify with.
@@ -104,7 +103,8 @@ def _solve_rune(self, model, sct):
         print('\nSolving rune:')
         inferences = []
         for _ in range(15):
-            frame = np.array(sct.grab(config.MONITOR))
+            frame = np.array(ImageGrab.grab(config.capture.window))
+            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
             solution = detection.merge_detection(model, frame)
             if solution:
                 print(', '.join(solution))
@@ -115,7 +115,8 @@ def _solve_rune(self, model, sct):
                     time.sleep(1)
                     for _ in range(3):
                         time.sleep(0.3)
-                        frame = np.array(sct.grab(config.MONITOR))
+                        frame = np.array(ImageGrab.grab(config.capture.window))
+                        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
                         rune_buff = utils.multi_match(frame[:frame.shape[0]//8, :],
                                                       RUNE_BUFF_TEMPLATE,
                                                       threshold=0.9)

capture.py

@@ -2,25 +2,37 @@
 
 import config
 import utils
-import mss
-import mss.windows
 import time
 import cv2
 import threading
+import ctypes
 import numpy as np
+from ctypes import wintypes
+from PIL import ImageGrab
+user32 = ctypes.windll.user32
+user32.SetProcessDPIAware()
 
 
 # The distance between the top of the minimap and the top of the screen
-MINIMAP_TOP_BORDER = 21
+MINIMAP_TOP_BORDER = 5
 
 # The thickness of the other three borders of the minimap
 MINIMAP_BOTTOM_BORDER = 8
 
-# The bottom right corner of the minimap
-MINIMAP_TEMPLATE = cv2.imread('assets/minimap_template.jpg', 0)
+# Offset in pixels to adjust for windowed mode
+WINDOWED_OFFSET_TOP = 36
+WINDOWED_OFFSET_LEFT = 10
+
+# The top-left and bottom-right corners of the minimap
+MM_TL_TEMPLATE = cv2.imread('assets/minimap_tl_template.png', 0)
+MM_BR_TEMPLATE = cv2.imread('assets/minimap_br_template.png', 0)
+
+MMT_HEIGHT = max(MM_TL_TEMPLATE.shape[0], MM_BR_TEMPLATE.shape[0])
+MMT_WIDTH = max(MM_TL_TEMPLATE.shape[1], MM_BR_TEMPLATE.shape[1])
 
 # The player's symbol on the minimap
 PLAYER_TEMPLATE = cv2.imread('assets/player_template.png', 0)
+PT_HEIGHT, PT_WIDTH = PLAYER_TEMPLATE.shape
 
 
 class Capture:
@@ -39,6 +51,7 @@ def __init__(self):
         self.minimap = {}
         self.minimap_ratio = 1
         self.minimap_sample = None
+        self.window = (0, 0, 1366, 768)
 
         self.ready = False
         self.calibrated = False
@@ -54,39 +67,68 @@ def start(self):
     def _main(self):
         """Constantly monitors the player's position and in-game events."""
 
-        mss.windows.CAPTUREBLT = 0
-        with mss.mss() as sct:
-            while True:
-                self.frame = np.array(sct.grab(config.MONITOR))
-
-                if not self.calibrated:
-                    # Calibrate by finding the bottom right corner of the minimap
-                    _, br = utils.single_match(self.frame[:round(self.frame.shape[0] / 4),
-                                               :round(self.frame.shape[1] / 3)],
-                                               MINIMAP_TEMPLATE)
-                    mm_tl = (MINIMAP_BOTTOM_BORDER, MINIMAP_TOP_BORDER)
-                    mm_br = tuple(max(75, a - MINIMAP_BOTTOM_BORDER) for a in br)
-                    self.minimap_ratio = (mm_br[0] - mm_tl[0]) / (mm_br[1] - mm_tl[1])
-                    self.minimap_sample = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]]
-                    self.calibrated = True
-
-                # Crop the frame to only show the minimap
-                minimap = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]]
-
-                # Determine the player's position
-                player = utils.multi_match(minimap, PLAYER_TEMPLATE, threshold=0.8)
-                if player:
-                    config.player_pos = utils.convert_to_relative(player[0], minimap)
-
-                # Package display information to be polled by GUI
-                self.minimap = {
-                    'minimap': minimap,
-                    'rune_active': config.bot.rune_active,
-                    'rune_pos': config.bot.rune_pos,
-                    'path': config.path,
-                    'player_pos': config.player_pos
-                }
-
-                if not self.ready:
-                    self.ready = True
-                time.sleep(0.001)
+        while True:
+            if not self.calibrated:
+                handle = user32.FindWindowW(None, 'MapleStory')
+                rect = wintypes.RECT()
+                user32.GetWindowRect(handle, ctypes.pointer(rect))
+                rect = (rect.left, rect.top, rect.right, rect.bottom)
+                rect = tuple(max(0, x) for x in rect)
+
+                # Preliminary window to template match minimap
+                self.window = (
+                    rect[0],
+                    rect[1],
+                    max(rect[2], rect[0] + MMT_WIDTH),     # Make room for minimap templates
+                    max(rect[3], rect[1] + MMT_HEIGHT)
+                )
+
+                # Calibrate by finding the bottom right corner of the minimap
+                self.frame = np.array(ImageGrab.grab(self.window))
+                self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR)
+                tl, _ = utils.single_match(self.frame, MM_TL_TEMPLATE)
+                _, br = utils.single_match(self.frame, MM_BR_TEMPLATE)
+                mm_tl = (
+                    tl[0] + MINIMAP_BOTTOM_BORDER,
+                    tl[1] + MINIMAP_TOP_BORDER
+                )
+                mm_br = (
+                    max(mm_tl[0] + PT_WIDTH, br[0] - MINIMAP_BOTTOM_BORDER),
+                    max(mm_tl[1] + PT_HEIGHT, br[1] - MINIMAP_BOTTOM_BORDER - 1)
+                )
+
+                # Resize window to encompass minimap if needed
+                self.window = (
+                    rect[0],
+                    rect[1],
+                    max(rect[2], mm_br[0]),
+                    max(rect[3], mm_br[1])
+                )
+                self.minimap_ratio = (mm_br[0] - mm_tl[0]) / (mm_br[1] - mm_tl[1])
+                self.minimap_sample = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]]
+                self.calibrated = True
+
+            # Take screenshot
+            self.frame = np.array(ImageGrab.grab(self.window))
+            self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR)
+
+            # Crop the frame to only show the minimap
+            minimap = self.frame[mm_tl[1]:mm_br[1], mm_tl[0]:mm_br[0]]
+
+            # Determine the player's position
+            player = utils.multi_match(minimap, PLAYER_TEMPLATE, threshold=0.8)
+            if player:
+                config.player_pos = utils.convert_to_relative(player[0], minimap)
+
+            # Package display information to be polled by GUI
+            self.minimap = {
+                'minimap': minimap,
+                'rune_active': config.bot.rune_active,
+                'rune_pos': config.bot.rune_pos,
+                'path': config.path,
+                'player_pos': config.player_pos
+            }
+
+            if not self.ready:
+                self.ready = True
+            time.sleep(0.001)

config.py

@@ -1,11 +1,5 @@
 """A collection of variables shared across multiple modules."""
 
-#################################
-#           CONSTANTS           #
-#################################
-# Describes the dimensions of the screen to capture with mss
-MONITOR = {'top': 0, 'left': 0, 'width': 1366, 'height': 768}
-
 
 #################################
 #       Global Variables        #

gui_components/__init__.py

@@ -1,7 +1,7 @@
-import gui_components.menu as menu
-import gui_components.view as view
-import gui_components.edit as edit
-import gui_components.settings as settings
+import gui_components.menu.main as menu
+import gui_components.view.main as view
+import gui_components.edit.main as edit
+import gui_components.settings.main as settings
 
 Menu = menu.Menu
 View = view.View