parameters + tools added, grab fix

final_project/main.py

 import cv2
-import random
 import mediapipe as mp
-import pickle
 import numpy as np
 from sklearn.ensemble import RandomForestClassifier
-import time
-import os
 from tkinter import Tk, Label
 from PIL import Image, ImageTk
 
 from mouse_class import Mouse
 from hand_detection import normalise_landmarks
+from tools import load_model, set_camera_window
 
 def main():
     #define Mouse
     mouse = Mouse()
 
-    # load model
-    current_dir = os.path.dirname(__file__)
-    model_path = os.path.abspath(os.path.join(current_dir, os.pardir, 'trained_models', 'trained_Moni_data.p'))
-    model_dict = pickle.load(open(model_path, 'rb'))
-    model = model_dict['model']
+    # load MOUSE model
+    model = load_model(device = "mouse")
     
     # create hand detection object
     mp_hands = mp.solutions.hands
@@ -35,25 +29,8 @@ def main():
         return
     
     # set up Tkinter window
-    root = Tk()
-    root.title("Hand Tracking - Always on Top")
-    root.attributes("-topmost", True)
-    video_label = Label(root)
-    video_label.pack()
-
-    # adjust window geometry
-    # Get the screen width and height
-    screen_width = root.winfo_screenwidth()
-    screen_height = root.winfo_screenheight()
+    root, video_label = set_camera_window()
     
-    # Define window size and position (e.g., 320x240 window at bottom-right corner)
-    window_width = 160*4
-    window_height = 120*4
-    x_position = screen_width - window_width - 10  # 10px margin from the right
-    y_position = screen_height - window_height - 70  # 50px margin from the bottom
-
-    # Set window geometry
-    root.geometry(f"{window_width}x{window_height}+{x_position}+{y_position}")
     # mediapipe hand object
     with mp_hands.Hands(max_num_hands=1, model_complexity=1,
                         min_detection_confidence=0.9, min_tracking_confidence=0.9) as hands:

final_project/mouse_class.py

@@ -3,11 +3,11 @@ import pyautogui
 from collections import Counter
 from screeninfo import get_monitors
 
+from parameters import DETECTION_SIDE_MARGIN, DETECTION_BOTTOM_MARGIN, DEFAULT_HAND_SIZE
+
 MONITOR = get_monitors()[0]
 WIDTH, HEIGHT = MONITOR.width, MONITOR.height
-SIDE_MARGIN = 0.06
-BOTTOM_MARGIN = 0.6
-DEFAULT_HAND_SIZE = 0.5
+
 
 class Mouse:
     def __init__(self) -> None:
@@ -17,6 +17,7 @@ class Mouse:
         self.freeze_action = False
         self.stop_pos = None
         self.hand_size = None
+        self.drag_start = None
 
         # parameters to fine-tune
         self.action_length = 5
@@ -48,6 +49,11 @@ class Mouse:
         if self.freeze_action and action == self.previous_action:
             self.update_init(action)
         else:
+            if self.previous_action != action and self.previous_action == "drag":
+                pyautogui.mouseUp()
+                self.drag_start = None
+                print("drop")
+
             self.mouse_control(action)
             self.update_init(action)
 
@@ -58,6 +64,7 @@ class Mouse:
         self.freeze_action = action in {"left click", "right click", "double click"} # maybe change to keyboard and drops
 
 
+
     def mouse_control(self, prediction):
         if prediction == "stop execution" or None:
             pass  # Stop movement
@@ -66,7 +73,7 @@ class Mouse:
             #hand_point = ([int(self.hand_pos_x*WIDTH), int(self.hand_pos_y*HEIGHT)])
             hand_x = np.clip(int(self.hand_pos_x*WIDTH), 0, WIDTH-1)
             hand_y = np.clip(int(self.hand_pos_y*HEIGHT), 0, HEIGHT-1)
-            pyautogui.moveTo(hand_x, hand_y)
+            pyautogui.moveTo(hand_x, hand_y, tween = pyautogui.easeOutQuad)
 
         elif prediction == "stop moving":
             pyautogui.move(0, 0)  # Stop cursor
@@ -88,9 +95,15 @@ class Mouse:
             pyautogui.hscroll(self.scroll_distance)    # Scroll left
             # THIS FUNCTION NOT WORKS ON WINDOWS
         elif prediction == "drag":
-            if self.previous_action == "stop moving":
-                pyautogui.moveTo(*self.stop_pos)
-            pyautogui.mouseDown()  
+            if self.previous_action == "stop moving" and self.drag_start is None:
+                self.drag_start = self.stop_pos
+                pyautogui.mouseDown(*self.drag_start) 
+            elif self.drag_start is None:
+                hand_x = np.clip(int(self.hand_pos_x*WIDTH), 0, WIDTH-1)
+                hand_y = np.clip(int(self.hand_pos_y*HEIGHT), 0, HEIGHT-1)
+                self.drag_start = [hand_x, hand_y]
+                pyautogui.mouseDown(*self.drag_start) 
+ 
             hand_x = np.clip(int(self.hand_pos_x*WIDTH), 0, WIDTH-1)
             hand_y = np.clip(int(self.hand_pos_y*HEIGHT), 0, HEIGHT-1)
             pyautogui.moveTo(hand_x, hand_y)
@@ -113,9 +126,9 @@ class Mouse:
         return major_element
     
     def resize_coordinates(self):
-        max_x = 1 - 2 * SIDE_MARGIN 
-        self.hand_pos_x = (self.hand_pos_x-SIDE_MARGIN) / (max_x - SIDE_MARGIN) #* DEFAULT_HAND_SIZE/self.hand_size
-        self.hand_pos_y = (self.hand_pos_y-SIDE_MARGIN) / (BOTTOM_MARGIN - SIDE_MARGIN) #* DEFAULT_HAND_SIZE/self.hand_size
+        max_x = 1 - 2 * DETECTION_SIDE_MARGIN 
+        self.hand_pos_x = (self.hand_pos_x-DETECTION_SIDE_MARGIN) / (max_x - DETECTION_SIDE_MARGIN) #* DEFAULT_HAND_SIZE/self.hand_size
+        self.hand_pos_y = (self.hand_pos_y-DETECTION_SIDE_MARGIN) / (DETECTION_BOTTOM_MARGIN - DETECTION_SIDE_MARGIN) #* DEFAULT_HAND_SIZE/self.hand_size
 
     def get_hand_size(self, middle_tip_coord, palm_coord):
         self.hand_size = palm_coord[1] - middle_tip_coord[1]

final_project/parameters.py

+
+MOUSE_MODEL_NAME = 'mouse_v2.p'
+KEYBOARD_MODEL_NAME = ""
+
+# CREATED CAMERA WINDOW
+CAMERA_WINDOW_WIDTH = 160*4
+CAMERA_WINDOW_HEIGHT = 120*4
+CAMERA_MARGIN_RIGHT = 10
+CAMERA_MARGIN_BOTTON = 70
+
+# DETECTION WINDOW
+DETECTION_SIDE_MARGIN = 0.06
+DETECTION_BOTTOM_MARGIN = 0.6
+
+DEFAULT_HAND_SIZE = 0.5

final_project/tools.py

+import os
+import pickle
+from tkinter import Tk, Label
+
+from parameters import (MOUSE_MODEL_NAME, KEYBOARD_MODEL_NAME, 
+                        CAMERA_WINDOW_WIDTH, CAMERA_WINDOW_HEIGHT, CAMERA_MARGIN_RIGHT, CAMERA_MARGIN_BOTTON)
+
+def load_model(device):
+    current_dir = os.path.dirname(__file__)
+    if device == "mouse":
+        model_path = os.path.abspath(os.path.join(current_dir, os.pardir, 'trained_models', MOUSE_MODEL_NAME))
+    elif device == "keyboard":
+        model_path = os.path.abspath(os.path.join(current_dir, os.pardir, 'trained_models', KEYBOARD_MODEL_NAME))
+    model_dict = pickle.load(open(model_path, 'rb'))
+    model = model_dict['model']
+
+    return model
+
+def set_camera_window():
+    # set up Tkinter window
+    root = Tk()
+    root.title("Hand Tracking - Always on Top")
+    root.attributes("-topmost", True)
+    video_label = Label(root)
+    video_label.pack()
+
+    # adjust window geometry
+    # Get the screen width and height
+    screen_width = root.winfo_screenwidth()
+    screen_height = root.winfo_screenheight()
+    
+    # Define window size and position (e.g., 320x240 window at bottom-right corner)
+    x_position = screen_width - CAMERA_WINDOW_WIDTH - CAMERA_MARGIN_RIGHT  # 10px margin from the right
+    y_position = screen_height - CAMERA_WINDOW_HEIGHT - CAMERA_MARGIN_BOTTON  # 50px margin from the bottom
+
+    # Set window geometry
+    root.geometry(f"{CAMERA_WINDOW_WIDTH}x{CAMERA_WINDOW_HEIGHT}+{x_position}+{y_position}")
+
+    return root, video_label
\ No newline at end of file