mouse_class.py

import numpy as np
import pyautogui
from collections import Counter
from screeninfo import get_monitors

from parameters import DETECTION_SIDE_MARGIN, DEFAULT_HAND_SIZE, VISUALIZATION_MARGIN

MONITOR = get_monitors()[0]
WIDTH, HEIGHT = MONITOR.width, MONITOR.height


class Mouse:
    def __init__(self) -> None:
        
        self.predictions = []
        self.previous_action = None
        self.freeze_action = False
        self.stop_pos = None
        self.hand_size = None
        self.drag_start = None

        # parameters to fine-tune
        self.action_length = 5
        self.scroll_distance = 50   
        

    def get_hand_pos(self, hand_pos):
        self.hand_pos_x = hand_pos[0]
        self.hand_pos_y = hand_pos[1]
        #print(f"hand size: {self.hand_size}")
        #print(f"hand coord: {self.hand_pos_x}, {self.hand_pos_y} \n")
        self.resize_coordinates()


    def add_prediction(self, prediction):
        self.predictions.append(prediction)
        if len(self.predictions) == self.action_length:
            self.make_action()
        elif self.previous_action in {"move cursor", "scrolling up", "scrolling down", "scrolling left", "scrolling right"}:
            if len(self.predictions) > 1:
                safe_action = self.get_major_element(self.predictions[-3:])
                if safe_action == self.previous_action:
                    self.mouse_control(self.previous_action)
            else:
                self.mouse_control(self.previous_action)

    def make_action(self):
        action = self.get_major_element(self.predictions)
        if self.freeze_action and action == self.previous_action:
            self.update_init(action)
        else:
            if self.previous_action != action and self.previous_action == "drag":
                pyautogui.mouseUp()
                self.drag_start = None

            self.mouse_control(action)
            self.update_init(action)

    def update_init(self, action):
        self.predictions = []
        self.previous_action = action

        self.freeze_action = action in {"left click", "right click", "double click"} # maybe change to keyboard and drops



    def mouse_control(self, prediction):
        if prediction == "stop execution" or None:
            pass  # Stop movement
        elif prediction == "move cursor":
            
            #hand_point = ([int(self.hand_pos_x*WIDTH), int(self.hand_pos_y*HEIGHT)])
            hand_x = np.clip(int(self.hand_pos_x*WIDTH), VISUALIZATION_MARGIN, WIDTH-VISUALIZATION_MARGIN)
            hand_y = np.clip(int(self.hand_pos_y*HEIGHT), VISUALIZATION_MARGIN, HEIGHT-VISUALIZATION_MARGIN)
            #print(f"x: {hand_x}, y: {hand_y} \n width: {WIDTH}, height: {HEIGHT}")
            #pyautogui.moveTo(hand_x, hand_y, tween = pyautogui.easeOutQuad)
            pyautogui.moveTo(hand_x, hand_y)

        elif prediction == "stop moving":
            pyautogui.move(0, 0)  # Stop cursor
            self.stop_pos = pyautogui.position()
        elif prediction == "left click":
            pyautogui.click()  # Left click 
        elif prediction == "right click":
            pyautogui.click(button='right')  # Right click
        elif prediction == "double click":
            pyautogui.click(clicks=2)    # Double click
        elif prediction == "scrolling up":
            pyautogui.scroll(self.scroll_distance)  # Scroll up
        elif prediction == "scrolling down":
            pyautogui.scroll(-self.scroll_distance)  # Scroll down
        elif prediction == "scrolling right":
            pyautogui.hscroll(self.scroll_distance)    # Scroll right
            # THIS FUNCTION NOT WORKS ON WINDOWS
        elif prediction == "scrolling left":
            pyautogui.hscroll(self.scroll_distance)    # Scroll left
            # THIS FUNCTION NOT WORKS ON WINDOWS
        elif prediction == "drag":
            if self.previous_action == "stop moving" and self.drag_start is None:
                self.drag_start = self.stop_pos
                pyautogui.mouseDown(*self.drag_start) 
            elif self.drag_start is None:
                hand_x = np.clip(int(self.hand_pos_x*WIDTH), VISUALIZATION_MARGIN, WIDTH-VISUALIZATION_MARGIN)
                hand_y = np.clip(int(self.hand_pos_y*HEIGHT), VISUALIZATION_MARGIN, HEIGHT-VISUALIZATION_MARGIN)
                self.drag_start = [hand_x, hand_y]
                pyautogui.mouseDown(*self.drag_start) 
 
            hand_x = np.clip(int(self.hand_pos_x*WIDTH), VISUALIZATION_MARGIN, WIDTH-VISUALIZATION_MARGIN)
            hand_y = np.clip(int(self.hand_pos_y*HEIGHT), VISUALIZATION_MARGIN, HEIGHT-VISUALIZATION_MARGIN)
            pyautogui.moveTo(hand_x, hand_y)
        elif prediction == "drop":
            pyautogui.mouseUp()
        elif prediction == "multiple item selection grab":
            pyautogui.mouseDown()
        elif prediction == "multiple item selection drop":
            pyautogui.mouseUp()
        elif prediction == "change to keyboard":
            pass
        
        #time.sleep(self.time_checking)  # Adjust speed of movement
    
    def get_major_element(self, string_list):
        counts = Counter(string_list)
        # Find the element with the maximum count
        major_element, _ = counts.most_common(1)[0]
        
        return major_element
    
    def resize_coordinates(self):
        max_x = 1 - 2 * DETECTION_SIDE_MARGIN 
        calculated_bottom_margin = (-0.7)*self.hand_size + 1
        #default_bottom_margin = -0.7*DEFAULT_HAND_SIZE + 1
        if self.hand_size > DEFAULT_HAND_SIZE:                          # close
            #speed = default_bottom_margin / calculated_bottom_margin
            speed = 2*(self.hand_size-DEFAULT_HAND_SIZE)**2 + 1
        else:                                                           # far away
            #speed = calculated_bottom_margin / default_bottom_margin
            speed = 7*(self.hand_size-DEFAULT_HAND_SIZE)**2 + 1

        self.hand_pos_x = (self.hand_pos_x-DETECTION_SIDE_MARGIN) / (max_x - DETECTION_SIDE_MARGIN) * speed
        self.hand_pos_y = (self.hand_pos_y-DETECTION_SIDE_MARGIN) / (calculated_bottom_margin - DETECTION_SIDE_MARGIN) * speed

        #print(f"DETECTION_BOTTOM_MARGIN: {calculated_bottom_margin} DEFAULT_BOTTOM_MARGIN: {default_bottom_margin} \n SPEED: {speed} \n")
    def get_hand_size(self, middle_tip_coord, palm_coord):
        self.hand_size = palm_coord[1] - middle_tip_coord[1]