Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import numpy as np
import pyautogui
from collections import Counter
from screeninfo import get_monitors
MONITOR = get_monitors()[0]
WIDTH, HEIGHT = MONITOR.width, MONITOR.height
class Mouse:
def __init__(self) -> None:
self.predictions = []
self.previous_action = None
self.freeze_action = False
# parameters to fine-tune
self.action_length = 5
#self.move_distance = 10
self.scroll_distance = 50
#self.time_checking = 0.5
self.stop_pos = None
def get_hand_pos(self, hand_pos):
self.hand_pos_x = hand_pos[0]
self.hand_pos_y = hand_pos[1]
def add_prediction(self, prediction):
self.predictions.append(prediction)
if len(self.predictions) == self.action_length:
self.make_action()
def make_action(self):
action = self.get_major_element(self.predictions)
if self.freeze_action and action == self.previous_action:
self.update_init(action)
else:
self.mouse_control(action)
self.update_init(action)
def update_init(self, action):
self.predictions = []
self.previous_action = action
self.freeze_action = action in {"left click", "right click", "double click"} # maybe change to keyboard and drops
def mouse_hand_parameters(self):
pass
def mouse_control(self, prediction):
if prediction == "stop execution" or None:
pass # Stop movement
elif prediction == "move cursor":
#hand_point = ([int(self.hand_pos_x*WIDTH), int(self.hand_pos_y*HEIGHT)])
hand_x = np.clip(int(self.hand_pos_x*WIDTH), 0, WIDTH-1)
hand_y = np.clip(int(self.hand_pos_y*HEIGHT), 0, HEIGHT-1)
pyautogui.moveTo(hand_x, hand_y)
elif prediction == "stop moving":
pyautogui.move(0, 0) # Stop cursor
self.stop_pos = pyautogui.position()
elif prediction == "left click":
pyautogui.click() # Left click
elif prediction == "right click":
pyautogui.click(button='right') # Right click
elif prediction == "double click":
pyautogui.click(clicks=2) # Double click
elif prediction == "scrolling up":
pyautogui.scroll(self.scroll_distance) # Scroll up
elif prediction == "scrolling down":
pyautogui.scroll(-self.scroll_distance) # Scroll down
elif prediction == "scrolling right":
pyautogui.hscroll(self.scroll_distance) # Scroll right
# THIS FUNCTION NOT WORKS ON WINDOWS
elif prediction == "scrolling left":
pyautogui.hscroll(self.scroll_distance) # Scroll left
# THIS FUNCTION NOT WORKS ON WINDOWS
elif prediction == "drag":
if self.previous_action == "stop moving":
pyautogui.moveTo(*self.stop_pos)
pyautogui.mouseDown()
hand_x = np.clip(int(self.hand_pos_x*WIDTH), 0, WIDTH-1)
hand_y = np.clip(int(self.hand_pos_y*HEIGHT), 0, HEIGHT-1)
pyautogui.moveTo(hand_x, hand_y)
elif prediction == "drop":
pyautogui.mouseUp()
elif prediction == "multiple item selection grab":
pyautogui.mouseDown()
elif prediction == "multiple item selection drop":
pyautogui.mouseUp()
elif prediction == "change to keyboard":
pass
#time.sleep(self.time_checking) # Adjust speed of movement
def get_major_element(self, string_list):
counts = Counter(string_list)
# Find the element with the maximum count
major_element, _ = counts.most_common(1)[0]
return major_element