Command actions & activation key recording (#116)

* first implementation of hotkey recording

* refactor command actions

* refactor record hotkey

* refactor command execution

* bugfixing

* start refactor config template

* convert rest of the config (untested, key names might be wrong)

x

* add commands and stubs

* add response command

* added mouse button push to talk support

* add missing await

* add record hotkey to command handler

* fix function call

* add timeout to record key function

* monkeyfix MacOS thread generation

---------

Co-authored-by: Timo Korinth <[email protected]>

Author: Shackless

api/commands.py

@@ -1,6 +1,7 @@
 from typing import Literal, Optional
 from pydantic import BaseModel
 from api.enums import CommandTag, LogSource, LogType, ToastType
+from api.interface import CommandActionConfig
 
 
 # We use this Marker base class for reflection to "iterate all commands"
@@ -22,6 +23,18 @@ class SaveSecretCommand(WebSocketCommandModel):
     secret_value: str
 
 
+class RecordKeyboardActionsCommand(WebSocketCommandModel):
+    command: Literal["record_keyboard_actions"] = "record_keyboard_actions"
+
+
+class RecordMouseActionsCommand(WebSocketCommandModel):
+    command: Literal["record_mouse_actions"] = "record_mouse_actions"
+
+
+class StopRecordingCommand(WebSocketCommandModel):
+    command: Literal["stop_recording"] = "stop_recording"
+
+
 # SENT TO CLIENT
 
 
@@ -44,3 +57,8 @@ class ToastCommand(WebSocketCommandModel):
     command: Literal["toast"] = "toast"
     text: str
     toast_type: ToastType
+
+
+class ActionsRecordedCommand(WebSocketCommandModel):
+    command: Literal["actions_recorded"] = "actions_recorded"
+    actions: list[CommandActionConfig]

api/interface.py

@@ -273,27 +273,40 @@ class FeaturesConfig(BaseModel):
     remember_messages: Optional[int] = None
 
 
-class KeyPressConfig(BaseModel):
-    key: str
-    """The key the wingman will press when executing the command.  Use 'primary', 'secondary' or 'middle' for mouse buttons. Use 'scroll' to scroll."""
+class CommandKeyboardConfig(BaseModel):
+    hotkey: str
+    """The hotkey. Can be a single key like 'a' or a combination like 'ctrl+shift+a'."""
 
-    modifier: Optional[str] = None
-    """This will press "modifier + key" instead of just "modifier". Optional."""
+    hold: Optional[float] = None
+    """The duration the key will be pressed in seconds. Optional."""
 
-    wait: Optional[float] = None
-    """Wait time in seconds before pressing the next key. Optional."""
+
+class CommandMouseConfig(BaseModel):
+    button: Optional[str] = None
+    """The mouse button to press. Optional."""
 
     hold: Optional[float] = None
-    """The duration the key will be pressed in seconds. Optional."""
+    """The duration the button will be pressed in seconds. Optional."""
+
+    scroll: Optional[int] = None
+    """The amount to scroll up (positive integer) or down (negative integer), example 10 or -10. Must have 'scroll' as key above to work."""
 
-    scroll_amount: Optional[int] = None
-    """The amount of clicks to scroll up (positive integer) or down (negative integer), example 10 or -10. Must have 'scroll' as key above to work."""
+    move: Optional[list[int]] = None
+    """The x, y coordinates to move to relative to the current mouse position, expected [x,y] format in yaml.  Must have associated button press to work."""
 
-    moveto: Optional[list[int]] = None
+    move_to: Optional[list[int]] = None
     """The x, y coordinates to move the mouse to on the screen, expected [x,y] format in yaml.  Must have associated button press to work."""
 
-    moveto_relative: Optional[list[int]] = None
-    """The x, y coordinates to move to relative to the current mouse position, expected [x,y] format in yaml.  Must have associated button press to work."""
+
+class CommandActionConfig(BaseModel):
+    keyboard: Optional[CommandKeyboardConfig] = None
+    """The keyboard configuration for this action. Optional."""
+
+    wait: Optional[float] = None
+    """Wait time in seconds before pressing the next key. Optional."""
+
+    mouse: Optional[CommandMouseConfig] = None
+    """The mouse configuration for this action. Optional."""
 
     write: Optional[str] = None
     """The word or phrase to type, for example, to type text in a login screen.  Must have associated button press to work.  May need special formatting for special characters."""
@@ -311,8 +324,8 @@ class CommandConfig(BaseModel):
     """Optional: Faster - like Voice Attack! Provide phrases that will instantly activate the command (without AI roundtripping). You need to say the exact phrase to execute the command"""
     responses: Optional[list[str]] = None
     """Optional: Provide responses that will be used when the command is executed. A random one will be chosen (if multiple)."""
-    keys: Optional[list[KeyPressConfig]] = None
-    """The key or keys to press when the command is executed."""
+    actions: Optional[list[CommandActionConfig]] = None
+    """The actions to execute when the command is called. You can use keyboard, mouse and wait actions here."""
 
 
 class CustomWingmanClassConfig(BaseModel):
@@ -334,7 +347,7 @@ class NestedConfig(BaseModel):
     elevenlabs: ElevenlabsConfig
     azure: AzureConfig
     xvasynth: XVASynthTtsConfig
-    commands: list[CommandConfig] | None = None
+    commands: Optional[list[CommandConfig]] = None
 
 
 class WingmanConfig(NestedConfig):
@@ -355,9 +368,12 @@ def __setitem__(self, key, value):
     """The "friendly" name of this Wingman. Can be changed by the user."""
     description: str
     """A short description of this Wingman."""
-    record_key: str
+    record_key: Optional[str] = None
     """The "push-to-talk" key for this wingman. Keep it pressed while talking!
     Modifiers for this key are not supported yet. Don't use the same key for multiple wingmen!"""
+    record_mouse_button: Optional[str] = None
+    """The "push-to-talk" mouse button for this wingman. Keep it pressed while talking!
+    Don't use the same button for multiple wingmen!"""
 
 
 class Config(NestedConfig):