From 449b34089bc07a1a4968a03442c2a1d29fbba164 Mon Sep 17 00:00:00 2001 From: Animesh404 Date: Mon, 21 Oct 2024 23:12:48 +0530 Subject: [PATCH 1/2] WIP for window-event-race-condition --- ...add_a11y_event_remove_state_from_window.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py diff --git a/openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py b/openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py new file mode 100644 index 000000000..42207354b --- /dev/null +++ b/openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py @@ -0,0 +1,48 @@ +"""add_a11y_event_remove_state_from_window + +Revision ID: 481360d535a2 +Revises: 98505a067995 +Create Date: 2024-10-21 22:46:25.844631 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite +import openadapt + +# revision identifiers, used by Alembic. +revision = '481360d535a2' +down_revision = '98505a067995' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('a11y_event', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True), + sa.Column('handle', sa.Integer(), nullable=True), + sa.Column('data', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['handle', 'timestamp'], ['window_event.handle', 'window_event.timestamp'], name=op.f('fk_a11y_event_handle_window_event')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_a11y_event')) + ) + with op.batch_alter_table('window_event', schema=None) as batch_op: + batch_op.add_column(sa.Column('handle', sa.Integer(), nullable=True)) + batch_op.create_unique_constraint('uix_handle_timestamp', ['handle', 'timestamp']) + batch_op.drop_column('state') + batch_op.drop_column('window_id') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('window_event', schema=None) as batch_op: + batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True)) + batch_op.add_column(sa.Column('state', sqlite.JSON(), nullable=True)) + batch_op.drop_constraint('uix_handle_timestamp', type_='unique') + batch_op.drop_column('handle') + + op.drop_table('a11y_event') + # ### end Alembic commands ### From 3fa14bda941a8b58491da4bb45e0b34f413c4939 Mon Sep 17 00:00:00 2001 From: Animesh404 Date: Mon, 21 Oct 2024 23:21:15 +0530 Subject: [PATCH 2/2] wip: this still fails to stop recording --- ...dd_a11y_event_remove_state_from_window.py} | 19 ++- openadapt/app/dashboard/api/recordings.py | 3 + .../components/ActionEvent/ActionEvent.tsx | 52 ++++++- openadapt/app/dashboard/types/action-event.ts | 1 + openadapt/config.py | 2 +- openadapt/db/crud.py | 49 +++++++ openadapt/events.py | 41 +++++- openadapt/models.py | 127 +++++++++++++----- openadapt/record.py | 118 ++++++++++++++-- openadapt/visualize.py | 5 + openadapt/window/__init__.py | 21 +-- openadapt/window/_macos.py | 8 +- openadapt/window/_windows.py | 68 ++++++---- 13 files changed, 408 insertions(+), 106 deletions(-) rename openadapt/alembic/versions/{481360d535a2_add_a11y_event_remove_state_from_window.py => f9397786028d_add_a11y_event_remove_state_from_window.py} (76%) diff --git a/openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py b/openadapt/alembic/versions/f9397786028d_add_a11y_event_remove_state_from_window.py similarity index 76% rename from openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py rename to openadapt/alembic/versions/f9397786028d_add_a11y_event_remove_state_from_window.py index 42207354b..fdf6703db 100644 --- a/openadapt/alembic/versions/481360d535a2_add_a11y_event_remove_state_from_window.py +++ b/openadapt/alembic/versions/f9397786028d_add_a11y_event_remove_state_from_window.py @@ -1,8 +1,8 @@ """add_a11y_event_remove_state_from_window -Revision ID: 481360d535a2 +Revision ID: f9397786028d Revises: 98505a067995 -Create Date: 2024-10-21 22:46:25.844631 +Create Date: 2024-10-21 23:15:25.932599 """ from alembic import op @@ -11,7 +11,7 @@ import openadapt # revision identifiers, used by Alembic. -revision = '481360d535a2' +revision = 'f9397786028d' down_revision = '98505a067995' branch_labels = None depends_on = None @@ -19,6 +19,14 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.create_table('replay', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True), + sa.Column('strategy_name', sa.String(), nullable=True), + sa.Column('strategy_args', sa.JSON(), nullable=True), + sa.Column('git_hash', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_replay')) + ) op.create_table('a11y_event', sa.Column('id', sa.Integer(), nullable=False), sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True), @@ -30,8 +38,8 @@ def upgrade() -> None: with op.batch_alter_table('window_event', schema=None) as batch_op: batch_op.add_column(sa.Column('handle', sa.Integer(), nullable=True)) batch_op.create_unique_constraint('uix_handle_timestamp', ['handle', 'timestamp']) - batch_op.drop_column('state') batch_op.drop_column('window_id') + batch_op.drop_column('state') # ### end Alembic commands ### @@ -39,10 +47,11 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('window_event', schema=None) as batch_op: - batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True)) batch_op.add_column(sa.Column('state', sqlite.JSON(), nullable=True)) + batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True)) batch_op.drop_constraint('uix_handle_timestamp', type_='unique') batch_op.drop_column('handle') op.drop_table('a11y_event') + op.drop_table('replay') # ### end Alembic commands ### diff --git a/openadapt/app/dashboard/api/recordings.py b/openadapt/app/dashboard/api/recordings.py index 217024992..2f8ffabaf 100644 --- a/openadapt/app/dashboard/api/recordings.py +++ b/openadapt/app/dashboard/api/recordings.py @@ -11,6 +11,7 @@ from openadapt.models import Recording from openadapt.plotting import display_event from openadapt.utils import image2utf8, row2dict +from openadapt.visualize import dict2html class RecordingsAPI: @@ -112,6 +113,8 @@ def convert_to_str(event_dict: dict) -> dict: for action_event in action_events: event_dict = row2dict(action_event) + a11y_dict = row2dict(action_event.window_event.a11y_event) + event_dict["a11y_data"] = dict2html(a11y_dict) try: image = display_event(action_event) width, height = image.size diff --git a/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx b/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx index 69dc506c3..c3fa7ba5b 100644 --- a/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx +++ b/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx @@ -41,7 +41,7 @@ export const ActionEvent = ({ let content = ( - +
{typeof event.id === 'number' && ( @@ -65,12 +65,6 @@ export const ActionEvent = ({ {timeStampToDateString(event.screenshot_timestamp)} )} - {event.browser_event_timestamp && ( - - browser event timestamp - {timeStampToDateString(event.browser_event_timestamp)} - - )} window event timestamp {timeStampToDateString(event.window_event_timestamp)} @@ -142,6 +136,50 @@ export const ActionEvent = ({
+ {event.a11y_data && ( + + + + +
+ Accessibility Data +
+ + + + + +
+ +
+ + + + + )} diff --git a/openadapt/app/dashboard/types/action-event.ts b/openadapt/app/dashboard/types/action-event.ts index f99757bbb..a3155879d 100644 --- a/openadapt/app/dashboard/types/action-event.ts +++ b/openadapt/app/dashboard/types/action-event.ts @@ -6,6 +6,7 @@ export type ActionEvent = { screenshot_timestamp?: number; window_event_timestamp: number; browser_event_timestamp: number; + a11y_data: string; mouse_x: number | null; mouse_y: number | null; mouse_dx: number | null; diff --git a/openadapt/config.py b/openadapt/config.py index 432c44e5f..5e5a417d3 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -135,7 +135,7 @@ class SegmentationAdapter(str, Enum): # Record and replay EVENT_BUFFER_QUEUE_SIZE: int = 100 - RECORD_WINDOW_DATA: bool = True + RECORD_A11Y_DATA: bool = True RECORD_READ_ACTIVE_ELEMENT_STATE: bool RECORD_VIDEO: bool RECORD_AUDIO: bool diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index b0a9ff12d..2f31a4fa6 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -28,6 +28,7 @@ Screenshot, ScrubbedRecording, WindowEvent, + A11yEvent, copy_sa_instance, ) from openadapt.privacy.base import ScrubbingProvider @@ -287,6 +288,25 @@ def insert_recording(session: SaSession, recording_data: dict) -> Recording: return db_obj +def insert_a11y_event( + session: SaSession, + event_data: dict, +) -> None: + """Insert an a11y event into the database. + + Args: + session (sa.orm.Session): The database session. + event_data (dict): The data of the event + """ + handle = event_data["handle"] + a11y_data = event_data["a11y_data"] + timestamp = event_data["timestamp"] + a11y_event = A11yEvent(timestamp=timestamp, handle=handle, data=a11y_data) + + session.add(a11y_event) + session.commit() + + def delete_recording(session: SaSession, recording: Recording) -> None: """Remove the recording from the db. @@ -615,6 +635,35 @@ def get_window_events( ) +def get_a11y_events( + session: SaSession, + recording: Recording, +) -> list[A11yEvent]: + """Get accessibility events for a given recording. + + Args: + session (SaSession): The SQLAlchemy session. + recording (Recording): The recording object. + + Returns: + list[A11yEvent]: A list of accessibility events for the recording. + """ + return ( + session.query(A11yEvent) + .join( + WindowEvent, + (A11yEvent.handle == WindowEvent.handle) + & (A11yEvent.timestamp == WindowEvent.timestamp), + ) + .filter(WindowEvent.recording_id == recording.id) + .options( + joinedload(A11yEvent.window_event).joinedload(WindowEvent.recording), + ) + .order_by(A11yEvent.timestamp) + .all() + ) + + def get_browser_events(session: SaSession, recording: Recording) -> list[BrowserEvent]: """Get browser events for a given recording. diff --git a/openadapt/events.py b/openadapt/events.py index 5866a3656..0468a9f1f 100644 --- a/openadapt/events.py +++ b/openadapt/events.py @@ -46,6 +46,7 @@ def get_events( action_events = crud.get_action_events(db, recording) window_events = crud.get_window_events(db, recording) browser_events = crud.get_browser_events(db, recording) + a11y_events = crud.get_a11y_events(db, recording) screenshots = crud.get_screenshots(db, recording) browser_stats = browser.assign_browser_events(db, action_events, browser_events) @@ -67,11 +68,13 @@ def get_events( num_window_events = len(window_events) num_screenshots = len(screenshots) num_browser_events = len(browser_events) + num_a11y_events = len(a11y_events) num_action_events_raw = num_action_events num_window_events_raw = num_window_events num_screenshots_raw = num_screenshots num_browser_events_raw = num_browser_events + num_a11y_events_raw = num_a11y_events duration_raw = action_events[-1].timestamp - action_events[0].timestamp num_process_iters = 0 @@ -83,23 +86,27 @@ def get_events( f"{num_window_events=} " f"{num_screenshots=}" f"{num_browser_events=}" + f"{num_a11y_events=}" ) ( action_events, window_events, screenshots, browser_events, + a11y_events, ) = merge_events( action_events, window_events, screenshots, browser_events, + a11y_events, ) if ( len(action_events) == num_action_events and len(window_events) == num_window_events and len(screenshots) == num_screenshots and len(browser_events) == num_browser_events + and len(a11y_events) == num_a11y_events ): break num_process_iters += 1 @@ -107,6 +114,7 @@ def get_events( num_window_events = len(window_events) num_screenshots = len(screenshots) num_browser_events = len(browser_events) + num_a11y_events = len(a11y_events) if num_process_iters == MAX_PROCESS_ITERS: break @@ -131,6 +139,10 @@ def get_events( num_browser_events, num_browser_events_raw, ) + meta["num_a11y_events"] = format_num( + num_a11y_events, + num_a11y_events_raw, + ) duration = action_events[-1].timestamp - action_events[0].timestamp if len(action_events) > 1: @@ -823,10 +835,12 @@ def merge_events( window_events: list[models.WindowEvent], screenshots: list[models.Screenshot], browser_events: list[models.BrowserEvent], + a11y_events: list[models.A11yEvent], ) -> tuple[ list[models.ActionEvent], list[models.WindowEvent], list[models.Screenshot], + list[models.A11yEvent], ]: """Merge redundant action events, window events, and screenshots. @@ -834,6 +848,7 @@ def merge_events( action_events (list): The list of action events. window_events (list): The list of window events. screenshots (list): The list of screenshots. + a11y_events (list): The list of a11y events. Returns: tuple: A tuple containing the processed action events, window events, @@ -843,13 +858,19 @@ def merge_events( num_window_events = len(window_events) num_screenshots = len(screenshots) num_browser_events = len(browser_events) + num_a11y_events = len(a11y_events) num_total = ( - num_action_events + num_window_events + num_screenshots + num_browser_events + num_action_events + + num_window_events + + num_screenshots + + num_browser_events + + num_a11y_events ) logger.info( "before" f" {num_action_events=} {num_window_events=}" f" {num_screenshots=} {num_browser_events=} " + f"{num_a11y_events=} " f"{num_total=}" ) process_fns = [ @@ -893,12 +914,22 @@ def merge_events( action_events, "browser_event_timestamp", ) + a11y_events = discard_unused_events( + a11y_events, + action_events, + "timestamp", + ) num_action_events_ = len(action_events) num_window_events_ = len(window_events) num_screenshots_ = len(screenshots) num_browser_events_ = len(browser_events) + num_a11y_events_ = len(a11y_events) num_total_ = ( - num_action_events_ + num_window_events_ + num_screenshots_ + num_browser_events_ + num_action_events_ + + num_window_events_ + + num_screenshots_ + + num_browser_events_ + + num_a11y_events_ ) pct_action_events = num_action_events_ / num_action_events pct_window_events = num_window_events_ / num_window_events @@ -906,15 +937,17 @@ def merge_events( pct_browser_events = ( num_browser_events_ / num_browser_events if num_browser_events else None ) + pct_a11y_events = num_a11y_events_ / num_a11y_events pct_total = num_total_ / num_total logger.info( "after" f" {num_action_events_=} {num_window_events_=}" f" {num_screenshots_=} {num_browser_events_=}" + f"{num_a11y_events_=} {num_a11y_events_=} " f" {num_total_=}" ) logger.info( f"{pct_action_events=} {pct_window_events=} {pct_screenshots=}" - f" {pct_browser_events=} {pct_total=}" + f" {pct_browser_events=} {pct_a11y_events=} {pct_total=}" ) - return action_events, window_events, screenshots, browser_events + return action_events, window_events, screenshots, browser_events, a11y_events diff --git a/openadapt/models.py b/openadapt/models.py index 1df82c45e..4b60fcbd1 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -504,6 +504,25 @@ def to_prompt_dict(self) -> dict[str, Any]: return action_dict +class A11yEvent(db.Base): + """Class representing an accessibility (a11y) event in the database.""" + + __tablename__ = "a11y_event" + + id = sa.Column(sa.Integer, primary_key=True) + timestamp = sa.Column(ForceFloat) + handle = sa.Column(sa.Integer) + data = sa.Column(sa.JSON) + + __table_args__ = ( + sa.ForeignKeyConstraint( + ["handle", "timestamp"], ["window_event.handle", "window_event.timestamp"] + ), + ) + + window_event = sa.orm.relationship("WindowEvent", back_populates="a11y_event") + + class WindowEvent(db.Base): """Class representing a window event in the database.""" @@ -513,34 +532,53 @@ class WindowEvent(db.Base): recording_timestamp = sa.Column(ForceFloat) recording_id = sa.Column(sa.ForeignKey("recording.id")) timestamp = sa.Column(ForceFloat) - state = sa.Column(sa.JSON) title = sa.Column(sa.String) left = sa.Column(sa.Integer) top = sa.Column(sa.Integer) width = sa.Column(sa.Integer) height = sa.Column(sa.Integer) - window_id = sa.Column(sa.String) + handle = sa.Column(sa.Integer) + + __table_args__ = ( + sa.UniqueConstraint("handle", "timestamp", name="uix_handle_timestamp"), + ) recording = sa.orm.relationship("Recording", back_populates="window_events") action_events = sa.orm.relationship("ActionEvent", back_populates="window_event") + a11y_event = sa.orm.relationship( + "A11yEvent", uselist=False, back_populates="window_event" + ) @classmethod def get_active_window_event( cls: "WindowEvent", - # TODO: rename to include_a11y_data - include_window_data: bool = True, + include_a11y_data: bool = True, ) -> "WindowEvent": """Get the active window event. Args: - include_window_data (bool): whether to include a11y data. + include_a11y_data (bool): whether to include a11y data. Returns: (WindowEvent) the active window event. """ from openadapt import window - return WindowEvent(**window.get_active_window_data(include_window_data)) + window_event_data = window.get_active_window_data(include_a11y_data) + a11y_event = None + + if include_a11y_data: + a11y_event_data = window_event_data.get("state") + window_event_data.pop("state", None) + a11y_event_handle = window_event_data.get("handle") + a11y_event = A11yEvent(data=a11y_event_data, handle=a11y_event_handle) + + window_event = WindowEvent(**window_event_data) + + if a11y_event: + window_event.a11y_event = a11y_event + + return window_event def scrub(self, scrubber: ScrubbingProvider | TextScrubbingMixin) -> None: """Scrub the window event.""" @@ -563,6 +601,7 @@ def to_prompt_dict( Returns: dictionary containing relevant properties from the WindowEvent. """ + a11y_data = self.a11y_event.data window_dict = deepcopy( { key: val @@ -596,37 +635,51 @@ def to_prompt_dict( window_dict.pop("width") window_dict.pop("height") - if "state" in window_dict: - if include_data: - key_suffixes = [ - "value", - "h", - "w", - "x", - "y", - "description", - "title", - "help", - ] - if sys.platform == "win32": - logger.warning( - "key_suffixes have not yet been defined on Windows." - "You can help by uncommenting the lines below and pasting " - "the contents of the window_dict into a new GitHub Issue." - ) - # from pprint import pformat - # logger.info(f"window_dict=\n{pformat(window_dict)}") - # import ipdb; ipdb.set_trace() - window_state = window_dict["state"] - window_state["data"] = utils.clean_dict( - utils.filter_keys( - window_state["data"], - key_suffixes, - ) - ) - else: - window_dict["state"].pop("data") - window_dict["state"].pop("meta") + if a11y_data: + window_dict = deepcopy( + { + key: val + for key, val in utils.row2dict( + self.a11y_event, follow=False + ).items() + if val not in EMPTY_VALS + and not key.endswith("timestamp") + and not key.endswith("id") + # and not isinstance(getattr(models.WindowEvent, key), property) + } + ) + if "a11y_data" in window_dict: + if include_data: + key_suffixes = [ + "value", + "h", + "w", + "x", + "y", + "description", + "title", + "help", + ] + if sys.platform == "win32": + logger.warning( + "key_suffixes have not yet been defined on Windows." + "You can help by uncommenting the lines below and pasting " + "the contents of the window_dict into a new GitHub Issue." + ) + # from pprint import pformat + # logger.info(f"window_dict=\n{pformat(window_dict)}") + # import ipdb; ipdb.set_trace() + if "a11y_data" in window_dict: + window_state = window_dict["a11y_data"] + window_state["data"] = utils.clean_dict( + utils.filter_keys( + window_state["data"], + key_suffixes, + ) + ) + else: + window_dict["a11y_data"].pop("data") + window_dict["a11y_data"].pop("meta") return window_dict diff --git a/openadapt/record.py b/openadapt/record.py index 27eb9e578..ecb10dd4f 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -47,7 +47,7 @@ Event = namedtuple("Event", ("timestamp", "type", "data")) -EVENT_TYPES = ("screen", "action", "window", "browser") +EVENT_TYPES = ("screen", "action", "window", "browser", "a11y") LOG_LEVEL = "INFO" # whether to write events of each type in a separate process PROC_WRITE_BY_EVENT_TYPE = { @@ -56,6 +56,7 @@ "action": True, "window": True, "browser": True, + "a11y": True, } PLOT_PERFORMANCE = config.PLOT_PERFORMANCE NUM_MEMORY_STATS_TO_LOG = 3 @@ -133,6 +134,7 @@ def process_events( action_write_q: sq.SynchronizedQueue, window_write_q: sq.SynchronizedQueue, browser_write_q: sq.SynchronizedQueue, + a11y_write_q: sq.SynchronizedQueue, video_write_q: sq.SynchronizedQueue, perf_q: sq.SynchronizedQueue, recording: Recording, @@ -142,6 +144,7 @@ def process_events( num_action_events: multiprocessing.Value, num_window_events: multiprocessing.Value, num_browser_events: multiprocessing.Value, + num_a11y_events: multiprocessing.Value, num_video_events: multiprocessing.Value, ) -> None: """Process events from the event queue and write them to write queues. @@ -152,6 +155,7 @@ def process_events( action_write_q: A queue for writing action events. window_write_q: A queue for writing window events. browser_write_q: A queue for writing browser events, + a11y_write_q: A queue for writing a11y events. video_write_q: A queue for writing video events. perf_q: A queue for collecting performance data. recording: The recording object. @@ -161,6 +165,7 @@ def process_events( num_action_events: A counter for the number of action events. num_window_events: A counter for the number of window events. num_browser_events: A counter for the number of browser events. + num_a11y_events: A counter for the number of a11y events. num_video_events: A counter for the number of video events. """ utils.set_start_time(recording.timestamp) @@ -173,6 +178,7 @@ def process_events( prev_saved_screen_timestamp = 0 prev_saved_window_timestamp = 0 started = False + window_events_waiting_for_a11y = queue.Queue() while not terminate_processing.is_set() or not event_q.empty(): event = event_q.get() if not started: @@ -208,6 +214,8 @@ def process_events( num_video_events.value += 1 elif event.type == "window": prev_window_event = event + if config.RECORD_A11Y_DATA: + window_events_waiting_for_a11y.put_nowait(event) elif event.type == "browser": if config.RECORD_BROWSER_EVENTS: process_event( @@ -270,6 +278,24 @@ def process_events( ) num_window_events.value += 1 prev_saved_window_timestamp = prev_window_event.timestamp + elif event.type == "a11y": + try: + window_event = window_events_waiting_for_a11y.get_nowait() + except queue.Empty as exc: + logger.warning( + f"Discarding A11yEvent with no corresponding WindowEvent: {exc}" + ) + continue + event.data["timestamp"] = window_event.timestamp + process_event( + event, + a11y_write_q, + write_a11y_event, + recording, + perf_q, + ) + num_a11y_events.value += 1 + logger.debug(f"A11yEvent processed: {event}") else: raise Exception(f"unhandled {event.type=}") del prev_event @@ -342,6 +368,26 @@ def write_window_event( perf_q.put((event.type, event.timestamp, utils.get_timestamp())) +def write_a11y_event( + db: crud.SaSession, + recording: Recording, + event: Event, + perf_q: sq.SynchronizedQueue, +) -> None: + """Write an a11y event to the database and update the performance queue. + + Args: + db: The database session. + recording: The recording object. + event: An a11y event to be written. + perf_q: A queue for collecting performance data. + """ + assert event.type == "a11y", event + data = event.data + crud.insert_a11y_event(db, data) + perf_q.put((event.type, event.timestamp, utils.get_timestamp())) + + def write_browser_event( db: crud.SaSession, recording: Recording, @@ -741,6 +787,8 @@ def read_window_events( terminate_processing: multiprocessing.Event, recording: Recording, started_counter: multiprocessing.Value, + read_a11y_data: bool, + event_name: str, ) -> None: """Read window events and add them to the event queue. @@ -749,6 +797,8 @@ def read_window_events( terminate_processing: An event to signal the termination of the process. recording: The recording object. started_counter: Value to increment once started. + read_a11y_data: Whether to read a11y_data. + event_name: The name of the event. """ utils.set_start_time(recording.timestamp) @@ -756,7 +806,7 @@ def read_window_events( prev_window_data = {} started = False while not terminate_processing.is_set(): - window_data = window.get_active_window_data() + window_data = window.get_active_window_data(include_a11y_data=read_a11y_data) if not window_data: continue @@ -765,9 +815,8 @@ def read_window_events( started_counter.value += 1 started = True - if window_data["title"] != prev_window_data.get("title") or window_data[ - "window_id" - ] != prev_window_data.get("window_id"): + # for logging purposes only + if window_data["handle"] != prev_window_data.get("handle"): # TODO: fix exception sometimes triggered by the next line on win32: # File "\Python39\lib\threading.py" line 917, in run # File "...\openadapt\record.py", line 277, in read window events @@ -775,15 +824,19 @@ def read_window_events( # File "...\env\lib\site-packages\loguru\_logger.py", line 1964, in _log # for handler in core.handlers.values): # RuntimeError: dictionary changed size during iteration - _window_data = window_data - _window_data.pop("state") - logger.info(f"{_window_data=}") + window_data["timestamp"] = utils.get_timestamp() + if read_a11y_data: + _window_data = window_data.copy() + _window_data.pop("a11y_data") + logger.info(f"{_window_data=}") + else: + logger.info(f"{window_data=}") if window_data != prev_window_data: - logger.debug("Queuing window event for writing") + logger.debug("Queuing {event_name} event for writing") event_q.put( Event( utils.get_timestamp(), - "window", + event_name, window_data, ) ) @@ -1318,6 +1371,7 @@ def record( action_write_q = sq.SynchronizedQueue() window_write_q = sq.SynchronizedQueue() browser_write_q = sq.SynchronizedQueue() + a11y_write_q = sq.SynchronizedQueue() video_write_q = sq.SynchronizedQueue() # TODO: save write times to DB; display performance plot in visualize.py perf_q = sq.SynchronizedQueue() @@ -1328,11 +1382,32 @@ def record( window_event_reader = threading.Thread( target=read_window_events, - args=(event_q, terminate_processing, recording, started_counter), + args=( + event_q, + terminate_processing, + recording, + started_counter, + False, + "window", + ), ) window_event_reader.start() task_by_name["window_event_reader"] = window_event_reader + if config.RECORD_A11Y_DATA: + a11y_event_reader = threading.Thread( + target=read_window_events, + args=( + event_q, + terminate_processing, + recording, + started_counter, + True, + "a11y", + ), + ) + a11y_event_reader.start() + task_by_name["a11y_event_reader"] = a11y_event_reader if config.RECORD_BROWSER_EVENTS: browser_event_reader = threading.Thread( target=run_browser_event_server, @@ -1366,6 +1441,7 @@ def record( num_screen_events = multiprocessing.Value("i", 0) num_window_events = multiprocessing.Value("i", 0) num_browser_events = multiprocessing.Value("i", 0) + num_a11y_events = multiprocessing.Value("i", 0) num_video_events = multiprocessing.Value("i", 0) event_processor = threading.Thread( @@ -1376,6 +1452,7 @@ def record( action_write_q, window_write_q, browser_write_q, + a11y_write_q, video_write_q, perf_q, recording, @@ -1385,6 +1462,7 @@ def record( num_action_events, num_window_events, num_browser_events, + num_a11y_events, num_video_events, ), ) @@ -1407,6 +1485,23 @@ def record( screen_event_writer.start() task_by_name["screen_event_writer"] = screen_event_writer + if config.RECORD_A11Y_DATA: + a11y_event_writer = multiprocessing.Process( + target=write_events, + args=( + "a11y", + write_a11y_event, + a11y_write_q, + num_a11y_events, + perf_q, + recording, + terminate_processing, + started_counter, + ), + ) + a11y_event_writer.start() + task_by_name["a11y_event_writer"] = a11y_event_writer + if config.RECORD_BROWSER_EVENTS: browser_event_writer = multiprocessing.Process( target=write_events, @@ -1568,6 +1663,7 @@ def join_tasks(task_names: list[str]) -> None: "event_processor", "screen_event_writer", "browser_event_writer", + "a11y_event_writer", "action_event_writer", "window_event_writer", "video_writer", diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 240362b14..83c62d983 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -342,11 +342,13 @@ def main( action_event_dict = row2dict(action_event) window_event_dict = row2dict(action_event.window_event) browser_event_dict = row2dict(action_event.browser_event) + a11y_event_dict = row2dict(action_event.window_event.a11y_event) if SCRUB: action_event_dict = scrub.scrub_dict(action_event_dict) window_event_dict = scrub.scrub_dict(window_event_dict) browser_event_dict = scrub.scrub_dict(browser_event_dict) + a11y_event_dict = scrub.scrub_dict(a11y_event_dict) rows.append( [ @@ -376,6 +378,9 @@ def main( {dict2html(window_event_dict , None)}
+ + {dict2html(a11y_event_dict, None)} +
{dict2html(browser_event_dict , None)}
diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index fe0cb9e9f..64340a0c0 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -19,18 +19,18 @@ def get_active_window_data( - include_window_data: bool = config.RECORD_WINDOW_DATA, + include_a11y_data: bool = config.RECORD_A11Y_DATA, ) -> dict[str, Any] | None: """Get data of the active window. Args: - include_window_data (bool): whether to include a11y data. + include_a11y_data (bool): whether to include a11y data. Returns: dict or None: A dictionary containing information about the active window, or None if the state is not available. """ - state = get_active_window_state(include_window_data) + state = get_active_window_state(include_a11y_data) if not state: return {} title = state["title"] @@ -38,29 +38,32 @@ def get_active_window_data( top = state["top"] width = state["width"] height = state["height"] - window_id = state["window_id"] + handle = state["handle"] window_data = { "title": title, "left": left, "top": top, "width": width, "height": height, - "window_id": window_id, - "state": state, + "handle": handle, } + + if include_a11y_data: + window_data["a11y_data"] = state + return window_data -def get_active_window_state(read_window_data: bool) -> dict | None: +def get_active_window_state(read_a11y_data: bool) -> dict | None: """Get the state of the active window. Returns: - dict or None: A dictionary containing the state of the active window, + dict or None: A dictionary containing the a11y_data of the active window, or None if the state is not available. """ # TODO: save window identifier (a window's title can change, or try: - return impl.get_active_window_state(read_window_data) + return impl.get_active_window_state(read_a11y_data) except Exception as exc: logger.warning(f"{exc=}") return None diff --git a/openadapt/window/_macos.py b/openadapt/window/_macos.py index 3b9fd7625..327389ed8 100644 --- a/openadapt/window/_macos.py +++ b/openadapt/window/_macos.py @@ -13,7 +13,7 @@ from openadapt.custom_logger import logger -def get_active_window_state(read_window_data: bool) -> dict | None: +def get_active_window_state(read_a11y_data: bool) -> dict | None: """Get the state of the active window. Returns: @@ -23,7 +23,7 @@ def get_active_window_state(read_window_data: bool) -> dict | None: # pywinctl performance on macOS is unusable, see: # https://github.com/Kalmat/PyWinCtl/issues/29 meta = get_active_window_meta() - if read_window_data: + if read_a11y_data: data = get_window_data(meta) else: data = {} @@ -33,7 +33,7 @@ def get_active_window_state(read_window_data: bool) -> dict | None: ] title_parts = [part for part in title_parts if part] title = " ".join(title_parts) - window_id = meta["kCGWindowNumber"] + handle = meta["kCGWindowNumber"] bounds = meta["kCGWindowBounds"] left = bounds["X"] top = bounds["Y"] @@ -45,7 +45,7 @@ def get_active_window_state(read_window_data: bool) -> dict | None: "top": top, "width": width, "height": height, - "window_id": window_id, + "handle": handle, "meta": meta, "data": data, } diff --git a/openadapt/window/_windows.py b/openadapt/window/_windows.py index 9a49d6b7d..10caf8f5a 100644 --- a/openadapt/window/_windows.py +++ b/openadapt/window/_windows.py @@ -3,11 +3,12 @@ import time import pywinauto +import pygetwindow as gw from openadapt.custom_logger import logger -def get_active_window_state(read_window_data: bool) -> dict: +def get_active_window_state(read_a11y_data: bool) -> dict: """Get the state of the active window. Returns: @@ -20,35 +21,46 @@ def get_active_window_state(read_window_data: bool) -> dict: - "height": Height of the active window. - "meta": Meta information of the active window. - "data": None (to be filled with window data). - - "window_id": ID of the active window. + - "handle": ID of the active window. """ - # catch specific exceptions, when except happens do log.warning - try: - active_window = get_active_window() - except RuntimeError as e: - logger.warning(e) - return {} - meta = get_active_window_meta(active_window) - rectangle_dict = dictify_rect(meta["rectangle"]) - if read_window_data: + if read_a11y_data: + try: + active_window, handle = get_active_window() + except RuntimeError as e: + logger.warning(e) + return {} + meta = get_active_window_meta(active_window) + rectangle_dict = dictify_rect(meta["rectangle"]) data = get_element_properties(active_window) + state = { + "title": meta["texts"][0], + "left": meta["rectangle"].left, + "top": meta["rectangle"].top, + "width": meta["rectangle"].width(), + "height": meta["rectangle"].height(), + "meta": {**meta, "rectangle": rectangle_dict}, + "data": data, + "handle": handle, + } + try: + pickle.dumps(state) + except Exception as exc: + logger.warning(f"{exc=}") + state.pop("data") else: - data = {} - state = { - "title": meta["texts"][0], - "left": meta["rectangle"].left, - "top": meta["rectangle"].top, - "width": meta["rectangle"].width(), - "height": meta["rectangle"].height(), - "meta": {**meta, "rectangle": rectangle_dict}, - "data": data, - "window_id": meta["control_id"], - } - try: - pickle.dumps(state) - except Exception as exc: - logger.warning(f"{exc=}") - state.pop("data") + try: + active_window = gw.getActiveWindow() + except RuntimeError as e: + logger.warning(e) + return {} + state = { + "title": active_window.title if active_window.title else "None", + "left": active_window.left, + "top": active_window.top, + "width": active_window.width, + "height": active_window.height, + "handle": active_window._hWnd, + } return state @@ -96,7 +108,7 @@ def get_active_window() -> pywinauto.application.WindowSpecification: """ app = pywinauto.application.Application(backend="uia").connect(active_only=True) window = app.top_window() - return window.wrapper_object() + return [window.wrapper_object(), window.handle] def get_element_properties(element: pywinauto.application.WindowSpecification) -> dict: