diff --git a/castervoice/rules/core/text_manipulation_rules/accessibility_text.py b/castervoice/rules/core/text_manipulation_rules/accessibility_text.py new file mode 100644 index 000000000..503bbac80 --- /dev/null +++ b/castervoice/rules/core/text_manipulation_rules/accessibility_text.py @@ -0,0 +1,320 @@ +import re + +from castervoice.lib.actions import Key, Text + +try: + from dragonfly.accessibility import get_accessibility_controller as dragonfly_get_accessibility_controller +except Exception: + try: + from dragonfly import get_accessibility_controller as dragonfly_get_accessibility_controller + except Exception: + dragonfly_get_accessibility_controller = None + + +TEXT_NODE_DELIMITER = u"\u00a6" + + +def _previous_boundary(text, index): + position = max(0, min(index, len(text))) - 1 + while position >= 0: + character = text[position] + if character == "\n": + if position > 0 and text[position - 1] == "\r": + return (position - 1, position + 1) + return (position, position + 1) + if character == "\r": + if position + 1 < len(text) and text[position + 1] == "\n": + return (position, position + 2) + return (position, position + 1) + if character == TEXT_NODE_DELIMITER: + return (position, position + 1) + position -= 1 + return None + + +def _next_boundary(text, index): + position = max(0, min(index, len(text))) + while position < len(text): + character = text[position] + if character == "\r": + if position + 1 < len(text) and text[position + 1] == "\n": + return (position, position + 2) + return (position, position + 1) + if character == "\n" or character == TEXT_NODE_DELIMITER: + return (position, position + 1) + position += 1 + return None + + +def _window_start(text, cursor, number_of_lines_to_search): + start = cursor + lines_to_include = max(0, number_of_lines_to_search) + 1 + for _ in range(lines_to_include): + boundary = _previous_boundary(text, start) + if boundary is None: + return 0 + start = boundary[0] + return boundary[1] + + +def _window_end(text, cursor, number_of_lines_to_search): + end = cursor + lines_to_include = max(0, number_of_lines_to_search) + 1 + for _ in range(lines_to_include): + boundary = _next_boundary(text, end) + if boundary is None: + return len(text) + end = boundary[1] + return boundary[0] + + +def _phrase_pattern(phrase, dictation_versus_character): + if dictation_versus_character == "character": + return re.escape(phrase) + if dictation_versus_character == "dictation": + # Keep the same word-boundary behavior as text_manipulation_support. + return r"(?:[^A-Za-z]|\A)({})(?:[^A-Za-z]|\Z)".format(phrase.lower()) + raise ValueError("dictation_versus_character must be 'character' or 'dictation'") + + +def get_start_end_position(text, phrase, direction, occurrence_number, dictation_versus_character): + pattern = _phrase_pattern(phrase, dictation_versus_character) + lowered_text = text.lower() + matches = re.finditer(pattern, lowered_text) + if dictation_versus_character == "character": + ranges = [(match.start(), match.end()) for match in matches] + else: + ranges = [(match.start(1), match.end(1)) for match in matches] + + if not ranges: + return None + + try: + if direction == "left": + return ranges[-1 * occurrence_number] + if direction == "right": + return ranges[occurrence_number - 1] + except IndexError: + return None + + return None + + +def _get_controller(): + if dragonfly_get_accessibility_controller is None: + return None + try: + return dragonfly_get_accessibility_controller() + except Exception: + return None + + +def _get_focused_text(accessibility_context): + focused = getattr(accessibility_context, "focused", None) + if not focused: + return None + try: + if hasattr(focused, "is_editable") and not focused.is_editable(): + return None + focused_text = focused.as_text() + except Exception: + return None + + if not focused_text: + return None + if getattr(focused_text, "cursor", None) is None: + return None + if getattr(focused_text, "expanded_text", None) is None: + return None + return focused_text + + +def _snapshot(controller): + if not controller or not getattr(controller, "os_controller", None): + return None + + def capture(accessibility_context): + focused_text = _get_focused_text(accessibility_context) + if not focused_text: + return None + text = focused_text.expanded_text + cursor = max(0, min(focused_text.cursor, len(text))) + return {"text": text, "cursor": cursor} + + try: + return controller.os_controller.run_sync(capture) + except Exception: + return None + + +def _controller_and_snapshot(): + controller = _get_controller() + snapshot = _snapshot(controller) + if not snapshot: + return (None, None) + return (controller, snapshot) + + +def _search_window(snapshot, direction, number_of_lines_to_search): + text = snapshot["text"] + cursor = snapshot["cursor"] + if direction == "left": + start = _window_start(text, cursor, number_of_lines_to_search) + return (text[start:cursor], start) + if direction == "right": + end = _window_end(text, cursor, number_of_lines_to_search) + return (text[cursor:end], cursor) + return (None, None) + + +def _target_range(snapshot, phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + window_text, base_offset = _search_window(snapshot, direction, number_of_lines_to_search) + if window_text is None: + return None + + match = get_start_end_position(window_text, phrase, direction, occurrence_number, + dictation_versus_character) + if not match: + return None + return (base_offset + match[0], base_offset + match[1]) + + +def _until_range(snapshot, phrase, direction, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + target = _target_range(snapshot, phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return None + + cursor = snapshot["cursor"] + left_index, right_index = target + if direction == "left": + target_index = left_index if before_after == "before" else right_index + elif direction == "right": + target_index = right_index if before_after == "after" else left_index + else: + return None + return (min(cursor, target_index), max(cursor, target_index)) + + +def _select_range(controller, start, end): + if start == end: + return False + + def select(accessibility_context): + focused_text = _get_focused_text(accessibility_context) + if not focused_text: + return False + focused_text.select_range(start, end) + return True + + try: + return bool(controller.os_controller.run_sync(select)) + except Exception: + return False + + +def _set_cursor(controller, offset): + def move(accessibility_context): + focused_text = _get_focused_text(accessibility_context) + if not focused_text: + return False + focused_text.set_cursor(offset) + return True + + try: + return bool(controller.os_controller.run_sync(move)) + except Exception: + return False + + +def _replace_range(controller, start, end, replacement): + if not _select_range(controller, start, end): + return False + try: + if replacement: + Text(str(replacement).replace("%", "%%")).execute() + else: + Key("backspace").execute() + except Exception: + # Once the accessibility selection has been made, do not fall back into + # the clipboard path and risk acting on the already-selected range twice. + return True + return True + + +def select_phrase(phrase, direction, number_of_lines_to_search, occurrence_number, + dictation_versus_character): + controller, snapshot = _controller_and_snapshot() + if not controller: + return False + target = _target_range(snapshot, phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return False + return _select_range(controller, target[0], target[1]) + + +def select_until_phrase(direction, phrase, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + controller, snapshot = _controller_and_snapshot() + if not controller: + return False + target = _until_range(snapshot, phrase, direction, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return False + return _select_range(controller, target[0], target[1]) + + +def move_until_phrase(direction, before_after, phrase, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + controller, snapshot = _controller_and_snapshot() + if not controller: + return False + target = _target_range(snapshot, phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return False + return _set_cursor(controller, target[0] if before_after == "before" else target[1]) + + +def replace_phrase_with_phrase(replaced_phrase, replacement_phrase, direction, + number_of_lines_to_search, occurrence_number, + dictation_versus_character): + controller, snapshot = _controller_and_snapshot() + if not controller: + return False + target = _target_range(snapshot, replaced_phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return False + return _replace_range(controller, target[0], target[1], replacement_phrase) + + +def remove_phrase_from_text(phrase, direction, number_of_lines_to_search, occurrence_number, + dictation_versus_character): + controller, snapshot = _controller_and_snapshot() + if not controller: + return False + target = _target_range(snapshot, phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return False + start, end = target + if dictation_versus_character != "character" and start > 0 and snapshot["text"][start - 1] == " ": + start -= 1 + return _replace_range(controller, start, end, "") + + +def delete_until_phrase(direction, phrase, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + controller, snapshot = _controller_and_snapshot() + if not controller: + return False + target = _until_range(snapshot, phrase, direction, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character) + if not target: + return False + return _replace_range(controller, target[0], target[1], "") diff --git a/castervoice/rules/core/text_manipulation_rules/text_manipulation_support.py b/castervoice/rules/core/text_manipulation_rules/text_manipulation_support.py index 293407a71..23cd038bf 100644 --- a/castervoice/rules/core/text_manipulation_rules/text_manipulation_support.py +++ b/castervoice/rules/core/text_manipulation_rules/text_manipulation_support.py @@ -4,6 +4,11 @@ from castervoice.lib import context from castervoice.lib.actions import Key +try: + from castervoice.rules.core.text_manipulation_rules import accessibility_text +except ImportError: + import accessibility_text + contexts = { "texstudio": AppContext(executable="texstudio"), "lyx": AppContext(executable="lyx"), @@ -159,12 +164,16 @@ def copypaste_replace_phrase_with_phrase(replaced_phrase, replacement_phrase, di # "up" and "down" get treated just as the "left" and "right" # except that the default number of lines to search get set to three instead of zero number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) + replaced_phrase = str(replaced_phrase) + replacement_phrase = str(replacement_phrase) + if accessibility_text.replace_phrase_with_phrase(replaced_phrase, replacement_phrase, direction, + number_of_lines_to_search, occurrence_number, + dictation_versus_character): + return application = get_application() selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) if not selected_text: - return - replaced_phrase = str(replaced_phrase) - replacement_phrase = str(replacement_phrase) + return new_text = replace_phrase_with_phrase(selected_text, replaced_phrase, replacement_phrase, direction, occurrence_number, dictation_versus_character) if not new_text: # replaced_phrase not found @@ -184,16 +193,20 @@ def copypaste_change_phrase_capitalization(phrase, direction, number_of_lines_to # "up" and "down" get treated just as the "left" and "right" # except that the default number of lines to search get set to three instead of zero number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) - application = get_application() - selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) - if not selected_text: - return replaced_phrase = phrase if letter_size == "lower": replacement_phrase = phrase[0].lower() else: replacement_phrase = phrase[0].upper() replacement_phrase += phrase[1:] + if accessibility_text.replace_phrase_with_phrase(str(replaced_phrase), str(replacement_phrase), direction, + number_of_lines_to_search, occurrence_number, + dictation_versus_character): + return + application = get_application() + selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) + if not selected_text: + return new_text = replace_phrase_with_phrase(selected_text, replaced_phrase, replacement_phrase, direction, occurrence_number, dictation_versus_character) if not new_text: # replaced_phrase not found @@ -232,11 +245,14 @@ def remove_phrase_from_text(text, phrase, direction, occurrence_number, dictatio def copypaste_remove_phrase_from_text(phrase, direction, number_of_lines_to_search, occurrence_number, dictation_versus_character): if direction == "up" or direction == "down": number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) + phrase = str(phrase) + if accessibility_text.remove_phrase_from_text(phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + return application = get_application() selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) if not selected_text: return - phrase = str(phrase) new_text = remove_phrase_from_text(selected_text, phrase, direction, occurrence_number, dictation_versus_character) if not new_text: # phrase not found @@ -281,7 +297,6 @@ def delete_until_phrase(text, phrase, direction, before_after, occurrence_number def copypaste_delete_until_phrase(direction, phrase, number_of_lines_to_search, before_after, occurrence_number, dictation_versus_character): if direction == "up" or direction == "down": number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) - application = get_application() if not before_after: # default to delete all the way through the phrase not just up until it if direction == "left": @@ -289,10 +304,15 @@ def copypaste_delete_until_phrase(direction, phrase, number_of_lines_to_search, if direction == "right": before_after = "after" + phrase = str(phrase) + if accessibility_text.delete_until_phrase(direction, phrase, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + return + + application = get_application() selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) if not selected_text: return - phrase = str(phrase) new_text = delete_until_phrase(selected_text, phrase, direction, before_after, occurrence_number, dictation_versus_character) if new_text is None: @@ -318,7 +338,6 @@ def copypaste_delete_until_phrase(direction, phrase, number_of_lines_to_search, def move_until_phrase(direction, before_after, phrase, number_of_lines_to_search, occurrence_number, dictation_versus_character): if direction == "up" or direction == "down": number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) - application = get_application() if not before_after: # default to whatever is closest to the cursor if direction == "left": @@ -328,11 +347,16 @@ def move_until_phrase(direction, before_after, phrase, number_of_lines_to_search elif before_after not in ("before", "after"): print("before_after must be either 'before' or 'after' (got '{}')".format(before_after)) return - + + phrase = str(phrase) + if accessibility_text.move_until_phrase(direction, before_after, phrase, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + return + + application = get_application() selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) if not selected_text: return - phrase = str(phrase) match_index = get_start_end_position(selected_text, phrase, direction, occurrence_number, dictation_versus_character) if match_index: left_index, right_index = match_index @@ -409,11 +433,14 @@ def move_until_phrase(direction, before_after, phrase, number_of_lines_to_search def select_phrase(phrase, direction, number_of_lines_to_search, occurrence_number, dictation_versus_character): if direction == "up" or direction == "down": number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) + phrase = str(phrase) + if accessibility_text.select_phrase(phrase, direction, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + return application = get_application() selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) if not selected_text: return - phrase = str(phrase) match_index = get_start_end_position(selected_text, phrase, direction, occurrence_number, dictation_versus_character) if match_index: left_index, right_index = match_index @@ -463,7 +490,6 @@ def select_phrase(phrase, direction, number_of_lines_to_search, occurrence_numbe def select_until_phrase(direction, phrase, before_after, number_of_lines_to_search, occurrence_number, dictation_versus_character): if direction == "up" or direction == "down": number_of_lines_to_search, direction = deal_with_up_down_directions(direction, number_of_lines_to_search) - application = get_application() if not before_after: # default to select all the way through the phrase not just up until it if direction == "left": @@ -473,11 +499,16 @@ def select_until_phrase(direction, phrase, before_after, number_of_lines_to_sear elif before_after not in ("before", "after"): print("before_after must be either 'before' or 'after' (got '{}')".format(before_after)) return - + + phrase = str(phrase) + if accessibility_text.select_until_phrase(direction, phrase, before_after, number_of_lines_to_search, + occurrence_number, dictation_versus_character): + return + + application = get_application() selected_text = select_text_and_return_it(direction, number_of_lines_to_search, application) if not selected_text: return - phrase = str(phrase) match_index = get_start_end_position(selected_text, phrase, direction, occurrence_number, dictation_versus_character) if match_index: left_index, right_index = match_index diff --git a/docs/Caster_Commands/Text_Manipulation.md b/docs/Caster_Commands/Text_Manipulation.md index ad39a8fda..283ac676f 100644 --- a/docs/Caster_Commands/Text_Manipulation.md +++ b/docs/Caster_Commands/Text_Manipulation.md @@ -2,6 +2,8 @@ Caster provides powerful text manipulation and navigation features. These functions are experimental and subject to change (perhaps based on your feedback!). We encourage contributions; please discuss your ideas [here](https://github.com/dictation-toolbox/Caster/issues/579). These commands are "CCR" and so are able to be combined with other CCR commands. Enable these commands by saying "Enable text manipulation". +When Dragonfly exposes an accessibility controller for the focused editable text field, Caster first tries that controller for text manipulation. This can select, move through, replace, and remove text through OS accessibility-backed ranges instead of copying the surrounding text through the clipboard. If accessibility is unavailable for the focused application, or the requested phrase is not found through accessibility, Caster falls back to the existing clipboard-based behavior. + ## Common elements - `direction` _sauce_ (up), _dunce_ (down), _lease_ (left) or _ross_ (right). Direction _must_ be included for all commands. diff --git a/tests/lib/test_text_manipulation_accessibility.py b/tests/lib/test_text_manipulation_accessibility.py new file mode 100644 index 000000000..677eefb4c --- /dev/null +++ b/tests/lib/test_text_manipulation_accessibility.py @@ -0,0 +1,191 @@ +import unittest + +try: + from unittest import mock +except ImportError: + import mock + +from castervoice.rules.core.text_manipulation_rules import accessibility_text +from castervoice.rules.core.text_manipulation_rules import text_manipulation_support + + +class _FakeFocusedText(object): + + def __init__(self, text, cursor): + self.expanded_text = text + self.cursor = cursor + self.selected_ranges = [] + self.cursor_offsets = [] + + def select_range(self, start, end): + self.selected_ranges.append((start, end)) + + def set_cursor(self, offset): + self.cursor = offset + self.cursor_offsets.append(offset) + + +class _FakeFocused(object): + + def __init__(self, focused_text, editable=True): + self.focused_text = focused_text + self.editable = editable + + def is_editable(self): + return self.editable + + def as_text(self): + return self.focused_text + + +class _FakeAccessibilityContext(object): + + def __init__(self, focused): + self.focused = focused + + +class _FakeOsController(object): + + def __init__(self, focused): + self.focused = focused + + def run_sync(self, callback): + return callback(_FakeAccessibilityContext(self.focused)) + + +class _FakeController(object): + + def __init__(self, focused): + self.os_controller = _FakeOsController(focused) + + +class _FakeTextAction(object): + calls = [] + + def __init__(self, spec, *args, **kwargs): + self.spec = spec + + def execute(self): + self.calls.append(self.spec) + + +class _FakeKeyAction(object): + calls = [] + + def __init__(self, spec, *args, **kwargs): + self.spec = spec + + def execute(self): + self.calls.append(self.spec) + + +class TestTextManipulationAccessibility(unittest.TestCase): + + def setUp(self): + self._old_controller = accessibility_text.dragonfly_get_accessibility_controller + self._old_text = accessibility_text.Text + self._old_key = accessibility_text.Key + _FakeTextAction.calls = [] + _FakeKeyAction.calls = [] + accessibility_text.Text = _FakeTextAction + accessibility_text.Key = _FakeKeyAction + + def tearDown(self): + accessibility_text.dragonfly_get_accessibility_controller = self._old_controller + accessibility_text.Text = self._old_text + accessibility_text.Key = self._old_key + + def _install_focused_text(self, text, cursor, editable=True): + focused_text = _FakeFocusedText(text, cursor) + focused = _FakeFocused(focused_text, editable=editable) + controller = _FakeController(focused) + accessibility_text.dragonfly_get_accessibility_controller = lambda: controller + return focused_text + + def test_select_phrase_uses_current_line_to_the_right(self): + text = "alpha target\nbeta target" + focused_text = self._install_focused_text(text, 0) + + self.assertTrue(accessibility_text.select_phrase("target", "right", 0, 1, "dictation")) + + self.assertEqual([(6, 12)], focused_text.selected_ranges) + + def test_left_window_includes_requested_previous_line_with_crlf(self): + text = "one target\r\ntwo target\r\nthree target" + focused_text = self._install_focused_text(text, text.index("three")) + expected_start = text.index("target", text.index("two")) + + self.assertTrue(accessibility_text.select_phrase("target", "left", 1, 1, "dictation")) + + self.assertEqual([(expected_start, expected_start + len("target"))], focused_text.selected_ranges) + + def test_text_node_delimiter_limits_current_line_window(self): + text = "first target" + accessibility_text.TEXT_NODE_DELIMITER + "second target" + focused_text = self._install_focused_text(text, len(text)) + expected_start = text.index("target", text.index("second")) + + self.assertTrue(accessibility_text.select_phrase("target", "left", 0, 1, "dictation")) + + self.assertEqual([(expected_start, expected_start + len("target"))], focused_text.selected_ranges) + + def test_right_window_includes_requested_next_line(self): + text = "alpha\nbeta target\ngamma target" + focused_text = self._install_focused_text(text, 0) + expected_start = text.index("target") + + self.assertTrue(accessibility_text.select_phrase("target", "right", 1, 1, "dictation")) + + self.assertEqual([(expected_start, expected_start + len("target"))], focused_text.selected_ranges) + + def test_move_until_phrase_sets_requested_boundary(self): + text = "alpha beta gamma" + focused_text = self._install_focused_text(text, text.index(" gamma")) + + self.assertTrue(accessibility_text.move_until_phrase("left", "before", "beta", 0, 1, "dictation")) + + self.assertEqual([6], focused_text.cursor_offsets) + + def test_select_until_phrase_uses_cursor_to_requested_boundary(self): + text = "alpha beta gamma" + focused_text = self._install_focused_text(text, text.index(" beta")) + + self.assertTrue(accessibility_text.select_until_phrase("right", "beta", "after", 0, 1, "dictation")) + + self.assertEqual([(5, 10)], focused_text.selected_ranges) + + def test_replace_phrase_escapes_percent_for_text_action(self): + text = "alpha target" + focused_text = self._install_focused_text(text, 0) + + self.assertTrue(accessibility_text.replace_phrase_with_phrase("target", "100%", "right", 0, 1, "dictation")) + + self.assertEqual([(6, 12)], focused_text.selected_ranges) + self.assertEqual(["100%%"], _FakeTextAction.calls) + + def test_remove_phrase_includes_preceding_space_for_dictation(self): + text = "alpha target beta" + focused_text = self._install_focused_text(text, text.index(" beta")) + + self.assertTrue(accessibility_text.remove_phrase_from_text("target", "left", 0, 1, "dictation")) + + self.assertEqual([(5, 12)], focused_text.selected_ranges) + self.assertEqual(["backspace"], _FakeKeyAction.calls) + + def test_non_editable_focus_falls_back_without_selection(self): + text = "alpha target" + focused_text = self._install_focused_text(text, 0, editable=False) + + self.assertFalse(accessibility_text.select_phrase("target", "right", 0, 1, "dictation")) + + self.assertEqual([], focused_text.selected_ranges) + + def test_support_function_returns_before_clipboard_fallback_when_accessibility_handles_it(self): + with mock.patch.object(text_manipulation_support.accessibility_text, "select_phrase", return_value=True): + with mock.patch.object(text_manipulation_support, "get_application") as get_application: + text_manipulation_support.select_phrase("target", "right", 0, 1, "dictation") + + self.assertFalse(get_application.called) + + +if __name__ == "__main__": + unittest.main()