-
Notifications
You must be signed in to change notification settings - Fork 454
Add a speech Lua library for screen reader and brail output #4762
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
RealAmethyst
wants to merge
2
commits into
TASEmulators:master
Choose a base branch
from
RealAmethyst:speech-lua-library
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+150
−0
Open
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Binary file not shown.
150 changes: 150 additions & 0 deletions
150
src/BizHawk.Client.Common/lua/CommonLibs/SpeechLuaLibrary.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| using System.Runtime.InteropServices; | ||
|
|
||
| namespace BizHawk.Client.Common | ||
| { | ||
| /// <summary> | ||
| /// Lua access to screen-reader speech and braille output, for blind accessibility. Backed by the | ||
| /// prism screen-reader abstraction library (https://github.com/ethindp/prism, MPL-2.0), which routes | ||
| /// to the user's active screen reader (NVDA, JAWS, Narrator/OneCore, ...) and falls back to SAPI. | ||
| /// </summary> | ||
| [LuaLibrary(released: true)] | ||
| public sealed class SpeechLuaLibrary : LuaLibraryBase | ||
| { | ||
| public SpeechLuaLibrary(ILuaLibraries luaLibsImpl, ApiContainer apiContainer, Action<string> logOutputCallback) | ||
| : base(luaLibsImpl, apiContainer, logOutputCallback) {} | ||
|
|
||
| public override string Name => "speech"; | ||
|
|
||
| [LuaMethodExample("speech.say(\"Charizard, 142 of 142 HP\");")] | ||
| [LuaMethod("say", "Speaks the given text through the active screen reader (or SAPI fallback). When interrupt is true (the default), any current speech is cancelled first.")] | ||
| public void Say(string text, bool interrupt = true) | ||
| { | ||
| if (!PrismSpeech.Speak(text ?? string.Empty, interrupt, out var error)) Log($"speech.say failed: {error}"); | ||
| } | ||
|
|
||
| [LuaMethodExample("speech.output(\"State 1 saved\");")] | ||
| [LuaMethod("output", "Speaks and brailles the given text (whichever the active screen reader supports). interrupt defaults to true.")] | ||
| public void Output(string text, bool interrupt = true) | ||
| { | ||
| if (!PrismSpeech.Output(text ?? string.Empty, interrupt, out var error)) Log($"speech.output failed: {error}"); | ||
| } | ||
|
|
||
| [LuaMethodExample("speech.braille(\"HP 142/142\");")] | ||
| [LuaMethod("braille", "Sends the given text to a connected braille display via the active screen reader, if supported.")] | ||
| public void Braille(string text) | ||
| { | ||
| if (!PrismSpeech.Braille(text ?? string.Empty, out var error)) Log($"speech.braille failed: {error}"); | ||
| } | ||
|
|
||
| [LuaMethodExample("speech.stop();")] | ||
| [LuaMethod("stop", "Stops/silences any speech currently in progress.")] | ||
| public void Stop() | ||
| { | ||
| if (!PrismSpeech.Stop(out var error)) Log($"speech.stop failed: {error}"); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Minimal P/Invoke wrapper around prism's C API (prism.dll, shipped in dll/). Initialised lazily on | ||
| /// first use and kept for the process lifetime. If prism or a speech backend isn't available, every | ||
| /// call becomes a no-op that returns an error string, so missing DLLs never crash EmuHawk. | ||
| /// </summary> | ||
| internal static class PrismSpeech | ||
| { | ||
| // PrismConfig is { uint8_t version }; current ABI version is PRISM_CONFIG_VERSION (2) for the | ||
| // prism.dll we ship. (See prism.h.) | ||
| private const byte ConfigVersion = 2; | ||
|
|
||
| [StructLayout(LayoutKind.Sequential)] | ||
| private struct PrismConfig { public byte Version; } | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_init", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern IntPtr PrismInit(ref PrismConfig cfg); | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_registry_acquire_best", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern IntPtr PrismRegistryAcquireBest(IntPtr ctx); | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_backend_initialize", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern int PrismBackendInitialize(IntPtr backend); | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_backend_speak", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern int PrismBackendSpeak(IntPtr backend, [MarshalAs(UnmanagedType.LPUTF8Str)] string text, [MarshalAs(UnmanagedType.I1)] bool interrupt); | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_backend_output", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern int PrismBackendOutput(IntPtr backend, [MarshalAs(UnmanagedType.LPUTF8Str)] string text, [MarshalAs(UnmanagedType.I1)] bool interrupt); | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_backend_braille", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern int PrismBackendBraille(IntPtr backend, [MarshalAs(UnmanagedType.LPUTF8Str)] string text); | ||
|
|
||
| [DllImport("prism", EntryPoint = "prism_backend_stop", CallingConvention = CallingConvention.Cdecl)] | ||
| private static extern int PrismBackendStop(IntPtr backend); | ||
|
|
||
| private const int PRISM_OK = 0; | ||
| private const int PRISM_ERROR_ALREADY_INITIALIZED = 15; // prism_registry_acquire_best already returns an initialised backend | ||
|
|
||
| private static readonly object _lock = new(); | ||
| private static bool _tried; | ||
| private static IntPtr _backend; | ||
| private static string _initError; | ||
|
|
||
| private static bool EnsureInit(out string error) | ||
| { | ||
| lock (_lock) | ||
| { | ||
| if (_backend != IntPtr.Zero) { error = null; return true; } | ||
| if (_tried) { error = _initError; return false; } | ||
| _tried = true; | ||
| try | ||
| { | ||
| var cfg = new PrismConfig { Version = ConfigVersion }; | ||
| var ctx = PrismInit(ref cfg); | ||
| if (ctx == IntPtr.Zero) { _initError = "prism_init returned null"; error = _initError; return false; } | ||
|
|
||
| var backend = PrismRegistryAcquireBest(ctx); | ||
| if (backend == IntPtr.Zero) { _initError = "no speech backend available"; error = _initError; return false; } | ||
|
|
||
| var rc = PrismBackendInitialize(backend); | ||
| if (rc != PRISM_OK && rc != PRISM_ERROR_ALREADY_INITIALIZED) { _initError = $"prism_backend_initialize returned {rc}"; error = _initError; return false; } | ||
|
|
||
| _backend = backend; | ||
| error = null; | ||
| return true; | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _initError = ex.Message; | ||
| error = _initError; | ||
| return false; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public static bool Speak(string text, bool interrupt, out string error) | ||
| { | ||
| if (!EnsureInit(out error)) return false; | ||
| try { return PrismBackendSpeak(_backend, text, interrupt) == PRISM_OK; } | ||
| catch (Exception ex) { error = ex.Message; return false; } | ||
| } | ||
|
|
||
| public static bool Output(string text, bool interrupt, out string error) | ||
| { | ||
| if (!EnsureInit(out error)) return false; | ||
| try { return PrismBackendOutput(_backend, text, interrupt) == PRISM_OK; } | ||
| catch (Exception ex) { error = ex.Message; return false; } | ||
| } | ||
|
|
||
| public static bool Braille(string text, out string error) | ||
| { | ||
| if (!EnsureInit(out error)) return false; | ||
| try { return PrismBackendBraille(_backend, text) == PRISM_OK; } | ||
| catch (Exception ex) { error = ex.Message; return false; } | ||
| } | ||
|
|
||
| public static bool Stop(out string error) | ||
| { | ||
| if (!EnsureInit(out error)) return false; | ||
| try { return PrismBackendStop(_backend) == PRISM_OK; } | ||
| catch (Exception ex) { error = ex.Message; return false; } | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is it reasonable to announce an empty (or all-whitespace) string?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I mean there is no reason why speech should announce empty stuff.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I occasionally use
printto see some information and it ends up being actually empty, usually due to a bug in my script. In this case I can still see an empty line once the next print happens. It'd be much more frustrating if theprintcall did nothing, and I imagine it will also be very frustrating if a speech call that accidentally contains a blank string is ignored.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I actually didn't think about that, and yeah you're right, that would be frustrating, so having it still speak it is the best.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Explain how that applies to TTS.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I mean, I would be curious how TTS handles the empty string, but SuuperW's "debug empty log messages" use-case seems relevant whether you can see the screen or not.