SeeItSayItLabel improvements (#593)

UnityProjects/MRTKDevTemplate/Packages/packages-lock.json

@@ -431,7 +431,7 @@
       "source": "local",
       "dependencies": {
         "com.microsoft.mrtk.graphicstools.unity": "0.5.12",
-        "org.mixedrealitytoolkit.core": "3.2.0",
+        "org.mixedrealitytoolkit.core": "3.3.0",
         "com.unity.inputsystem": "1.6.1",
         "com.unity.textmeshpro": "3.0.6",
         "com.unity.xr.interaction.toolkit": "2.3.0"

UnityProjects/MRTKDevTemplate/ProjectSettings/EditorBuildSettings.asset

@@ -117,7 +117,7 @@ EditorBuildSettings:
     path: Assets/Scenes/Experimental/ScrollingExample.unity
     guid: 77e21ebc978cbc84f9ebe061742ae42d
   - enabled: 1
-    path: Assets/Scenes/SeeItSayIt Example.unity
+    path: Assets/Scenes/SeeItSayItExample.unity
     guid: cb7d2aaea1c5a4d469a6408ee9ddc5fc
   - enabled: 1
     path: Assets/Scenes/SlateDrawingExample.unity

org.mixedrealitytoolkit.core/CHANGELOG.md

@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 * Updated tests to follow existing MRTK test patterns. [PR #1046](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/1046)
 
+### Added
+
+* Added event `OnSpeechRecognitionKeywordChanged` to allow UI updates when the speech recognition keyword has changed. [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792/)
+
 ### Fixed
 
 * Fixed broken project validation help link, for item 'MRTK3 profile may need to be assigned for the Standalone build target' (Issue #882) [PR #886](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/886)

org.mixedrealitytoolkit.core/Editor/Editors/StatefulInteractableEditor.cs

@@ -23,6 +23,7 @@ public class StatefulInteractableEditor : BaseInteractableEditor
         private SerializedProperty allowSelectByVoice;
         private SerializedProperty SelectRequiresHover;
         private SerializedProperty speechRecognitionKeyword;
+        private SerializedProperty OnSpeechRecognitionKeywordChanged;
         private SerializedProperty VoiceRequiresFocus;
         private SerializedProperty UseGazeDwell;
         private SerializedProperty GazeDwellTime;
@@ -34,6 +35,7 @@ public class StatefulInteractableEditor : BaseInteractableEditor
         private SerializedProperty OnEnabled;
         private SerializedProperty OnDisabled;
         private static bool advancedFoldout = false;
+        private static bool speechRecognitionKeywordEventFoldout = false;
         private static bool enabledEventsFoldout = false;
 
         /// <summary>
@@ -57,6 +59,7 @@ protected override void OnEnable()
 
             allowSelectByVoice = SetUpProperty(nameof(allowSelectByVoice));
             speechRecognitionKeyword = SetUpProperty(nameof(speechRecognitionKeyword));
+            OnSpeechRecognitionKeywordChanged = SetUpAutoProperty(nameof(OnSpeechRecognitionKeywordChanged));
             VoiceRequiresFocus = SetUpAutoProperty(nameof(VoiceRequiresFocus));
 
             SelectRequiresHover = SetUpAutoProperty(nameof(SelectRequiresHover));
@@ -166,8 +169,13 @@ protected void DrawProperties(bool showToggleMode)
                     {
                         using (new EditorGUI.IndentLevelScope())
                         {
-                            EditorGUILayout.PropertyField(speechRecognitionKeyword);
                             EditorGUILayout.PropertyField(VoiceRequiresFocus);
+                            EditorGUILayout.PropertyField(speechRecognitionKeyword);
+                            speechRecognitionKeywordEventFoldout = EditorGUILayout.Foldout(speechRecognitionKeywordEventFoldout, EditorGUIUtility.TrTempContent("Speech Recognition Keyword event"), true);
+                            if (speechRecognitionKeywordEventFoldout)
+                            {
+                                EditorGUILayout.PropertyField(OnSpeechRecognitionKeywordChanged);
+                            }
                         }
                     }
 

org.mixedrealitytoolkit.core/Interactables/StatefulInteractable.cs

@@ -148,10 +148,17 @@ public string SpeechRecognitionKeyword
                     {
                         interactionManager.RegisterInteractable(this as IXRInteractable);
                     }
+                    OnSpeechRecognitionKeywordChanged.Invoke(speechRecognitionKeyword);
                 }
             }
         }
-
+
+        /// <summary>
+        /// Fired when the <see cref="SpeechRecognitionKeyword"/> has changed.
+        /// </summary>
+        [field: SerializeField, Tooltip("Fired when the Speech Recognition Keyword has changed.")]
+        public UnityEvent<string> OnSpeechRecognitionKeywordChanged { get; private set; } = new UnityEvent<string>();
+
         /// <summary>
         /// Does the voice command require this to have focus?
         /// If true, then the voice command will only respond to voice commands while this Interactable has focus.

org.mixedrealitytoolkit.core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "org.mixedrealitytoolkit.core",
-  "version": "3.2.3-development",
+  "version": "3.3.0-development",
   "description": "A limited collection of common interfaces and utilities that most MRTK packages share. Most implementations of these interfaces are contained in other packages in the MRTK ecosystem.",
   "displayName": "MRTK Core Definitions",
   "msftFeatureCategory": "MRTK3",

org.mixedrealitytoolkit.uxcore/CHANGELOG.md

@@ -4,6 +4,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Unreleased
 
+### Added
+
+* Added automatic update for the `See It Say It Label` when the `SpeechRecognitionKeyword` of a `StatefulInteractable` has changed. Added ability to change the pattern, from inspector or code. [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792)
+* When Unity Localization package is installed, a `LocalizedString` is used as pattern. [PR #792](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/792)
+
 ### Changed
 
 * StateVisualizer: Modified access modifiers of State, stateContainers and UpdateStateValue to protected internal to allow adding states through subclassing. [PR #926](https://github.com/MixedRealityToolkit/MixedRealityToolkit-Unity/pull/926)

org.mixedrealitytoolkit.uxcore/Editor/Inspectors/SeeItSayIt/SeeItSayItLabelEnablerInspector.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Mixed Reality Toolkit Contributors
+// Licensed under the BSD 3-Clause
+
+using MixedReality.Toolkit.UX;
+using UnityEditor;
+
+namespace MixedReality.Toolkit.Editor
+{
+    [CustomEditor(typeof(SeeItSayItLabelEnabler))]
+    public class SeeItSayItLabelEnablerInspector : UnityEditor.Editor
+    {
+        private SerializedProperty localizedPattern = null;
+        private SerializedProperty pattern = null;
+
+        private void OnEnable()
+        {
+            localizedPattern = serializedObject.FindProperty(nameof(localizedPattern));
+            pattern = serializedObject.FindProperty(nameof(pattern));
+        }
+
+        /// <summary>
+        /// Called by the Unity editor to render custom inspector UI for this component.
+        /// </summary>
+        public override void OnInspectorGUI()
+        {
+            DrawPropertiesExcluding(serializedObject, "localizedPattern", "pattern");
+
+            if (localizedPattern != null)
+            {
+                EditorGUILayout.PropertyField(localizedPattern);
+            }
+
+            if (localizedPattern == null ||
+                (string.IsNullOrEmpty(localizedPattern.FindPropertyRelative("m_TableEntryReference").FindPropertyRelative("m_Key").stringValue)
+                && localizedPattern.FindPropertyRelative("m_TableEntryReference").FindPropertyRelative("m_KeyId").longValue == 0)
+                || string.IsNullOrEmpty(localizedPattern.FindPropertyRelative("m_TableReference").FindPropertyRelative("m_TableCollectionName").stringValue))
+            {
+                if (localizedPattern != null)
+                {
+                    EditorGUILayout.HelpBox("Pattern is only used when the Localized Pattern above is not set.", MessageType.Info);
+                }
+                EditorGUILayout.PropertyField(pattern);
+            }
+
+            serializedObject.ApplyModifiedProperties();
+        }
+    }
+}

org.mixedrealitytoolkit.uxcore/MRTK.UXCore.asmdef

@@ -6,6 +6,7 @@
         "MixedReality.Toolkit.Data",
         "Microsoft.MixedReality.GraphicsTools",
         "Unity.InputSystem",
+        "Unity.Localization",
         "Unity.TextMeshPro",
         "Unity.XR.Interaction.Toolkit",
         "Unity.XR.CoreUtils",
@@ -39,6 +40,11 @@
             "name": "org.mixedrealitytoolkit.windowsspeech",
             "expression": "",
             "define": "MRTK_SPEECH_PRESENT"
+        },
+        {
+            "name": "com.unity.localization",
+            "expression": "",
+            "define": "UNITY_LOCALIZATION_PRESENT"
         }
     ],
     "noEngineReferences": false

org.mixedrealitytoolkit.uxcore/SeeItSayIt/SeeItSayItLabelEnabler.cs

@@ -6,6 +6,9 @@
 #if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT
 using MixedReality.Toolkit.Input;
 #endif
+#if UNITY_LOCALIZATION_PRESENT
+using UnityEngine.Localization;
+#endif
 
 namespace MixedReality.Toolkit.UX
 {
@@ -16,15 +19,25 @@ namespace MixedReality.Toolkit.UX
     /// <remarks>
     /// A "see-it say-it" label is used for accessibility and voice input. The label displays the keyword
     /// that can be spoken to active or click the associated <see cref="PressableButton"/>.
-    /// 
+    ///
     /// This class will only enable a "see-it say-it" label if the application has included the MRTK input
-    /// package, and has an active <see cref="SpeechInteractor"/> object when this component's 
+    /// package, and has an active <see cref="SpeechInteractor"/> object when this component's
     /// <see cref="Start"/> method is invoked.
     /// </remarks>
     [RequireComponent(typeof(PressableButton))]
     [AddComponentMenu("MRTK/UX/See It Say It Label")]
     public class SeeItSayItLabelEnabler : MonoBehaviour
     {
+        /// <summary>
+        /// The <see cref="PressableButton"/> present on the same GameObject.
+        /// </summary>
+        private PressableButton pressableButton;
+
+        /// <summary>
+        /// The <see cref="TMP_Text"/> used to display the label present in child.
+        /// </summary>
+        private TMP_Text labelText;
+
         [SerializeField]
         [Tooltip("The GameObject for the see-it say-it label to be enabled.")]
         private GameObject seeItSayItLabel;
@@ -38,6 +51,32 @@ public GameObject SeeItSayItLabel
             set => seeItSayItLabel = value;
         }
 
+#if UNITY_LOCALIZATION_PRESENT
+        [SerializeField]
+        [Tooltip("The LocalizedString that define the label pattern. Use a smart string with one argument that will be replaced by the button's speech recognition keyword (e.g: \"Say '{0}'\").")]
+        private LocalizedString localizedPattern;
+#endif
+
+        [SerializeField]
+        [Tooltip("The pattern for the see-it say-it label using string.Format() when localization isn't used.")]
+        private string pattern = "Say '{0}'";
+
+        /// <summary>
+        /// The pattern for the see-it say-it label using string.Format() when localization isn't used.
+        /// </summary>
+        public string Pattern
+        {
+            get => pattern;
+            set
+            {
+                pattern = value;
+                if (pressableButton != null)
+                {
+                    UpdateLabel(pressableButton.SpeechRecognitionKeyword);
+                }
+            }
+        }
+
         [SerializeField]
         [Tooltip("The Transform that the label will be dynamically positioned off of. Empty by default. If positioning a Canvas label, this must be a RectTransform.")]
         private Transform positionControl;
@@ -51,16 +90,20 @@ public Transform PositionControl
             set => positionControl = value;
         }
 
-        private float canvasOffset = -10f;
-        private float nonCanvasOffset = -.004f;
+        private const float CanvasOffset = -10f;
+        private const float NonCanvasOffset = -0.004f;
+
+        protected virtual void Awake()
+        {
+            pressableButton = GetComponent<PressableButton>();
+        }
 
         /// <summary>
         /// A Unity event function that is called on the frame when a script is enabled just before any of the update methods are called the first time.
-        /// </summary> 
+        /// </summary>
         protected virtual void Start()
         {
             // Check if voice commands are enabled for this button
-            PressableButton pressableButton = gameObject.GetComponent<PressableButton>();
             if (pressableButton != null && pressableButton.AllowSelectByVoice)
             {
                 // Check if input and speech packages are present
@@ -72,23 +115,17 @@ protected virtual void Start()
                 }
 
                 SeeItSayItLabel.SetActive(true);
+                labelText = SeeItSayItLabel.GetComponentInChildren<TMP_Text>(true);
+                pressableButton.OnSpeechRecognitionKeywordChanged.AddListener(UpdateLabel);
 
-                // Children must be disabled so that they are not initially visible 
+                // Children must be disabled so that they are not initially visible
                 foreach (Transform child in SeeItSayItLabel.transform)
                 {
                     child.gameObject.SetActive(false);
                 }
 
                 // Set the label text to reflect the speech recognition keyword
-                string keyword = pressableButton.SpeechRecognitionKeyword;
-                if (keyword != null)
-                {
-                    TMP_Text labelText = SeeItSayItLabel.GetComponentInChildren<TMP_Text>(true);
-                    if (labelText != null)
-                    {
-                        labelText.text = $"Say '{keyword}'";
-                    }
-                }
+                UpdateLabel(pressableButton.SpeechRecognitionKeyword);
 
                 // If a Transform is specified, use it to reposition the object dynamically
                 if (positionControl != null)
@@ -97,26 +134,73 @@ protected virtual void Start()
                     RectTransform controlRectTransform = PositionControl.gameObject.GetComponent<RectTransform>();
 
                     // If PositionControl is a RectTransform, reposition label relative to Canvas button
-                    if (controlRectTransform != null &&  SeeItSayItLabel.transform.childCount > 0)
+                    if (controlRectTransform != null && SeeItSayItLabel.transform.childCount > 0)
                     {
                         // The parent RectTransform used to center the label
                         RectTransform canvasTransform = SeeItSayItLabel.GetComponent<RectTransform>();
 
-                        // The child RectTransform used to set the final position of the label 
+                        // The child RectTransform used to set the final position of the label
                         RectTransform labelTransform = SeeItSayItLabel.transform.GetChild(0).gameObject.GetComponent<RectTransform>();
 
                         if (labelTransform != null && canvasTransform != null)
                         {
-                            labelTransform.anchoredPosition3D = new Vector3(canvasTransform.rect.width / 2f, canvasTransform.rect.height / 2f + (controlRectTransform.rect.height /  2f * -1) + canvasOffset, canvasOffset);
+                            labelTransform.anchoredPosition3D = new Vector3(canvasTransform.rect.width / 2f, canvasTransform.rect.height / 2f + (controlRectTransform.rect.height / 2f * -1) + CanvasOffset, CanvasOffset);
                         }
                     }
                     else
                     {
-                        SeeItSayItLabel.transform.localPosition = new Vector3(PositionControl.localPosition.x, (PositionControl.lossyScale.y / 2f * -1) + nonCanvasOffset, PositionControl.localPosition.z + nonCanvasOffset);
+                        SeeItSayItLabel.transform.localPosition = new Vector3(PositionControl.localPosition.x, (PositionControl.lossyScale.y / 2f * -1) + NonCanvasOffset, PositionControl.localPosition.z + NonCanvasOffset);
                     }
                 }
+
+#if UNITY_LOCALIZATION_PRESENT
+                if (!localizedPattern.IsEmpty)
+                {
+                    localizedPattern.StringChanged += OnLocalizedPatternChanged;
+                }
+#endif
 #endif
             }
         }
+
+        protected virtual void OnDestroy()
+        {
+#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT
+            if (pressableButton != null)
+            {
+                pressableButton.OnSpeechRecognitionKeywordChanged.RemoveListener(UpdateLabel);
+#if UNITY_LOCALIZATION_PRESENT
+                if (!localizedPattern.IsEmpty)
+                {
+                    localizedPattern.StringChanged -= OnLocalizedPatternChanged;
+                }
+#endif
+            }
+#endif
+        }
+
+        protected virtual void UpdateLabel(string keyword)
+        {
+#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT
+            if (!string.IsNullOrWhiteSpace(keyword) && labelText != null)
+            {
+#if UNITY_LOCALIZATION_PRESENT
+                if (!localizedPattern.IsEmpty)
+                {
+                    labelText.text = localizedPattern.GetLocalizedString(keyword);
+                    return;
+                }
+#endif
+                labelText.text = string.Format(pattern, keyword);
+            }
+#endif
+        }
+
+#if MRTK_INPUT_PRESENT && MRTK_SPEECH_PRESENT && UNITY_LOCALIZATION_PRESENT
+        protected virtual void OnLocalizedPatternChanged(string value)
+        {
+            UpdateLabel(pressableButton.SpeechRecognitionKeyword);
+        }
+#endif
     }
 }