NAVAND-552: predownload voice instructions (#6771)

Author: dzinad

changelog/unreleased/bugfixes/6771.md

@@ -0,0 +1,3 @@
+- Introduced `VoiceInstructionsPrefetcher` `MapboxSpeechAPI#generatePredownloaded` to use predownloaded voice instructions instead of downloading them on demand. Example usage can be found in the examples directory ( see `MapboxVoiceActivity`).
+- Enabled voice instructions predownloading for those who use `MapboxAudioGuidance`.
+- Fixed an issue where with low connectivity voice instruction might have been played too late for those who use `MapboxAudioGuidance`. If you use `MapboxSpeechAPI` directly, switch to voice instructions predownloading as described above if you encounter said issue.

examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt

@@ -22,6 +22,7 @@ import com.mapbox.maps.plugin.gestures.OnMapLongClickListener
 import com.mapbox.maps.plugin.gestures.gestures
 import com.mapbox.maps.plugin.locationcomponent.LocationComponentPlugin
 import com.mapbox.maps.plugin.locationcomponent.location
+import com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI
 import com.mapbox.navigation.base.extensions.applyDefaultNavigationOptions
 import com.mapbox.navigation.base.extensions.applyLanguageAndVoiceUnitOptions
 import com.mapbox.navigation.base.options.NavigationOptions
@@ -60,6 +61,7 @@ import com.mapbox.navigation.ui.utils.internal.resource.ResourceLoadRequest
 import com.mapbox.navigation.ui.utils.internal.resource.ResourceLoaderFactory
 import com.mapbox.navigation.ui.voice.api.MapboxSpeechApi
 import com.mapbox.navigation.ui.voice.api.MapboxVoiceInstructionsPlayer
+import com.mapbox.navigation.ui.voice.api.VoiceInstructionsPrefetcher
 import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement
 import com.mapbox.navigation.ui.voice.model.SpeechError
 import com.mapbox.navigation.ui.voice.model.SpeechValue
@@ -79,6 +81,7 @@ import java.util.Locale
  * attention to its usage. Long press anywhere on the map to set a destination and trigger
  * navigation.
  */
+@OptIn(ExperimentalPreviewMapboxNavigationAPI::class)
 class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
 
     private var isMuted: Boolean = false
@@ -201,9 +204,13 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         }
     }
 
+    private val voiceInstructionsPrefetcher by lazy {
+        VoiceInstructionsPrefetcher(speechApi)
+    }
+
     private val voiceInstructionsObserver =
         VoiceInstructionsObserver { voiceInstructions -> // The data obtained must be used to generate the synthesized speech mp3 file.
-            speechApi.generate(
+            speechApi.generatePredownloaded(
                 voiceInstructions,
                 speechCallback
             )
@@ -382,6 +389,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
                 .build()
         )
         init()
+        voiceInstructionsPrefetcher.onAttached(mapboxNavigation)
     }
 
     override fun onStart() {
@@ -413,6 +421,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         mapboxReplayer.finish()
         mapboxNavigation.onDestroy()
         speechApi.cancel()
+        voiceInstructionsPrefetcher.onDetached(mapboxNavigation)
         voiceInstructionsPlayer.shutdown()
     }
 

libnavigation-util/src/main/java/com/mapbox/navigation/utils/internal/Time.kt

@@ -1,12 +1,28 @@
 package com.mapbox.navigation.utils.internal
 
+import android.os.SystemClock
+import java.util.concurrent.TimeUnit
+
 interface Time {
+
     fun nanoTime(): Long
+
     fun millis(): Long
 
+    fun seconds(): Long
     object SystemImpl : Time {
         override fun nanoTime(): Long = System.nanoTime()
 
         override fun millis(): Long = System.currentTimeMillis()
+
+        override fun seconds(): Long = TimeUnit.MILLISECONDS.toSeconds(millis())
+    }
+
+    object SystemClockImpl : Time {
+        override fun nanoTime(): Long = SystemClock.elapsedRealtimeNanos()
+
+        override fun millis(): Long = SystemClock.elapsedRealtime()
+
+        override fun seconds(): Long = TimeUnit.MILLISECONDS.toSeconds(millis())
     }
 }

libnavui-voice/api/current.txt

@@ -64,6 +64,7 @@ package com.mapbox.navigation.ui.voice.api {
     method public void cancel();
     method public void clean(com.mapbox.navigation.ui.voice.model.SpeechAnnouncement announcement);
     method public void generate(com.mapbox.api.directions.v5.models.VoiceInstructions voiceInstruction, com.mapbox.navigation.ui.base.util.MapboxNavigationConsumer<com.mapbox.bindgen.Expected<com.mapbox.navigation.ui.voice.model.SpeechError,com.mapbox.navigation.ui.voice.model.SpeechValue>> consumer);
+    method @com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI public void generatePredownloaded(com.mapbox.api.directions.v5.models.VoiceInstructions voiceInstruction, com.mapbox.navigation.ui.base.util.MapboxNavigationConsumer<com.mapbox.bindgen.Expected<com.mapbox.navigation.ui.voice.model.SpeechError,com.mapbox.navigation.ui.voice.model.SpeechValue>> consumer);
   }
 
   @UiThread public final class MapboxVoiceInstructionsPlayer {
@@ -92,6 +93,18 @@ package com.mapbox.navigation.ui.voice.api {
     property public abstract com.mapbox.navigation.ui.voice.options.VoiceInstructionsPlayerOptions options;
   }
 
+  @com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI public final class VoiceInstructionsPrefetcher implements com.mapbox.navigation.core.lifecycle.MapboxNavigationObserver {
+    ctor public VoiceInstructionsPrefetcher(com.mapbox.navigation.ui.voice.api.MapboxSpeechApi speechApi);
+    method public void onAttached(com.mapbox.navigation.core.MapboxNavigation mapboxNavigation);
+    method public void onDetached(com.mapbox.navigation.core.MapboxNavigation mapboxNavigation);
+    field public static final com.mapbox.navigation.ui.voice.api.VoiceInstructionsPrefetcher.Companion Companion;
+    field public static final int DEFAULT_OBSERVABLE_TIME_SECONDS = 180; // 0xb4
+    field public static final double DEFAULT_TIME_PERCENTAGE_TO_TRIGGER_AFTER = 0.5;
+  }
+
+  public static final class VoiceInstructionsPrefetcher.Companion {
+  }
+
 }
 
 package com.mapbox.navigation.ui.voice.model {

libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/MapboxAudioGuidance.kt

@@ -2,6 +2,7 @@ package com.mapbox.navigation.ui.voice.api
 
 import androidx.annotation.VisibleForTesting
 import com.mapbox.api.directions.v5.models.VoiceInstructions
+import com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI
 import com.mapbox.navigation.core.MapboxNavigation
 import com.mapbox.navigation.core.lifecycle.MapboxNavigationApp
 import com.mapbox.navigation.core.lifecycle.MapboxNavigationObserver
@@ -41,6 +42,9 @@ internal constructor(
 
     private var dataStoreOwner: NavigationDataStoreOwner? = null
     private var configOwner: NavigationConfigOwner? = null
+
+    @OptIn(ExperimentalPreviewMapboxNavigationAPI::class)
+    private var trigger: VoiceInstructionsPrefetcher? = null
     private var mutedStateFlow = MutableStateFlow(false)
     private val internalStateFlow = MutableStateFlow(MapboxAudioGuidanceState())
     private val scope = CoroutineScope(SupervisorJob() + dispatcher)
@@ -70,8 +74,10 @@ internal constructor(
     /**
      * @see [MapboxNavigationApp]
      */
+    @OptIn(ExperimentalPreviewMapboxNavigationAPI::class)
     override fun onDetached(mapboxNavigation: MapboxNavigation) {
         mapboxVoiceInstructions.unregisterObservers(mapboxNavigation)
+        trigger?.onDetached(mapboxNavigation)
         job?.cancel()
         job = null
     }
@@ -160,15 +166,22 @@ internal constructor(
         }
     }
 
-    private fun MapboxNavigation.audioGuidanceVoice(): Flow<MapboxAudioGuidanceVoice> =
-        combine(
+    @OptIn(ExperimentalPreviewMapboxNavigationAPI::class)
+    private fun MapboxNavigation.audioGuidanceVoice(): Flow<MapboxAudioGuidanceVoice> {
+        return combine(
             mapboxVoiceInstructions.voiceLanguage(),
             configOwner!!.language(),
         ) { voiceLanguage, deviceLanguage -> voiceLanguage ?: deviceLanguage }
             .distinctUntilChanged()
             .map { language ->
-                audioGuidanceServices.mapboxAudioGuidanceVoice(this, language)
+                trigger?.onDetached(this)
+                audioGuidanceServices.mapboxAudioGuidanceVoice(this, language).also {
+                    trigger = VoiceInstructionsPrefetcher(it.mapboxSpeechApi).also { trigger ->
+                        trigger.onAttached(this)
+                    }
+                }
             }
+    }
 
     private suspend fun restoreMutedState() {
         dataStoreOwner?.apply {

libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/MapboxSpeechApi.kt

@@ -1,16 +1,21 @@
 package com.mapbox.navigation.ui.voice.api
 
 import android.content.Context
+import androidx.annotation.UiThread
 import com.mapbox.api.directions.v5.models.VoiceInstructions
 import com.mapbox.bindgen.Expected
 import com.mapbox.bindgen.ExpectedFactory
+import com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI
+import com.mapbox.navigation.core.trip.session.VoiceInstructionsObserver
 import com.mapbox.navigation.ui.base.util.MapboxNavigationConsumer
 import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement
 import com.mapbox.navigation.ui.voice.model.SpeechError
 import com.mapbox.navigation.ui.voice.model.SpeechValue
+import com.mapbox.navigation.ui.voice.model.TypeAndAnnouncement
 import com.mapbox.navigation.ui.voice.model.VoiceState
 import com.mapbox.navigation.ui.voice.options.MapboxSpeechApiOptions
 import com.mapbox.navigation.utils.internal.InternalJobControlFactory
+import kotlinx.coroutines.cancel
 import kotlinx.coroutines.launch
 import java.util.Locale
 
@@ -28,7 +33,11 @@ class MapboxSpeechApi @JvmOverloads constructor(
     private val options: MapboxSpeechApiOptions = MapboxSpeechApiOptions.Builder().build()
 ) {
 
+    private val cachedFiles = mutableMapOf<TypeAndAnnouncement, SpeechValue>()
     private val mainJobController by lazy { InternalJobControlFactory.createMainScopeJobControl() }
+    private val predownloadJobController by lazy {
+        InternalJobControlFactory.createDefaultScopeJobControl()
+    }
     private val voiceAPI = VoiceApiProvider.retrieveMapboxVoiceApi(
         context,
         accessToken,
@@ -40,6 +49,9 @@ class MapboxSpeechApi @JvmOverloads constructor(
      * Given [VoiceInstructions] the method will try to generate the
      * voice instruction [SpeechAnnouncement] including the synthesized speech mp3 file
      * from Mapbox's API Voice.
+     * NOTE: this method will try downloading an mp3 file from server. If you use voice instructions
+     * predownloading (see [VoiceInstructionsPrefetcher]), invoke [generatePredownloaded]
+     * instead of this method in your [VoiceInstructionsObserver].
      * @param voiceInstruction VoiceInstructions object representing [VoiceInstructions]
      * @param consumer is a [SpeechValue] including the announcement to be played when the
      * announcement is ready or a [SpeechError] including the error information and a fallback
@@ -51,7 +63,42 @@ class MapboxSpeechApi @JvmOverloads constructor(
         consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>
     ) {
         mainJobController.scope.launch {
-            retrieveVoiceFile(voiceInstruction, consumer)
+            consumer.accept(retrieveVoiceFile(voiceInstruction))
+        }
+    }
+
+    /**
+     * Given [VoiceInstructions] the method will try to generate the
+     * voice instruction [SpeechAnnouncement] including the synthesized speech mp3 file
+     * from Mapbox's API Voice.
+     * NOTE: this method will NOT try downloading an mp3 file from server. It will either use
+     * an already predownloaded file or an onboard speech synthesizer. Only invoke this method
+     * if you use voice instructions predownloading (see [VoiceInstructionsPrefetcher]),
+     * otherwise invoke [generatePredownloaded] in your [VoiceInstructionsObserver].
+     * @param voiceInstruction VoiceInstructions object representing [VoiceInstructions]
+     * @param consumer is a [SpeechValue] including the announcement to be played when the
+     * announcement is ready or a [SpeechError] including the error information and a fallback
+     * with the raw announcement (without file) that can be played with a text-to-speech engine.
+     * @see [cancel]
+     */
+    @ExperimentalPreviewMapboxNavigationAPI
+    fun generatePredownloaded(
+        voiceInstruction: VoiceInstructions,
+        consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>
+    ) {
+        mainJobController.scope.launch {
+            val cachedValue = getFromCache(voiceInstruction)
+            if (cachedValue != null) {
+                consumer.accept(ExpectedFactory.createValue(cachedValue))
+            } else {
+                val fallback = getFallbackAnnouncement(voiceInstruction)
+                val speechError = SpeechError(
+                    "No predownloaded instruction for ${voiceInstruction.announcement()}",
+                    null,
+                    fallback
+                )
+                consumer.accept(ExpectedFactory.createError(speechError))
+            }
         }
     }
 
@@ -73,33 +120,62 @@ class MapboxSpeechApi @JvmOverloads constructor(
      */
     fun clean(announcement: SpeechAnnouncement) {
         voiceAPI.clean(announcement)
+        VoiceInstructionsParser.parse(announcement).onValue {
+            val value = cachedFiles[it]
+            // when we clear fallback announcement, there is a chance we will remove the key
+            // from map and not remove the file itself
+            // since for fallback SpeechAnnouncement file is null
+            if (value?.announcement == announcement) {
+                cachedFiles.remove(it)
+            }
+        }
+    }
+
+    @UiThread
+    internal fun predownload(instructions: List<VoiceInstructions>) {
+        instructions.forEach { instruction ->
+            val typeAndAnnouncement = VoiceInstructionsParser.parse(instruction).value
+            if (typeAndAnnouncement != null && !hasTypeAndAnnouncement(typeAndAnnouncement)) {
+                predownloadJobController.scope.launch {
+                    val voiceFile = retrieveVoiceFile(instruction)
+                    mainJobController.scope.launch {
+                        voiceFile.onValue { speechValue ->
+                            cachedFiles[typeAndAnnouncement] = speechValue
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    internal fun cancelPredownload() {
+        predownloadJobController.job.children.forEach { it.cancel() }
+        val announcements = cachedFiles.map { it.value.announcement }
+        announcements.forEach { clean(it) }
     }
 
     @Throws(IllegalStateException::class)
     private suspend fun retrieveVoiceFile(
         voiceInstruction: VoiceInstructions,
-        consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>
-    ) {
+    ): Expected<SpeechError, SpeechValue> {
         when (val result = voiceAPI.retrieveVoiceFile(voiceInstruction)) {
             is VoiceState.VoiceFile -> {
                 val announcement = voiceInstruction.announcement()
                 val ssmlAnnouncement = voiceInstruction.ssmlAnnouncement()
-                consumer.accept(
-                    ExpectedFactory.createValue(
-                        SpeechValue(
-                            // Can't be null as it's checked in retrieveVoiceFile
-                            SpeechAnnouncement.Builder(announcement!!)
-                                .ssmlAnnouncement(ssmlAnnouncement)
-                                .file(result.instructionFile)
-                                .build()
-                        )
+                return ExpectedFactory.createValue(
+                    SpeechValue(
+                        // Can't be null as it's checked in retrieveVoiceFile
+                        SpeechAnnouncement.Builder(announcement!!)
+                            .ssmlAnnouncement(ssmlAnnouncement)
+                            .file(result.instructionFile)
+                            .build()
                     )
                 )
             }
             is VoiceState.VoiceError -> {
                 val fallback = getFallbackAnnouncement(voiceInstruction)
                 val speechError = SpeechError(result.exception, null, fallback)
-                consumer.accept(ExpectedFactory.createError(speechError))
+                return ExpectedFactory.createError(speechError)
             }
         }
     }
@@ -117,4 +193,13 @@ class MapboxSpeechApi @JvmOverloads constructor(
             .ssmlAnnouncement(ssmlAnnouncement)
             .build()
     }
+
+    private fun hasTypeAndAnnouncement(typeAndAnnouncement: TypeAndAnnouncement): Boolean {
+        return typeAndAnnouncement in cachedFiles
+    }
+
+    private fun getFromCache(voiceInstruction: VoiceInstructions): SpeechValue? {
+        val key = VoiceInstructionsParser.parse(voiceInstruction).value
+        return key?.let { cachedFiles[it] }
+    }
 }

libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/MapboxSpeechProvider.kt

@@ -5,10 +5,12 @@
 package com.mapbox.navigation.ui.voice.api
 
 import android.net.Uri
+import com.mapbox.api.directions.v5.models.VoiceInstructions
 import com.mapbox.bindgen.Expected
 import com.mapbox.bindgen.ExpectedFactory.createError
 import com.mapbox.bindgen.ExpectedFactory.createValue
 import com.mapbox.common.ResourceLoadError
+import com.mapbox.common.ResourceLoadFlags
 import com.mapbox.common.ResourceLoadResult
 import com.mapbox.common.ResourceLoadStatus
 import com.mapbox.navigation.base.internal.accounts.UrlSkuTokenProvider
@@ -25,20 +27,27 @@ internal class MapboxSpeechProvider(
     private val language: String,
     private val urlSkuTokenProvider: UrlSkuTokenProvider,
     private val options: MapboxSpeechApiOptions,
-    private val resourceLoader: ResourceLoader
+    private val resourceLoader: ResourceLoader,
 ) {
 
-    suspend fun load(typeAndAnnouncement: TypeAndAnnouncement): Expected<Throwable, ByteArray> {
+    suspend fun load(voiceInstruction: VoiceInstructions): Expected<Throwable, ByteArray> {
         return runCatching {
-            val url = instructionUrl(typeAndAnnouncement.announcement, typeAndAnnouncement.type)
-            val response = resourceLoader.load(url)
+            val typeAndAnnouncement = VoiceInstructionsParser.parse(voiceInstruction)
+                .getValueOrElse { throw it }
+            val request = createRequest(typeAndAnnouncement)
+            val response = resourceLoader.load(request)
             return processResponse(response)
         }.getOrElse {
             createError(it)
         }
     }
 
-    private suspend fun ResourceLoader.load(url: String) = load(ResourceLoadRequest(url))
+    private fun createRequest(typeAndAnnouncement: TypeAndAnnouncement): ResourceLoadRequest {
+        val url = instructionUrl(typeAndAnnouncement.announcement, typeAndAnnouncement.type)
+        return ResourceLoadRequest(url).apply {
+            flags = ResourceLoadFlags.ACCEPT_EXPIRED
+        }
+    }
 
     private fun processResponse(
         response: Expected<ResourceLoadError, ResourceLoadResult>