Skip to content

Commit da5d44e

Browse files
hugmousehugmouse
authored
feat: Use Media3 player API for embedded media (#31)
* feat: add Media3 dependencies for inline audio/video playback * feat: add ByteArrayDataSource and GemcapPlayerManager for Media3 playback * feat: add inline audio/video players and fullscreen dialog with Media3 Compose UI * feat: wire GemcapPlayerManager through ViewModel to media composables * feat: add MediaSessionService for background audio with notification controls * feat: add download progress tracking for embedded media loading * chore: add ProGuard keep rules for Media3 service * fix: address code review issues — lifecycle, MediaSession, progress, fullscreen * feat: stream large media to disk for playback beyond 16MB limit * chore: bump AGP to 9.1.0 and Gradle to 9.3.1 * feat: active-item tracking, address bar select-all, and dot-prefix URL fix - Add currentItemId to GemcapPlayerManager with @stable annotation for single-player semantics across multiple loaded media items - Use Compose mutableStateOf for player and currentItemId reactivity - Select all address bar text on focus for easier URL editing - Fix normalizeHomeUrl to not treat dot-prefixed input as URLs * fix: media download, progress display, and playback lifecycle bugs - Support downloading file-backed media by passing dataFilePath through the callback chain instead of falling back to empty ByteArray - Fix size display showing "0B" for in-memory media by using actual data size instead of hardcoded fallback - Only release player on collapse if collapsing the active item, so unrelated playback is not disrupted - Throttle onProgress updates to every 16KB to prevent unbounded coroutine creation during downloads - Consolidate downloadProgress and downloadedBytes into a single DownloadProgress data class - Use Long instead of Int for progress callbacks to support large files - Auto-play audio/video loaded to memory for consistent UX with file-backed media - Reset playback error state when an item becomes active again - Document video-pauses / audio-continues background policy * fix: mark unstable api, clean up * fix: restrict media session service to internal access * fix: add validation and atomic writes for media data handling * fix: consolidate formatBytes, reduce polling, and stabilize DisposableEffect * fix: add main thread assertions to player methods * fix: buffer file download writes * fix: preserve playback across in-tab navigation * fix: safe file-size lookup in media cards * fix: clamp bytesRemaining to prevent out-of-bounds read * fix: safe copy fallback in file download * fix: stream file downloads instead of loading into memory * fix: stable media key, temp file cleanup, and playback guards * fix: cancellation, error handling, and media key robustness * fix: correct the comparison for media player * fix: weird color of text in embedded player * refactor(media session): wire it up to MediaSession * refactor: simplify if check for data size * fix: media notification controls and background playback * fix: keep playing media when switching tabs * fix: only pause media on background if current card is active * refactor: move video controls into separate components * refactor: reduce code duplications * fix: add OptIns to video controls * refactor: make ID of media element more unique * refactor: fetch all media into a file, check mime type after download * fix: propagate exceptions from JVM * refactor: use one ticker between slider and timestamp views
1 parent d697ca5 commit da5d44e

20 files changed

Lines changed: 1418 additions & 352 deletions

‎app/build.gradle.kts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,11 @@ dependencies {
5656
implementation(libs.coil.compose)
5757
implementation(libs.kotlinx.collections.immutable)
5858
implementation(libs.bcpkix.jdk18on)
59+
implementation(libs.androidx.media3.exoplayer)
60+
implementation(libs.androidx.media3.ui.compose)
61+
implementation(libs.androidx.media3.ui.compose.material3)
62+
implementation(libs.androidx.media3.session)
63+
implementation(libs.androidx.media3.common)
5964
testImplementation(libs.junit)
6065
androidTestImplementation(libs.androidx.junit)
6166
androidTestImplementation(libs.androidx.espresso.core)

‎app/proguard-rules.pro‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,7 @@
1818

1919
# If you keep the line number information, uncomment this to
2020
# hide the original source file name.
21-
#-renamesourcefileattribute SourceFile
21+
#-renamesourcefileattribute SourceFile
22+
23+
# Media3 - keep service for MediaSession
24+
-keep class mysh.dev.gemcap.media.GemcapMediaSessionService { *; }

‎app/src/main/AndroidManifest.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
33

44
<uses-permission android:name="android.permission.INTERNET" />
5+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
6+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
7+
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
58

69
<application
710
android:allowBackup="true"
@@ -32,6 +35,15 @@
3235
android:name="android.support.FILE_PROVIDER_PATHS"
3336
android:resource="@xml/file_paths" />
3437
</provider>
38+
39+
<service
40+
android:name=".media.GemcapMediaSessionService"
41+
android:foregroundServiceType="mediaPlayback"
42+
android:exported="false">
43+
<intent-filter>
44+
<action android:name="androidx.media3.session.MediaSessionService" />
45+
</intent-filter>
46+
</service>
3547
</application>
3648

3749
</manifest>

‎app/src/main/java/mysh/dev/gemcap/domain/GeminiModels.kt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@ sealed class GeminiContent {
105105
val url: String
106106
) : GeminiContent()
107107

108+
data class DownloadProgress(
109+
val fraction: Float, // 0.0 to ~1.0
110+
val bytesRead: Long
111+
)
112+
108113
// For media embedded in gemini pages (in-place loading like Lagrange)
109114
data class EmbeddedMedia(
110115
override val id: Int,
@@ -113,7 +118,9 @@ sealed class GeminiContent {
113118
val linkText: String,
114119
val state: EmbeddedMediaState,
115120
val data: StableByteArray? = null, // Populated when state is LOADED
116-
val errorMessage: String? = null
121+
val dataFilePath: String? = null, // Path to temp file for large media (audio/video)
122+
val errorMessage: String? = null,
123+
val downloadProgress: DownloadProgress? = null
117124
) : GeminiContent()
118125

119126
enum class EmbeddedMediaState {

‎app/src/main/java/mysh/dev/gemcap/media/ByteArrayDataSource.kt‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package mysh.dev.gemcap.media
2+
3+
import android.net.Uri
4+
import androidx.annotation.OptIn
5+
import androidx.media3.common.C
6+
import androidx.media3.common.util.UnstableApi
7+
import androidx.media3.datasource.BaseDataSource
8+
import androidx.media3.datasource.DataSource
9+
import androidx.media3.datasource.DataSpec
10+
import kotlin.math.min
11+
12+
@OptIn(UnstableApi::class)
13+
class ByteArrayDataSource(
14+
private val data: ByteArray
15+
) : BaseDataSource(/* isNetwork= */ false) {
16+
17+
private var uri: Uri? = null
18+
private var readPosition = 0
19+
private var bytesRemaining = 0
20+
21+
override fun open(dataSpec: DataSpec): Long {
22+
uri = dataSpec.uri
23+
transferInitializing(dataSpec)
24+
if (dataSpec.position > data.size) {
25+
throw java.io.IOException(
26+
"Position ${dataSpec.position} exceeds data size ${data.size}"
27+
)
28+
}
29+
readPosition = dataSpec.position.toInt()
30+
bytesRemaining = if (dataSpec.length != C.LENGTH_UNSET.toLong()) {
31+
min(dataSpec.length.toInt(), data.size - readPosition).coerceAtLeast(0)
32+
} else {
33+
data.size - readPosition
34+
}
35+
transferStarted(dataSpec)
36+
return bytesRemaining.toLong()
37+
}
38+
39+
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
40+
if (bytesRemaining == 0) return C.RESULT_END_OF_INPUT
41+
val bytesToRead = min(length, bytesRemaining)
42+
System.arraycopy(data, readPosition, buffer, offset, bytesToRead)
43+
readPosition += bytesToRead
44+
bytesRemaining -= bytesToRead
45+
bytesTransferred(bytesToRead)
46+
return bytesToRead
47+
}
48+
49+
override fun getUri(): Uri? = uri
50+
51+
override fun close() {
52+
uri = null
53+
transferEnded()
54+
}
55+
56+
class Factory(
57+
private val data: ByteArray
58+
) : DataSource.Factory {
59+
override fun createDataSource(): ByteArrayDataSource {
60+
return ByteArrayDataSource(data)
61+
}
62+
}
63+
}

‎app/src/main/java/mysh/dev/gemcap/media/GemcapMediaSessionService.kt‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package mysh.dev.gemcap.media
2+
3+
import android.content.Intent
4+
import androidx.media3.session.MediaSession
5+
import androidx.media3.session.MediaSessionService
6+
7+
/**
8+
* Wrapper service that hosts the [MediaSession] for system notification
9+
* controls. The session and player lifecycle are owned by [GemcapPlayerManager];
10+
* this service only holds a reference so [onGetSession] can return it.
11+
*
12+
* Call [publishSession] from [GemcapPlayerManager] when the session is created
13+
* or released. If the service hasn't started yet the session is kept as pending
14+
* and picked up in [onCreate].
15+
*/
16+
class GemcapMediaSessionService : MediaSessionService() {
17+
18+
private var mediaSession: MediaSession? = null
19+
20+
override fun onCreate() {
21+
super.onCreate()
22+
instance = this
23+
pendingSession?.let {
24+
mediaSession = it
25+
addSession(it)
26+
pendingSession = null
27+
}
28+
}
29+
30+
override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? {
31+
return mediaSession
32+
}
33+
34+
override fun onTaskRemoved(rootIntent: Intent?) {
35+
val player = mediaSession?.player
36+
if (player == null || !player.playWhenReady || player.mediaItemCount == 0) {
37+
stopSelf()
38+
}
39+
super.onTaskRemoved(rootIntent)
40+
}
41+
42+
override fun onDestroy() {
43+
mediaSession = null
44+
instance = null
45+
super.onDestroy()
46+
}
47+
48+
companion object {
49+
private var instance: GemcapMediaSessionService? = null
50+
private var pendingSession: MediaSession? = null
51+
52+
/**
53+
* Publishes or clears the [MediaSession] so that [onGetSession] can
54+
* return it. Called from [GemcapPlayerManager] when the session is
55+
* created or released.
56+
*/
57+
internal fun publishSession(session: MediaSession?) {
58+
pendingSession = session
59+
instance?.let { service ->
60+
service.mediaSession?.let { service.removeSession(it) }
61+
service.mediaSession = session
62+
session?.let { service.addSession(it) }
63+
}
64+
}
65+
}
66+
}

‎app/src/main/java/mysh/dev/gemcap/media/GemcapPlayerManager.kt‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package mysh.dev.gemcap.media
2+
3+
import android.content.Context
4+
import android.content.Intent
5+
import android.os.Looper
6+
import android.util.Log
7+
import androidx.annotation.OptIn
8+
import androidx.compose.runtime.Stable
9+
import androidx.compose.runtime.getValue
10+
import androidx.compose.runtime.mutableStateOf
11+
import androidx.compose.runtime.setValue
12+
import androidx.media3.common.AudioAttributes
13+
import androidx.media3.common.C
14+
import androidx.media3.common.MediaItem
15+
import androidx.media3.common.util.UnstableApi
16+
import androidx.media3.exoplayer.ExoPlayer
17+
import androidx.media3.exoplayer.source.ProgressiveMediaSource
18+
import androidx.media3.session.MediaSession
19+
20+
@Stable
21+
class GemcapPlayerManager(private val context: Context) {
22+
23+
var player: ExoPlayer? by mutableStateOf(null)
24+
private set
25+
26+
var currentMediaKey: String? by mutableStateOf(null)
27+
private set
28+
29+
private var mediaSession: MediaSession? = null
30+
31+
private fun getOrCreatePlayer(): ExoPlayer {
32+
player?.let { return it }
33+
val newPlayer = ExoPlayer.Builder(context).build().apply {
34+
setAudioAttributes(
35+
AudioAttributes.Builder()
36+
.setContentType(C.AUDIO_CONTENT_TYPE_MUSIC)
37+
.setUsage(C.USAGE_MEDIA)
38+
.build(),
39+
/* handleAudioFocus= */ true
40+
)
41+
}
42+
player = newPlayer
43+
mediaSession = MediaSession.Builder(context, newPlayer).build()
44+
GemcapMediaSessionService.publishSession(mediaSession)
45+
context.startService(Intent(context, GemcapMediaSessionService::class.java))
46+
return newPlayer
47+
}
48+
49+
/** Must be called on the main thread. */
50+
@OptIn(UnstableApi::class)
51+
fun play(data: ByteArray, mimeType: String, mediaKey: String? = null) {
52+
check(Looper.myLooper() == Looper.getMainLooper()) { "play() must be called on the main thread" }
53+
val exoPlayer = getOrCreatePlayer()
54+
exoPlayer.stop()
55+
exoPlayer.clearMediaItems()
56+
57+
currentMediaKey = mediaKey
58+
val dataSourceFactory = ByteArrayDataSource.Factory(data)
59+
val mediaItem = MediaItem.Builder()
60+
.setUri("gemini://local/media")
61+
.setMimeType(mimeType)
62+
.build()
63+
val mediaSource = ProgressiveMediaSource.Factory(dataSourceFactory)
64+
.createMediaSource(mediaItem)
65+
exoPlayer.setMediaSource(mediaSource)
66+
exoPlayer.prepare()
67+
exoPlayer.playWhenReady = true
68+
}
69+
70+
/** Must be called on the main thread. */
71+
@OptIn(UnstableApi::class)
72+
fun playFromFile(file: java.io.File, mimeType: String, mediaKey: String? = null): Boolean {
73+
check(Looper.myLooper() == Looper.getMainLooper()) { "playFromFile() must be called on the main thread" }
74+
if (!file.exists() || !file.canRead()) {
75+
Log.e(TAG, "Cannot play file: ${file.absolutePath} (exists=${file.exists()}, canRead=${file.canRead()})")
76+
return false
77+
}
78+
val exoPlayer = getOrCreatePlayer()
79+
exoPlayer.stop()
80+
exoPlayer.clearMediaItems()
81+
82+
currentMediaKey = mediaKey
83+
val mediaItem = MediaItem.Builder()
84+
.setUri(android.net.Uri.fromFile(file))
85+
.setMimeType(mimeType)
86+
.build()
87+
exoPlayer.setMediaItem(mediaItem)
88+
exoPlayer.prepare()
89+
exoPlayer.playWhenReady = true
90+
return true
91+
}
92+
93+
fun release() {
94+
GemcapMediaSessionService.publishSession(null)
95+
context.stopService(Intent(context, GemcapMediaSessionService::class.java))
96+
currentMediaKey = null
97+
mediaSession?.release()
98+
mediaSession = null
99+
player?.release()
100+
player = null
101+
}
102+
103+
companion object {
104+
private const val TAG = "GemcapPlayerManager"
105+
}
106+
}

0 commit comments

Comments
 (0)