feat(ios): Support for external subtitles on HLS streaming

examples/common/constants/general.ts

@@ -21,6 +21,18 @@ export const isIos = Platform.OS === 'ios';
 export const isAndroid = Platform.OS === 'android';
 
 export const srcAllPlatformList = [
+  {
+    description: 'HLS external texttrack',
+    uri: 'https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8',
+    textTracks: [
+      {
+        title: 'test',
+        language: 'en' as ISO639_1,
+        type: TextTrackType.VTT,
+        uri: 'https://gist.githubusercontent.com/chdemko/5356310/raw/c8f1e1424e8bbb2ffacd28541bfac87b351af8a4/example.vtt',
+      },
+    ],
+  },
   {
     description: 'local file landscape',
     uri: localeVideo.broadchurch,

ios/Video/Features/PlayerItem+HLS.swift

@@ -0,0 +1,225 @@
+//  Source: https://github.com/jbweimar/external-webvtt-example/blob/master/External%20WebVTT%20Example/CustomResourceLoaderDelegate.swift
+
+import AVFoundation
+import AVKit
+import Foundation
+
+extension PlayerItem {
+    static let hlsCommonScheme = "rnvideo-hls"
+    private static let hlsSubtitlesScheme = "rnvideo-hls-subtitles"
+    private static let hlsSubtitleGroupID = "rnvideo-sub"
+
+    func handleHLSRequest(loadingRequest: AVAssetResourceLoadingRequest) -> Bool {
+        guard let url = loadingRequest.request.url else {
+            return false
+        }
+
+        switch url.scheme {
+        case Self.hlsCommonScheme:
+            return handleM3U8(url, loadingRequest)
+        case Self.hlsSubtitlesScheme:
+            return handleSubtitleM3U8(url, loadingRequest)
+        default:
+            return false
+        }
+    }
+
+    private func downloadM3U8(_ url: URL, _ downloaded: @escaping (Result<Data, Error>) -> Void) -> Bool {
+        let task = URLSession.shared.dataTask(with: url) { data, _, error in
+            if let error {
+                downloaded(.failure(error))
+                return
+            }
+
+            downloaded(.success(data ?? .init()))
+        }
+        task.priority = URLSessionTask.highPriority
+        task.resume()
+        return true
+    }
+
+    private func handleM3U8(_ requestingUrl: URL, _ loadingRequest: AVAssetResourceLoadingRequest) -> Bool {
+        guard let m3u8Uri = URL(string: payload.uri ?? "") else {
+            return false
+        }
+        // Check if the requested url is the initial modified link
+        if requestingUrl == payload.modifiedLink {
+            return downloadM3U8(m3u8Uri) { [weak self] result in
+                switch result {
+                case let .success(data):
+                    guard let string = String(data: data, encoding: .utf8) else {
+                        loadingRequest.finishLoading(with: nil)
+                        return
+                    }
+
+                    let playlistData: Data
+
+                    if string.contains("#EXT-X-STREAM-INF") {
+                        let mainM3U8 = self?.parseMainMultiVariantPlaylist(string)
+                        playlistData = mainM3U8?.data(using: .utf8) ?? data
+                    } else {
+                        let m3u8 = self?.convertMainPlaylistToMultivariant(string)
+                        playlistData = m3u8?.data(using: .utf8) ?? data
+                    }
+
+                    loadingRequest.contentInformationRequest?.contentType = "public.m3u-playlist"
+                    loadingRequest.contentInformationRequest?.contentLength = Int64(data.count)
+                    loadingRequest.contentInformationRequest?.isByteRangeAccessSupported = true
+
+                    loadingRequest.dataRequest?.respond(with: playlistData)
+                    loadingRequest.finishLoading()
+
+                case let .failure(error):
+                    loadingRequest.finishLoading(with: error)
+                }
+            }
+        } else {
+            var request = URLRequest(url: requestingUrl.recoveryScheme)
+            request.httpMethod = "GET"
+            request.allHTTPHeaderFields = loadingRequest.request.allHTTPHeaderFields ?? [:]
+            loadingRequest.redirect = request
+
+            loadingRequest.response = HTTPURLResponse(
+                url: requestingUrl.recoveryScheme,
+                statusCode: 302,
+                httpVersion: nil,
+                headerFields: loadingRequest.request.allHTTPHeaderFields ?? [:]
+            )
+
+            loadingRequest.finishLoading()
+        }
+
+        return true
+    }
+
+    private func handleSubtitleM3U8(_ requestingUrl: URL, _ loadingRequest: AVAssetResourceLoadingRequest) -> Bool {
+        guard let withoutSchema = requestingUrl.absoluteString.components(separatedBy: "://").last else {
+            return false
+        }
+
+        guard let index = withoutSchema.components(separatedBy: ".").first.flatMap({ Int($0) }) else {
+            return false
+        }
+
+        guard let textTrack = payload.textTracks.enumerated().first(where: { $0.offset == index })?.element else {
+            return false
+        }
+
+        guard let textTrackUri = URL(string: textTrack.uri) else {
+            return false
+        }
+
+        let task = URLSession.shared.dataTask(with: .init(url: textTrackUri)) { data, _, error in
+            guard error == nil else {
+                return loadingRequest.finishLoading(with: error)
+            }
+
+            guard let vttString = String(data: data ?? .init(), encoding: .utf8) else {
+                return loadingRequest.finishLoading(with: nil)
+            }
+
+            let lastTimeStampString = (
+                try? NSRegularExpression(pattern: "(?:(\\d+):)?(\\d+):([\\d\\.]+)")
+                    .matches(
+                        in: vttString,
+                        range: .init(location: 0, length: vttString.utf16.count)
+                    )
+                    .last
+                    .flatMap { Range($0.range, in: vttString) }
+                    .flatMap { String(vttString[$0]) }
+            ) ?? "0.000"
+
+            let duration = lastTimeStampString.components(separatedBy: ":").reversed()
+                .compactMap { Double($0) }
+                .enumerated()
+                .map { pow(60.0, Double($0.offset)) * $0.element }
+                .reduce(0, +)
+
+            let m3u8Subtitle = """
+            #EXTM3U
+            #EXT-X-VERSION:3
+            #EXT-X-MEDIA-SEQUENCE:1
+            #EXT-X-PLAYLIST-TYPE:VOD
+            #EXT-X-ALLOW-CACHE:NO
+            #EXT-X-TARGETDURATION:\(Int(duration))
+            #EXTINF:\(String(format: "%.3f", duration)), no desc
+            \(textTrackUri.absoluteString)
+            #EXT-X-ENDLIST
+            """
+
+            let m3u8Data = m3u8Subtitle.data(using: .utf8) ?? .init()
+
+            loadingRequest.dataRequest?.respond(with: m3u8Data)
+
+            loadingRequest.contentInformationRequest?.contentType = "public.m3u-playlist"
+            loadingRequest.contentInformationRequest?.contentLength = Int64(m3u8Data.count)
+            loadingRequest.contentInformationRequest?.isByteRangeAccessSupported = true
+
+            loadingRequest.finishLoading()
+        }
+        task.resume()
+        return true
+    }
+
+    private func parseMainMultiVariantPlaylist(_ m3u8String: String) -> String {
+        var lines = m3u8String.split(separator: "\n", omittingEmptySubsequences: false).map { String($0) }
+        var lastPositionMedia: Int?
+        var firstPositionInf = 1
+
+        for (idx, line) in lines.enumerated() {
+            if line.hasPrefix("#EXT-X-STREAM-INF") {
+                firstPositionInf = idx
+                break
+            } else if line.hasPrefix("#EXT-X-MEDIA") {
+                lastPositionMedia = idx + 1
+            }
+        }
+
+        var subtitlePosition = lastPositionMedia ?? firstPositionInf
+
+        for (idx, subtitle) in payload.textTracks.enumerated() {
+            let m3u8SubtitlesString = makeSubtitleTypes(idx, subtitle)
+            if subtitlePosition <= lines.endIndex {
+                lines.insert(m3u8SubtitlesString, at: subtitlePosition)
+            } else {
+                lines.append(m3u8SubtitlesString)
+            }
+            subtitlePosition += 1
+        }
+
+        for (idx, line) in lines.enumerated() where line.contains("#EXT-X-STREAM-INF") {
+            lines[idx] = line + "," + "SUBTITLES=\"\(Self.hlsSubtitleGroupID)\""
+        }
+
+        return lines.joined(separator: "\n")
+    }
+
+    private func convertMainPlaylistToMultivariant(_: String) -> String {
+        // Build a multivariant playlist out of a single main playlist
+        let subtitlesMediaStrings = payload.textTracks.enumerated()
+            .map(makeSubtitleTypes)
+
+        return """
+        #EXTM3U
+        \(subtitlesMediaStrings.joined(separator: "\n"))
+        #EXT-X-STREAM-INF:BANDWIDTH=640000,SUBTITLES="\(Self.hlsSubtitleGroupID)"
+        \(payload.uri ?? "")
+        """
+    }
+
+    private func makeSubtitleTypes(_ idx: Int, _ textTrack: TextTrack) -> String {
+        "#EXT-X-MEDIA:" + [
+            "TYPE": "SUBTITLES",
+            "GROUP-ID": "\"\(Self.hlsSubtitleGroupID)\"",
+            "NAME": "\"\(textTrack.title)\"",
+            "CHARACTERISTICS": "\"public.accessibility.transcribes-spoken-dialog\"",
+            "DEFAULT": idx == 0 ? "YES" : "NO",
+            "AUTOSELECT": idx == 0 ? "YES" : "NO",
+            "FORCED": idx == 0 ? "YES" : "NO",
+            "URI": "\"\(Self.hlsSubtitlesScheme)://\(idx).subtitle.m3u8\"",
+            "LANGUAGE": "\"\(textTrack.language)\"",
+        ]
+        .map { "\($0.key)=\($0.value)" }
+        .joined(separator: ",")
+    }
+}

ios/Video/Features/PlayerItem.swift

@@ -0,0 +1,82 @@
+// Big thanks to https://github.com/erikbdev
+
+import AVFoundation
+import AVKit
+import Foundation
+
+// MARK: - MetaPayload
+
+@dynamicMemberLookup
+struct MetaPayload {
+    let modifiedLink: URL
+    let payload: VideoSource
+
+    subscript<Value>(dynamicMember keyPath: KeyPath<VideoSource, Value>) -> Value {
+        payload[keyPath: keyPath]
+    }
+}
+
+// MARK: - PlayerItem
+
+final class PlayerItem: AVPlayerItem {
+    var payload: MetaPayload
+
+    private let resourceQueue: DispatchQueue
+
+    enum ResourceLoaderError: Swift.Error {
+        case responseError
+        case emptyData
+        case failedToCreateM3U8
+    }
+
+    init(source: VideoSource, uri: String) throws {
+        guard let url = URL(string: uri) else {
+            DebugLog("Could not find video URL in source '\(String(describing: source))'")
+            throw NSError(domain: "", code: 0, userInfo: nil)
+        }
+        self.resourceQueue = DispatchQueue(label: "playeritem-\(url.absoluteString)", qos: .utility)
+
+        if source.textTracks.isEmpty || url.isFileURL {
+            self.payload = .init(modifiedLink: url, payload: source)
+        } else {
+            self.payload = .init(modifiedLink: url.change(scheme: Self.hlsCommonScheme), payload: source)
+        }
+        let modifiedSource = source.json
+        modifiedSource!.setValue(self.payload.modifiedLink.absoluteString, forKey: "uri")
+        guard let assetResult = RCTVideoUtils.prepareAsset(source: VideoSource(modifiedSource)),
+              let asset = assetResult.asset else {
+            DebugLog("Could not find video URL in source '\(String(describing: source))'")
+            throw NSError(domain: "", code: 0, userInfo: nil)
+        }
+
+        super.init(asset: asset as AVAsset, automaticallyLoadedAssetKeys: nil)
+
+        asset.resourceLoader.setDelegate(self, queue: resourceQueue)
+    }
+}
+
+// MARK: AVAssetResourceLoaderDelegate
+
+extension PlayerItem: AVAssetResourceLoaderDelegate {
+    func resourceLoader(
+        _: AVAssetResourceLoader,
+        shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest
+    ) -> Bool {
+        handleHLSRequest(loadingRequest: loadingRequest)
+    }
+}
+
+extension URL {
+    func change(scheme: String) -> URL {
+        var component = URLComponents(url: self, resolvingAgainstBaseURL: false)
+        component?.scheme = scheme
+        return component?.url ?? self
+    }
+
+    var recoveryScheme: URL {
+        var component = URLComponents(url: self, resolvingAgainstBaseURL: false)
+        let isDataScheme = component?.path.starts(with: "application/x-mpegURL") ?? false
+        component?.scheme = isDataScheme ? "data" : "https"
+        return component?.url ?? self
+    }
+}

ios/Video/RCTVideo.swift

@@ -696,7 +696,11 @@ class RCTVideo: UIView, RCTVideoPlayerViewControllerDelegate, RCTPlayerObserverH
 
     func playerItemPrepareText(source: VideoSource, asset: AVAsset!, assetOptions: NSDictionary?, uri: String) async -> AVPlayerItem {
         if source.textTracks.isEmpty == true || uri.hasSuffix(".m3u8") {
-            return await self.playerItemPropegateMetadata(AVPlayerItem(asset: asset))
+            do {
+                return try await self.playerItemPropegateMetadata(PlayerItem(source: source, uri: uri))
+            } catch {
+                return await self.playerItemPropegateMetadata(AVPlayerItem(asset: asset))
+            }
         }
 
         // AVPlayer can't airplay AVMutableCompositions
@@ -1051,7 +1055,7 @@ class RCTVideo: UIView, RCTVideoPlayerViewControllerDelegate, RCTPlayerObserverH
     func setSelectedTextTrack(_ selectedTextTrack: SelectedTrackCriteria?) {
         _selectedTextTrackCriteria = selectedTextTrack ?? SelectedTrackCriteria.none()
         guard let source = _source else { return }
-        if !source.textTracks.isEmpty { // sideloaded text tracks
+        if !source.textTracks.isEmpty && !(source.uri?.hasSuffix(".m3u8") ?? true) { // sideloaded text tracks
             RCTPlayerOperations.setSideloadedText(player: _player, textTracks: source.textTracks, criteria: _selectedTextTrackCriteria)
         } else { // text tracks included in the HLS playlist
             Task { [weak self] in