Skip to content

Commit 51479de

Browse files
richiemcilroyrichiemcilroy
authored
Merge pull request #1702 from CapSoftware/feat/web-raw-playback-fallback
feat(web): fall back to raw upload when MP4 playback fails
2 parents bd987b3 + 41bc0aa commit 51479de

File tree

15 files changed

+446
-719
lines changed

15 files changed

+446
-719
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎apps/cli/src/main.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ impl Export {
169169
compression: cap_export::mp4::ExportCompression::Maximum,
170170
custom_bpp: None,
171171
force_ffmpeg_decoder: false,
172+
optimize_filesize: false,
172173
}
173174
.export(exporter_base, move |_f| {
174175
// print!("\rrendered frame {f}");

‎apps/web/__tests__/unit/playback-source.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
canPlayRawContentType,
44
detectCrossOriginSupport,
55
resolvePlaybackSource,
6+
shouldFallbackToRawPlaybackSource,
67
} from "@/app/s/[videoId]/_components/playback-source";
78

89
function createResponse(
@@ -149,6 +150,40 @@ describe("resolvePlaybackSource", () => {
149150
});
150151
});
151152

153+
it("can prefer the raw preview after the MP4 source fails in the player", async () => {
154+
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
155+
createResponse("https://cap.so/raw-upload.webm", {
156+
status: 206,
157+
headers: { "content-type": "video/webm;codecs=vp9,opus" },
158+
redirected: true,
159+
}),
160+
);
161+
162+
const result = await resolvePlaybackSource({
163+
videoSrc: "/api/playlist?videoType=mp4",
164+
rawFallbackSrc: "/api/playlist?videoType=raw-preview",
165+
preferredSource: "raw",
166+
fetchImpl,
167+
now: () => 250,
168+
createVideoElement: () => ({
169+
canPlayType: vi.fn().mockReturnValue("probably"),
170+
}),
171+
});
172+
173+
expect(fetchImpl).toHaveBeenCalledTimes(1);
174+
expect(fetchImpl).toHaveBeenCalledWith(
175+
"/api/playlist?videoType=raw-preview&_t=250",
176+
{
177+
headers: { range: "bytes=0-0" },
178+
},
179+
);
180+
expect(result).toEqual({
181+
url: "https://cap.so/raw-upload.webm",
182+
type: "raw",
183+
supportsCrossOrigin: false,
184+
});
185+
});
186+
152187
it("rejects raw webm previews when the browser cannot play them", async () => {
153188
const fetchImpl = vi
154189
.fn<typeof fetch>()
@@ -196,3 +231,14 @@ describe("resolvePlaybackSource", () => {
196231
expect(result).toBeNull();
197232
});
198233
});
234+
235+
describe("shouldFallbackToRawPlaybackSource", () => {
236+
it("allows a single mp4-to-raw fallback", () => {
237+
expect(shouldFallbackToRawPlaybackSource("mp4", "/raw", false)).toBe(true);
238+
expect(shouldFallbackToRawPlaybackSource("mp4", "/raw", true)).toBe(false);
239+
expect(shouldFallbackToRawPlaybackSource("raw", "/raw", true)).toBe(false);
240+
expect(shouldFallbackToRawPlaybackSource("mp4", undefined, false)).toBe(
241+
false,
242+
);
243+
});
244+
});

‎apps/web/app/api/playlist/route.ts‎

Lines changed: 52 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,47 @@ const ApiLive = HttpApiBuilder.api(Api).pipe(
8383
),
8484
);
8585

86+
const resolveRawPreviewKey = (video: Video.Video) =>
87+
Effect.gen(function* () {
88+
const db = yield* Database;
89+
const [s3] = yield* S3Buckets.getBucketAccess(video.bucketId);
90+
const [uploadRecord] = yield* db.use((db) =>
91+
db
92+
.select({ rawFileKey: Db.videoUploads.rawFileKey })
93+
.from(Db.videoUploads)
94+
.where(eq(Db.videoUploads.videoId, video.id)),
95+
);
96+
97+
if (uploadRecord?.rawFileKey) {
98+
return uploadRecord.rawFileKey;
99+
}
100+
101+
if (video.source.type !== "webMP4") {
102+
return yield* Effect.fail(new HttpApiError.NotFound());
103+
}
104+
105+
const candidateKeys = [
106+
`${video.ownerId}/${video.id}/raw-upload.mp4`,
107+
`${video.ownerId}/${video.id}/raw-upload.webm`,
108+
];
109+
const headResults = yield* Effect.all(
110+
candidateKeys.map((key) => s3.headObject(key).pipe(Effect.option)),
111+
{ concurrency: "unbounded" },
112+
);
113+
for (const [index, candidateKey] of candidateKeys.entries()) {
114+
const rawHead = headResults[index];
115+
if (
116+
rawHead &&
117+
Option.isSome(rawHead) &&
118+
(rawHead.value.ContentLength ?? 0) > 0
119+
) {
120+
return candidateKey;
121+
}
122+
}
123+
124+
return yield* Effect.fail(new HttpApiError.NotFound());
125+
});
126+
86127
const getPlaylistResponse = (
87128
video: Video.Video,
88129
urlParams: (typeof GetPlaylistParams)["Type"],
@@ -93,20 +134,9 @@ const getPlaylistResponse = (
93134
video.source.type === "desktopMP4" || video.source.type === "webMP4";
94135

95136
if (urlParams.videoType === "raw-preview") {
96-
const db = yield* Database;
97-
const [uploadRecord] = yield* db.use((db) =>
98-
db
99-
.select({ rawFileKey: Db.videoUploads.rawFileKey })
100-
.from(Db.videoUploads)
101-
.where(eq(Db.videoUploads.videoId, urlParams.videoId)),
102-
);
103-
104-
if (!uploadRecord?.rawFileKey) {
105-
return yield* Effect.fail(new HttpApiError.NotFound());
106-
}
107-
137+
const rawFileKey = yield* resolveRawPreviewKey(video);
108138
return yield* s3
109-
.getSignedObjectUrl(uploadRecord.rawFileKey)
139+
.getSignedObjectUrl(rawFileKey)
110140
.pipe(Effect.map(HttpServerResponse.redirect));
111141
}
112142

@@ -201,12 +231,17 @@ const getPlaylistResponse = (
201231
s3.listObjects({ prefix: audioPrefix, maxKeys: 1 }),
202232
]);
203233

204-
const videoMetadata = yield* s3.headObject(
205-
videoSegment.Contents?.[0]?.Key ?? "",
206-
);
234+
const videoSegmentKey = videoSegment.Contents?.[0]?.Key;
235+
if (!videoSegmentKey) {
236+
return yield* Effect.fail(new HttpApiError.NotFound());
237+
}
238+
239+
const videoMetadata = yield* s3.headObject(videoSegmentKey);
207240
const audioMetadata =
208241
audioSegment?.KeyCount && audioSegment.KeyCount > 0
209-
? yield* s3.headObject(audioSegment.Contents?.[0]?.Key ?? "")
242+
? audioSegment.Contents?.[0]?.Key
243+
? yield* s3.headObject(audioSegment.Contents[0].Key)
244+
: undefined
210245
: undefined;
211246

212247
const generatedPlaylist = generateMasterPlaylist(

‎apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
import {
2424
type ResolvedPlaybackSource,
2525
resolvePlaybackSource,
26+
shouldFallbackToRawPlaybackSource,
2627
} from "./playback-source";
2728
import {
2829
MediaPlayer,
@@ -139,6 +140,8 @@ export function CapVideoPlayer({
139140
const [hasError, setHasError] = useState(false);
140141
const [isRetryingProcessing, setIsRetryingProcessing] = useState(false);
141142
const [playerDuration, setPlayerDuration] = useState(fallbackDuration ?? 0);
143+
const [preferredSource, setPreferredSource] = useState<"mp4" | "raw">("mp4");
144+
const [hasTriedRawFallback, setHasTriedRawFallback] = useState(false);
142145
const queryClient = useQueryClient();
143146

144147
useEffect(() => {
@@ -166,14 +169,21 @@ export function CapVideoPlayer({
166169
const shouldDeferResolvedSource = shouldDeferPlaybackSource(uploadProgress);
167170

168171
const resolvedSrc = useQuery<ResolvedPlaybackSource | null>({
169-
queryKey: ["resolvedSrc", videoSrc, rawFallbackSrc, enableCrossOrigin],
172+
queryKey: [
173+
"resolvedSrc",
174+
videoSrc,
175+
rawFallbackSrc,
176+
enableCrossOrigin,
177+
preferredSource,
178+
],
170179
queryFn: shouldDeferResolvedSource
171180
? skipToken
172181
: () =>
173182
resolvePlaybackSource({
174183
videoSrc,
175184
rawFallbackSrc,
176185
enableCrossOrigin,
186+
preferredSource,
177187
}),
178188
refetchOnWindowFocus: false,
179189
staleTime: Number.POSITIVE_INFINITY,
@@ -186,6 +196,8 @@ export function CapVideoPlayer({
186196
setVideoLoaded(false);
187197
setHasError(false);
188198
setShowPlayButton(false);
199+
setPreferredSource("mp4");
200+
setHasTriedRawFallback(false);
189201
}, [videoSrc, rawFallbackSrc]);
190202

191203
// Track video duration for comment markers
@@ -280,6 +292,21 @@ export function CapVideoPlayer({
280292
};
281293

282294
const handleError = () => {
295+
if (
296+
shouldFallbackToRawPlaybackSource(
297+
resolvedSrc.data?.type,
298+
rawFallbackSrc,
299+
hasTriedRawFallback,
300+
)
301+
) {
302+
setHasTriedRawFallback(true);
303+
setPreferredSource("raw");
304+
setVideoLoaded(false);
305+
setHasError(false);
306+
setShowPlayButton(false);
307+
return;
308+
}
309+
283310
setHasError(true);
284311
};
285312

@@ -365,7 +392,14 @@ export function CapVideoPlayer({
365392
captionTrack.removeEventListener("cuechange", handleCueChange);
366393
}
367394
};
368-
}, [hasPlayedOnce, resolvedSrc.isPending, videoRef.current]);
395+
}, [
396+
hasPlayedOnce,
397+
hasTriedRawFallback,
398+
rawFallbackSrc,
399+
resolvedSrc.data?.type,
400+
resolvedSrc.isPending,
401+
videoRef.current,
402+
]);
369403

370404
const generateVideoFrameThumbnail = useCallback(
371405
(time: number): string => {
@@ -443,12 +477,19 @@ export function CapVideoPlayer({
443477
) {
444478
setHasError(false);
445479
void queryClient.invalidateQueries({
446-
queryKey: ["resolvedSrc", videoSrc, rawFallbackSrc, enableCrossOrigin],
480+
queryKey: [
481+
"resolvedSrc",
482+
videoSrc,
483+
rawFallbackSrc,
484+
enableCrossOrigin,
485+
preferredSource,
486+
],
447487
});
448488
}
449489
prevUploadProgress.current = uploadProgress;
450490
}, [
451491
enableCrossOrigin,
492+
preferredSource,
452493
queryClient,
453494
rawFallbackSrc,
454495
uploadProgress,

‎apps/web/app/s/[videoId]/_components/playback-source.ts‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ type ResolvePlaybackSourceInput = {
1818
fetchImpl?: typeof fetch;
1919
now?: () => number;
2020
createVideoElement?: () => Pick<HTMLVideoElement, "canPlayType">;
21+
preferredSource?: "mp4" | "raw";
2122
};
2223

2324
function appendCacheBust(url: string, timestamp: number): string {
@@ -92,14 +93,56 @@ export function canPlayRawContentType(
9293
);
9394
}
9495

96+
export function shouldFallbackToRawPlaybackSource(
97+
resolvedSourceType: ResolvedPlaybackSource["type"] | null | undefined,
98+
rawFallbackSrc: string | undefined,
99+
hasTriedRawFallback: boolean,
100+
): boolean {
101+
return Boolean(
102+
rawFallbackSrc && resolvedSourceType === "mp4" && !hasTriedRawFallback,
103+
);
104+
}
105+
95106
export async function resolvePlaybackSource({
96107
videoSrc,
97108
rawFallbackSrc,
98109
enableCrossOrigin = false,
99110
fetchImpl = fetch,
100111
now = () => Date.now(),
101112
createVideoElement,
113+
preferredSource = "mp4",
102114
}: ResolvePlaybackSourceInput): Promise<ResolvedPlaybackSource | null> {
115+
const resolveRaw = async (): Promise<ResolvedPlaybackSource | null> => {
116+
if (!rawFallbackSrc) {
117+
return null;
118+
}
119+
120+
const rawResult = await probePlaybackSource(rawFallbackSrc, fetchImpl, now);
121+
122+
if (!rawResult) {
123+
return null;
124+
}
125+
126+
const contentType = rawResult.response.headers.get("content-type") ?? "";
127+
128+
if (
129+
!canPlayRawContentType(contentType, rawResult.url, createVideoElement)
130+
) {
131+
return null;
132+
}
133+
134+
return {
135+
url: rawResult.url,
136+
type: "raw",
137+
supportsCrossOrigin:
138+
enableCrossOrigin && detectCrossOriginSupport(rawResult.url),
139+
};
140+
};
141+
142+
if (preferredSource === "raw") {
143+
return await resolveRaw();
144+
}
145+
103146
const mp4Result = await probePlaybackSource(videoSrc, fetchImpl, now);
104147

105148
if (mp4Result) {
@@ -111,26 +154,5 @@ export async function resolvePlaybackSource({
111154
};
112155
}
113156

114-
if (!rawFallbackSrc) {
115-
return null;
116-
}
117-
118-
const rawResult = await probePlaybackSource(rawFallbackSrc, fetchImpl, now);
119-
120-
if (!rawResult) {
121-
return null;
122-
}
123-
124-
const contentType = rawResult.response.headers.get("content-type") ?? "";
125-
126-
if (!canPlayRawContentType(contentType, rawResult.url, createVideoElement)) {
127-
return null;
128-
}
129-
130-
return {
131-
url: rawResult.url,
132-
type: "raw",
133-
supportsCrossOrigin:
134-
enableCrossOrigin && detectCrossOriginSupport(rawResult.url),
135-
};
157+
return await resolveRaw();
136158
}

‎crates/cap-test/src/suites/performance.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,7 @@ async fn benchmark_export(recording_path: &Path) -> Result<ExportMetrics> {
546546
compression: ExportCompression::Social,
547547
custom_bpp: None,
548548
force_ffmpeg_decoder: false,
549+
optimize_filesize: false,
549550
};
550551

551552
let total_frames = exporter_base.total_frames(settings.fps);

‎crates/export/examples/export-benchmark-runner.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,7 @@ async fn run_mp4_export(
428428
compression,
429429
custom_bpp: None,
430430
force_ffmpeg_decoder: false,
431+
optimize_filesize: false,
431432
};
432433

433434
let total_frames = exporter_base.total_frames(fps);

‎crates/export/examples/export_startup_time.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ async fn main() -> Result<(), String> {
1818
compression: ExportCompression::Maximum,
1919
custom_bpp: None,
2020
force_ffmpeg_decoder: false,
21+
optimize_filesize: false,
2122
};
2223

2324
let temp_out = tempfile::Builder::new()

0 commit comments

Comments
 (0)