Fix fetch transcript (#2083)

download wasm files every time

Author: w3cj

.gitignore

@@ -25,4 +25,7 @@ test-results
 start_script/target
 bin
 
-db-data
+db-data
+
+core.wasm
+core.worker.cjs

package.json

@@ -6,9 +6,7 @@
 		"dev": "node ./scripts/preheat.js",
 		"preheat": "node ./scripts/preheat.js --env-only",
 		"vite-dev": "vite dev",
-		"copy-files": "zx ./why_do_i_need_this.mjs",
-		"build": "run-s --print-name build:svelte copy-files",
-		"build:svelte": "vite build",
+		"build": "vite build",
 		"preview": "vite preview",
 		"test": "svelte-kit sync && playwright test",
 		"test:ui": "svelte-kit sync && playwright test --ui",

src/server/transcripts/flagger.ts

@@ -1,19 +1,33 @@
-// import { createFFmpeg, fetchFile, type ProgressCallback } from '@ffmpeg.wasm/main';
+import { createWriteStream, existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { Readable } from 'stream';
+import { finished } from 'stream/promises';
 import { FFmpeg } from '@ffmpeg.wasm/main';
 import type { Show } from '@prisma/client';
-import { readFile, readdir } from 'fs/promises';
-import { logProgress } from './logProgress';
 import core from '@ffmpeg.wasm/core-mt';
-const flagPaths = ['./audio/wes-flagger.mp3', './audio/scott-flagger.mp3'];
-import wasmPathAb from '@ffmpeg.wasm/core-mt/dist/core.wasm?url';
 import { env } from '$env/dynamic/private';
+import { logProgress } from './logProgress';
+
+const flag_paths = ['./audio/wes-flagger.mp3', './audio/scott-flagger.mp3'];
+
 export type ProgressEvent = {
 	duration?: number;
 	ratio?: number;
 	time: number;
 	percentage: number;
 };
 
+async function downloadFile(url: string, path: string) {
+	if (existsSync(path)) return;
+
+	const stream = createWriteStream(path);
+	const response = await fetch(url);
+	if (response.body) {
+		// @ts-expect-error body is readable
+		await finished(Readable.fromWeb(response.body).pipe(stream));
+	}
+}
+
 /**
  * Concatenates a show with flagger audio to help with diatirization
  * @returns {Promise<Buffer>} - The concatenated show
@@ -23,71 +37,73 @@ export async function addFlaggerAudio(show: Show): Promise<Buffer> {
 	console.log('ADDING FLAGGER AUDIO');
 	const url = new URL(show.url);
 	// Get the filename
-	const fileName = `${show.number}.mp3`;
+	const file_name = `${show.number}.mp3`;
 	//  Get the base name
-	const [baseName, extension] = fileName.split('.');
+	const [base_name, extension] = file_name.split('.');
 	// create the output filename
-	const outputFilename = `${show.number}-flagged.${extension}`;
+	const output_filename = `${show.number}-flagged.${extension}`;
 	console.log(`Downloading #${show.number} - ${show.title}`);
 	console.log('Creating ffmpeg instance');
+	await downloadFile(
+		'https://cdn.jsdelivr.net/npm/@ffmpeg.wasm/core-mt@0.13.2/dist/core.wasm',
+		'./core.wasm'
+	);
+	await downloadFile(
+		'https://cdn.jsdelivr.net/npm/@ffmpeg.wasm/core-mt@0.13.2/dist/core.worker.js',
+		'./core.worker.cjs'
+	);
 	const ffmpeg = await FFmpeg.create({
 		log: true,
 		core: core,
-		// Specify WASM paths for Vercel. These are copied into the function via a post-build script https://github.com/syntaxfm/website/issues/1175
-		...(env.VERCEL && {
-			coreOptions: {
-				wasmPath: './core.wasm',
-				workerPath: './core.worker.cjs'
-			}
-		}),
+		coreOptions: {
+			wasmPath: './core.wasm',
+			workerPath: './core.worker.cjs'
+		},
 		logger: (type, ...message) => {
 			logProgress(message.join(' '));
 		}
 	});
 	console.log('Loading ffmpeg');
-	// await ffmpeg.load();
+
 	// 1. download the show
 	// 1.1 See if the file exists first
-	const { ok } = await fetch(url, { method: 'HEAD' });
+	await fetch(url, { method: 'HEAD' });
 	console.log(`Fetching ${url}`);
-	const fetchBuffer = await fetch(url)
+	const fetch_buffer = await fetch(url)
 		.then((res) => res.arrayBuffer())
 		.then((buf) => Buffer.from(new Uint8Array(buf)));
+
 	// Load it into ffmpeg memory
-	// ffmpeg.FS('writeFile', fileName, await fetchFile(fetchBuffer));
-	ffmpeg.fs.writeFile(fileName, fetchBuffer);
+	ffmpeg.fs.writeFile(file_name, fetch_buffer);
 
-	console.log(`wrote ${fileName} to ffmpeg memory`);
+	console.log(`wrote ${file_name} to ffmpeg memory`);
 	// Write Flaggers to ffmpeg memory
-	for (const [i, flagPath] of flagPaths.entries()) {
+	for (const [i, flag_path] of flag_paths.entries()) {
+		// eslint-disable-next-line @typescript-eslint/naming-convention
 		const __dirname = new URL('.', import.meta.url).pathname;
-		const flagBuffer = await readFile(env.VERCEL ? flagPath : `${__dirname}/${flagPath}`);
-		ffmpeg.fs.writeFile(`flagger-${baseName}-${i}.mp3`, flagBuffer);
-		console.log(`wrote flagger-${baseName}-${i}.mp3 to ffmpeg memory`);
+		const flag_buffer = await readFile(env.VERCEL ? flag_path : `${__dirname}/${flag_path}`);
+		ffmpeg.fs.writeFile(`flagger-${base_name}-${i}.mp3`, flag_buffer);
+		console.log(`wrote flagger-${base_name}-${i}.mp3 to ffmpeg memory`);
 	}
 
 	// Create the Command
 	const command = [
 		'-i',
-		fileName,
-		...flagPaths.map((flagPath, i) => ['-i', `flagger-${baseName}-${i}.mp3`]).flat(),
+		file_name,
+		...flag_paths.map((flagPath, i) => ['-i', `flagger-${base_name}-${i}.mp3`]).flat(),
 		'-filter_complex',
 		'[0:a:0][1:a:0][2:a:0]concat=n=3:v=0:a=1[outa]',
 		'-map',
 		'[outa]',
-		outputFilename
+		output_filename
 	];
 
 	console.log(`Running ffmpeg with command: ${command.join(' ')}`);
 	// Run ffmpeg
 	await ffmpeg.run(...command);
 	// Get the Uint8Array
-	const data = ffmpeg.fs.readFile(outputFilename);
+	const data = ffmpeg.fs.readFile(output_filename);
 	// Convert to buffer
 	const buffer = Buffer.from(data.buffer);
-	// progressBar.stop();
-	console.log(`FFMpeg Merging `);
-	// Write to disk from buffer for DEbugging
-	// await writeFile(`./audio-out/${outputFilename}`, buffer);
 	return buffer;
 }