diff --git a/.gitignore b/.gitignore index 0527f2d50..6a1d299c8 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,7 @@ test-results start_script/target bin -db-data \ No newline at end of file +db-data + +core.wasm +core.worker.cjs \ No newline at end of file diff --git a/package.json b/package.json index ef7996638..cba3094fc 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,7 @@ "dev": "node ./scripts/preheat.js", "preheat": "node ./scripts/preheat.js --env-only", "vite-dev": "vite dev", - "copy-files": "zx ./why_do_i_need_this.mjs", - "build": "run-s --print-name build:svelte copy-files", - "build:svelte": "vite build", + "build": "vite build", "preview": "vite preview", "test": "svelte-kit sync && playwright test", "test:ui": "svelte-kit sync && playwright test --ui", diff --git a/src/server/transcripts/flagger.ts b/src/server/transcripts/flagger.ts index 2cb420ffc..28551128e 100644 --- a/src/server/transcripts/flagger.ts +++ b/src/server/transcripts/flagger.ts @@ -1,12 +1,15 @@ -// import { createFFmpeg, fetchFile, type ProgressCallback } from '@ffmpeg.wasm/main'; +import { createWriteStream, existsSync } from 'fs'; +import { readFile } from 'fs/promises'; +import { Readable } from 'stream'; +import { finished } from 'stream/promises'; import { FFmpeg } from '@ffmpeg.wasm/main'; import type { Show } from '@prisma/client'; -import { readFile, readdir } from 'fs/promises'; -import { logProgress } from './logProgress'; import core from '@ffmpeg.wasm/core-mt'; -const flagPaths = ['./audio/wes-flagger.mp3', './audio/scott-flagger.mp3']; -import wasmPathAb from '@ffmpeg.wasm/core-mt/dist/core.wasm?url'; import { env } from '$env/dynamic/private'; +import { logProgress } from './logProgress'; + +const flag_paths = ['./audio/wes-flagger.mp3', './audio/scott-flagger.mp3']; + export type ProgressEvent = { duration?: number; ratio?: number; @@ -14,6 +17,17 @@ export type ProgressEvent = { percentage: number; }; +async function downloadFile(url: string, path: string) { + if (existsSync(path)) return; + + const stream = createWriteStream(path); + const response = await fetch(url); + if (response.body) { + // @ts-expect-error body is readable + await finished(Readable.fromWeb(response.body).pipe(stream)); + } +} + /** * Concatenates a show with flagger audio to help with diatirization * @returns {Promise} - The concatenated show @@ -23,71 +37,73 @@ export async function addFlaggerAudio(show: Show): Promise { console.log('ADDING FLAGGER AUDIO'); const url = new URL(show.url); // Get the filename - const fileName = `${show.number}.mp3`; + const file_name = `${show.number}.mp3`; // Get the base name - const [baseName, extension] = fileName.split('.'); + const [base_name, extension] = file_name.split('.'); // create the output filename - const outputFilename = `${show.number}-flagged.${extension}`; + const output_filename = `${show.number}-flagged.${extension}`; console.log(`Downloading #${show.number} - ${show.title}`); console.log('Creating ffmpeg instance'); + await downloadFile( + 'https://cdn.jsdelivr.net/npm/@ffmpeg.wasm/core-mt@0.13.2/dist/core.wasm', + './core.wasm' + ); + await downloadFile( + 'https://cdn.jsdelivr.net/npm/@ffmpeg.wasm/core-mt@0.13.2/dist/core.worker.js', + './core.worker.cjs' + ); const ffmpeg = await FFmpeg.create({ log: true, core: core, - // Specify WASM paths for Vercel. These are copied into the function via a post-build script https://github.com/syntaxfm/website/issues/1175 - ...(env.VERCEL && { - coreOptions: { - wasmPath: './core.wasm', - workerPath: './core.worker.cjs' - } - }), + coreOptions: { + wasmPath: './core.wasm', + workerPath: './core.worker.cjs' + }, logger: (type, ...message) => { logProgress(message.join(' ')); } }); console.log('Loading ffmpeg'); - // await ffmpeg.load(); + // 1. download the show // 1.1 See if the file exists first - const { ok } = await fetch(url, { method: 'HEAD' }); + await fetch(url, { method: 'HEAD' }); console.log(`Fetching ${url}`); - const fetchBuffer = await fetch(url) + const fetch_buffer = await fetch(url) .then((res) => res.arrayBuffer()) .then((buf) => Buffer.from(new Uint8Array(buf))); + // Load it into ffmpeg memory - // ffmpeg.FS('writeFile', fileName, await fetchFile(fetchBuffer)); - ffmpeg.fs.writeFile(fileName, fetchBuffer); + ffmpeg.fs.writeFile(file_name, fetch_buffer); - console.log(`wrote ${fileName} to ffmpeg memory`); + console.log(`wrote ${file_name} to ffmpeg memory`); // Write Flaggers to ffmpeg memory - for (const [i, flagPath] of flagPaths.entries()) { + for (const [i, flag_path] of flag_paths.entries()) { + // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = new URL('.', import.meta.url).pathname; - const flagBuffer = await readFile(env.VERCEL ? flagPath : `${__dirname}/${flagPath}`); - ffmpeg.fs.writeFile(`flagger-${baseName}-${i}.mp3`, flagBuffer); - console.log(`wrote flagger-${baseName}-${i}.mp3 to ffmpeg memory`); + const flag_buffer = await readFile(env.VERCEL ? flag_path : `${__dirname}/${flag_path}`); + ffmpeg.fs.writeFile(`flagger-${base_name}-${i}.mp3`, flag_buffer); + console.log(`wrote flagger-${base_name}-${i}.mp3 to ffmpeg memory`); } // Create the Command const command = [ '-i', - fileName, - ...flagPaths.map((flagPath, i) => ['-i', `flagger-${baseName}-${i}.mp3`]).flat(), + file_name, + ...flag_paths.map((flagPath, i) => ['-i', `flagger-${base_name}-${i}.mp3`]).flat(), '-filter_complex', '[0:a:0][1:a:0][2:a:0]concat=n=3:v=0:a=1[outa]', '-map', '[outa]', - outputFilename + output_filename ]; console.log(`Running ffmpeg with command: ${command.join(' ')}`); // Run ffmpeg await ffmpeg.run(...command); // Get the Uint8Array - const data = ffmpeg.fs.readFile(outputFilename); + const data = ffmpeg.fs.readFile(output_filename); // Convert to buffer const buffer = Buffer.from(data.buffer); - // progressBar.stop(); - console.log(`FFMpeg Merging `); - // Write to disk from buffer for DEbugging - // await writeFile(`./audio-out/${outputFilename}`, buffer); return buffer; } diff --git a/why_do_i_need_this.mjs b/why_do_i_need_this.mjs deleted file mode 100644 index b89b2e1d2..000000000 --- a/why_do_i_need_this.mjs +++ /dev/null @@ -1,17 +0,0 @@ -#!zx -const LOCAL_BUILD_PATH = '.svelte-kit/output/server/'; -const VERCEL_BUILD_BASE_PATH = '.vercel/output/functions'; -const vercelFuncDirs = (await $`ls -1d ${VERCEL_BUILD_BASE_PATH}/*.func`).stdout - .split('\n') - .map((v) => v.trim()) - .filter((v) => v !== ''); - -const outputDirs = [...vercelFuncDirs, LOCAL_BUILD_PATH]; - -for (const outputDir of outputDirs) { - // We have to explicitly copy the WASM files to the output directory of each serverless function because Sveltekit + Vercel have no way of knowing that they need to be copied via static analysis. - console.log(`Copying WASM files to ${outputDir}.`); - await $`cp node_modules/@ffmpeg.wasm/core-mt/dist/core.wasm ${outputDir}`; - await $`cp node_modules/@ffmpeg.wasm/core-mt/dist/core.worker.js ${outputDir}/core.worker.cjs`; - await $`cp -r ./src/server/transcripts/audio ${outputDir}`; -}