Skip to content

audio #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vjeux wants to merge 1 commit into
base: main
Choose a base branch
from
Open

audio #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions audio_flush.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
<script>
async function f() {
const audioDecoder = new AudioDecoder({
async output(audioData) {
console.log(arguments);
console.log('decoded', audioData.timestamp, audioData);
audioEncoder.encode(audioData, { keyFrame: true });
audioData.close();
},
error(error) {
console.error(error);
}
});

const configuration = {
codec: 'mp4a.40.02',
numberOfChannels: 1,
sampleRate: 48000,
}

audioDecoder.configure(configuration);

const audioEncoder = new AudioEncoder({
output(chunk, metadata) {
console.log('encoded', chunk.timestamp, chunk);
},
error(error) {
console.error(error);
}
});

audioEncoder.configure(configuration);

audioDecoder.decode(new EncodedAudioChunk({
duration: 21333.333333333332,
timestamp: 0,
type: "key",
data: new Uint8Array([0, 248, 23, 173, 52, 166, 85, 134, 2, 33, 96, 161, 152, 72, 103, 250, 248, 214, 166, 43, 139, 187, 67, 89, 139, 141, 10, 109, 168, 107, 172, 176, 160, 19, 133, 196, 201, 240, 10, 86, 109, 180, 181, 177, 186, 49, 223, 251, 71, 109, 151, 64, 180, 246, 210, 90, 190, 91, 41, 34, 211, 50, 118, 166, 219, 251, 110, 180, 221, 229, 76, 83, 34, 93, 129, 215, 177, 25, 118, 83, 109, 197, 42, 145, 117, 102, 101, 192, 136, 156, 232, 109, 29, 14, 97, 255, 119, 155, 114, 86, 170, 154, 84, 134, 128, 123, 113, 251, 182, 174, 234, 109, 139, 182, 232, 242, 223, 47, 130, 97, 182, 160, 107, 109, 175, 56, 196, 8, 185, 44, 146, 182, 113, 45, 221, 123, 50, 125, 27, 23, 192, 172, 59, 146, 124, 121, 69, 159, 249, 124, 210, 113, 245, 231, 54, 65, 60, 106, 238, 153, 169, 209, 162, 97, 182, 166, 153, 54, 24, 156, 62, 235, 170, 28, 82, 185, 64, 67, 13, 117, 34, 152, 95, 138, 135, 36, 29, 73, 222, 222, 235, 248, 89, 191, 103, 232, 219, 78, 14, 21, 0, 234, 112, 148, 17, 43, 71, 79, 242, 52, 225, 42, 230, 155, 63, 9, 164, 225, 196, 220, 64, 65, 146, 163, 69, 241, 117, 77, 109, 202, 36, 114, 148, 105, 44, 89, 43, 142, 54, 82, 90, 237, 26, 1, 130, 2, 239, 67, 232, 218, 120, 127, 5, 115, 238, 94, 103, 89, 234, 164, 112, 4, 185, 72, 218, 226, 106, 169, 13, 3, 69, 128, 183, 222, 224, 99, 37, 247, 185, 192, 201, 83, 119, 192, 91, 62, 197, 110, 164, 184, 121, 175, 254, 6, 172, 247, 185, 222, 169, 19, 127, 135, 72, 96, 110, 150, 2, 251, 72, 167, 224, 172, 175, 27, 30, 188, 180, 229, 78, 200, 231, 141, 157, 128, 227, 114, 53, 197, 56, 54, 250, 151, 224
]),
}));
await audioDecoder.flush();
await audioEncoder.flush();
}
f();
</script>
190 changes: 146 additions & 44 deletions mp4box.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,31 @@
async function reencode(file) {
const startNow = performance.now();

let decoder;
let encoder;
let videoDecoder;
let videoEncoder;

let track = null;
let decodedFrameIndex = 0;
let encodedFrameIndex = 0;
let nextKeyFrameTimestamp = 0;
let sampleDurations = [];
let trackID = null;
let videoTrack = null;
let audioTrack = null;
let decodedVideoFrameIndex = 0;
let encodedVideoFrameIndex = 0;
let videoFrameCount = 0;
let audioFrameCount = 0;
let decodedAudioFrameIndex = 0;
let encodedAudioFrameIndex = 0;
let nextVideoKeyFrameTimestamp = 0;
let nextAudioKeyFrameTimestamp = 0;
let sampleVideoDurations = [];
let videoTrackID = null;
let audioTrackID = null;

const mp4boxOutputFile = MP4Box.createFile();
const mp4boxInputFile = MP4Box.createFile();
mp4boxInputFile.onError = error => console.error(error);
mp4boxInputFile.onReady = async (info) => {
track = info.videoTracks[0];
videoTrack = info.videoTracks[0];
audioTrack = info.audioTracks[0];

decoder = new VideoDecoder({
videoDecoder = new VideoDecoder({
async output(inputFrame) {
const bitmap = await createImageBitmap(inputFrame);

Expand All @@ -30,15 +38,15 @@

const keyFrameEveryHowManySeconds = 2;
let keyFrame = false;
if (inputFrame.timestamp >= nextKeyFrameTimestamp) {
if (inputFrame.timestamp >= nextVideoKeyFrameTimestamp) {
keyFrame = true;
nextKeyFrameTimestamp = inputFrame.timestamp + keyFrameEveryHowManySeconds * 1e6;
nextVideoKeyFrameTimestamp = inputFrame.timestamp + keyFrameEveryHowManySeconds * 1e6;
}
encoder.encode(outputFrame, { keyFrame });
videoEncoder.encode(outputFrame, { keyFrame });
inputFrame.close();
outputFrame.close();

decodedFrameIndex++;
decodedVideoFrameIndex++;
displayProgress();
},
error(error) {
Expand All @@ -47,7 +55,7 @@
});

let description;
const trak = mp4boxInputFile.getTrackById(track.id);
const trak = mp4boxInputFile.getTrackById(videoTrack.id);
for (const entry of trak.mdia.minf.stbl.stsd.entries) {
if (entry.avcC || entry.hvcC) {
const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
Expand All @@ -61,91 +69,185 @@
}
}

decoder.configure({
codec: track.codec,
codedWidth: track.track_width,
codedHeight: track.track_height,
videoDecoder.configure({
codec: videoTrack.codec,
codedWidth: videoTrack.track_width,
codedHeight: videoTrack.track_height,
hardwareAcceleration: 'prefer-hardware',
description,
});

encoder = new VideoEncoder({
videoEncoder = new VideoEncoder({
output(chunk, metadata) {
let uint8 = new Uint8Array(chunk.byteLength);
chunk.copyTo(uint8);

const timescale = 90000;
if (trackID === null) {
if (videoTrackID === null) {
const description = metadata.decoderConfig.description;
trackID = mp4boxOutputFile.addTrack({
width: track.track_width,
height: track.track_height,
videoTrackID = mp4boxOutputFile.addTrack({
width: videoTrack.track_width,
height: videoTrack.track_height,
timescale,
avcDecoderConfigRecord: description,
});
}

const sampleDuration = sampleDurations.shift() / (1_000_000 / timescale);
mp4boxOutputFile.addSample(trackID, uint8, {
const sampleDuration = sampleVideoDurations.shift() / (1_000_000 / timescale);
mp4boxOutputFile.addSample(videoTrackID, uint8, {
duration: sampleDuration,
is_sync: chunk.type === 'key',
});

encodedFrameIndex++;
encodedVideoFrameIndex++;
displayProgress();
},
error(error) {
console.error(error);
}
});

encoder.configure({
videoEncoder.configure({
codec: 'avc1.4d0034',
width: track.track_width,
height: track.track_height,
width: videoTrack.track_width,
height: videoTrack.track_height,
hardwareAcceleration: 'prefer-hardware',
bitrate: 20_000_000,
});

mp4boxInputFile.setExtractionOptions(track.id, null, {nbSamples: Infinity});
mp4boxInputFile.setExtractionOptions(videoTrack.id, null, {nbSamples: Infinity});

if (audioTrack) {
audioDecoder = new AudioDecoder({
async output(audioData) {
audioEncoder.encode(audioData, { keyFrame: true });
audioData.close();

decodedAudioFrameIndex++;
displayProgress();
},
error(error) {
console.error(error);
}
});

const audioCodec = 'mp4a.40.02';

audioDecoder.configure({
codec: audioCodec, //audioTrack.codec,
numberOfChannels: audioTrack.audio.channel_count,
sampleRate: audioTrack.audio.sample_rate,
});

audioEncoder = new AudioEncoder({
output(chunk, metadata) {
let uint8 = new Uint8Array(chunk.byteLength);
chunk.copyTo(uint8);

const timescale = 90000;
if (audioTrackID === null) {
// audioTrackID = mp4boxOutputFile.addTrack({
// type: 'mp4a',
// timescale,
// });
}

console.log(encodedAudioFrameIndex, uint8);

// mp4boxOutputFile.addSample(audioTrackID, uint8, {
// duration: chunk.duration / (1_000_000 / timescale),
// is_sync: chunk.type === 'key',
// });

encodedAudioFrameIndex++;
displayProgress();
},
error(error) {
console.error(error);
}
});


audioEncoder.configure({
codec: audioCodec,
numberOfChannels: audioTrack.audio.channel_count,
sampleRate: audioTrack.audio.sample_rate,
});

mp4boxInputFile.setExtractionOptions(audioTrack.id, null, {nbSamples: Infinity});
}

mp4boxInputFile.start();
};

function displayProgress() {
progress.innerText =
"Decoding frame " + decodedFrameIndex + " (" + Math.round(100 * decodedFrameIndex / track.nb_samples) + "%)\n" +
"Encoding frame " + encodedFrameIndex + " (" + Math.round(100 * encodedFrameIndex / track.nb_samples) + "%)\n";
"Decoding video frame " + decodedVideoFrameIndex + " (" + Math.round(100 * decodedVideoFrameIndex / videoFrameCount) + "%)\n" +
"Encoding video frame " + encodedVideoFrameIndex + " (" + Math.round(100 * encodedVideoFrameIndex / videoFrameCount) + "%)\n";
if (audioTrack) {
progress.innerText +=
"Decoding audio frame " + decodedAudioFrameIndex + " (" + Math.round(100 * decodedAudioFrameIndex / audioFrameCount) + "%)\n" +
"Encoding audio frame " + encodedAudioFrameIndex + " (" + Math.round(100 * encodedAudioFrameIndex / audioFrameCount) + "%)\n";
}
}

mp4boxInputFile.onSamples = async (track_id, ref, samples) => {
mp4boxInputFile.onSamples = function(track_id, ref, samples) {
for (const sample of samples) {
sampleDurations.push(sample.duration * 1_000_000 / sample.timescale);
decoder.decode(new EncodedVideoChunk({
const chunk = {
type: sample.is_sync ? "key" : "delta",
timestamp: sample.cts * 1_000_000 / sample.timescale,
duration: sample.duration * 1_000_000 / sample.timescale,
data: sample.data
}));
};
if (track_id === videoTrack.id) {
videoFrameCount++;
sampleVideoDurations.push(sample.duration * 1_000_000 / sample.timescale);
videoDecoder.decode(new EncodedVideoChunk(chunk));
} else {
audioFrameCount++;
audioDecoder.decode(new EncodedAudioChunk(chunk));
}
}
if (track_id !== videoTrack.id) {
console.log('Number of audio frames:', audioFrameCount);
}
await decoder.flush();
await encoder.flush();
encoder.close();
decoder.close();
}

async function onComplete() {
mp4boxOutputFile.save("mp4box.mp4");

const seconds = (performance.now() - startNow) / 1000;
progress.innerText =
"Re-encoded " + encodedFrameIndex + " frames in " + (Math.round(seconds * 100) / 100) + "s at " +
Math.round(encodedFrameIndex / seconds) + " fps";
"Re-encoded " + encodedVideoFrameIndex + " video frames in " + (Math.round(seconds * 100) / 100) + "s at " +
Math.round(encodedVideoFrameIndex / seconds) + " fps\n";
if (audioTrack) {
progress.innerText +=
"Re-encoded audio " + encodedAudioFrameIndex + " video frames in " + (Math.round(seconds * 100) / 100) + "s at " +
Math.round(encodedAudioFrameIndex / seconds) + " fps";
}
};

var reader = new FileReader();
reader.onload = function() {
reader.onload = async function() {
this.result.fileStart = 0;
mp4boxInputFile.appendBuffer(this.result);
mp4boxInputFile.flush();

if (audioTrack) {
await audioDecoder.flush();
audioDecoder.close();
}
await videoDecoder.flush();
videoDecoder.close();

if (audioTrack) {
await audioEncoder.flush();
audioEncoder.close();
}
await videoEncoder.flush();
videoEncoder.close();

onComplete();
};
reader.readAsArrayBuffer(file);
}
Expand Down