add velocity editor and asset controls to piano roll (#11288)

* measure indicator

* velocity editor

* rename file and save more state

* coat of paint

* gallery myassets

* asset name plus done button

Author: riknoll

localtypings/pxtmusic.d.ts

@@ -45,6 +45,7 @@ declare namespace pxt.assets.music {
         notes: Note[];
         startTick: number;
         endTick: number;
+        velocity?: number;
     }
 
     export interface Note {

pxtlib/music.ts

@@ -32,6 +32,7 @@ namespace pxt.assets.music {
      *     5 measures
      *     6 number of tracks
      *     ...tracks
+     *     ...track velocities
      *
      * track(6 + instrument length + note length bytes)
      *     0 id
@@ -83,13 +84,24 @@ namespace pxt.assets.music {
      *          0 = normal
      *          1 = flat
      *          2 = sharp
+     *
+     * track velocity
+     *     0 track id
+     *     1...velocities
+     *
+     * velocty
+     *     1 byte
      */
 
     function encodeSong(song: Song) {
         const encodedTracks = song.tracks
             .filter((track) => track.notes.length > 0)
             .map(encodeTrack);
-        const trackLength = encodedTracks.reduce((d, c) => c.length + d, 0);
+        const encodedTrackVelocities = song.tracks
+            .map(encodeTrackVelocity)
+            .filter((v): v is Uint8Array => !!v);
+
+        const trackLength = (encodedTracks.concat(encodedTrackVelocities)).reduce((d, c) => c.length + d, 0);
 
         const out = new Uint8Array(7 + trackLength);
         out[0] = 0; // encoding version
@@ -105,6 +117,10 @@ namespace pxt.assets.music {
             current += track.length;
         }
 
+        for (const trackVelocity of encodedTrackVelocities) {
+            out.set(trackVelocity, current);
+            current += trackVelocity.length;
+        }
         return out;
     }
 
@@ -181,6 +197,17 @@ namespace pxt.assets.music {
         return encodeMelodicTrack(track);
     }
 
+    function encodeTrackVelocity(track: Track) {
+        if (!track.notes.some(note => note.velocity !== undefined && note.velocity < 128)) return undefined;
+
+        const out = new Uint8Array(1 + track.notes.length);
+        out[0] = track.id;
+        for (let i = 0; i < track.notes.length; i++) {
+            out[1 + i] = track.notes[i].velocity || 0;
+        }
+        return out;
+    }
+
     function encodeMelodicTrack(track: Track) {
         const encodedInstrument = encodeInstrument(track.instrument);
         const encodedNotes = track.notes.map(note => encodeNoteEvent(note, track.instrument.octave, false));
@@ -248,14 +275,19 @@ namespace pxt.assets.music {
             tracks: []
         };
 
+        const numTracks = buf[6];
         let current = 7;
 
-        while (current < buf.length) {
+        for (let i = 0; i < numTracks; i++) {
             const [track, pointer] = decodeTrack(buf, current);
             current = pointer;
             res.tracks.push(track);
         }
 
+        while (current < buf.length) {
+            current = decodeTrackVelocity(buf, res.tracks, current);
+        }
+
         return res;
     }
 
@@ -296,6 +328,16 @@ namespace pxt.assets.music {
         return decodeMelodicTrack(buf, offset);
     }
 
+    function decodeTrackVelocity(buf: Uint8Array, tracks: Track[], offset: number): number {
+        const trackId = buf[offset];
+        const track = tracks.find(t => t.id === trackId);
+        if (!track) throw new Error(`Track with id ${trackId} not found`);
+        for (let i = 0; i < track.notes.length; i++) {
+            track.notes[i].velocity = buf[offset + i + 1];
+        }
+        return offset + track.notes.length + 1;
+    }
+
     function decodeDrumInstrument(buf: Uint8Array, offset: number): DrumInstrument {
         const res: DrumInstrument = {
             startFrequency: get16BitNumber(buf, offset + 1),

pxtsim/sound/sequencer.ts

@@ -160,17 +160,17 @@ namespace pxsim.music {
             this.spatialAudioPlayer = player;
         }
 
-        protected getMelodicTrackVolume(trackIndex: number) {
+        protected getMelodicTrackVolume(trackIndex: number, velocity: number) {
             let trackVolume = 1024;
             if (trackIndex < this.trackVolumes.length) {
                 trackVolume = this.trackVolumes[trackIndex];
             }
 
-            return this.globalVolume * (trackVolume / 1024);
+            return this.globalVolume * (trackVolume / 1024) * (velocity / 128);
         }
 
-        protected getDrumTrackVolume(trackIndex: number, drumIndex: number) {
-            const trackVolume = this.getMelodicTrackVolume(trackIndex);
+        protected getDrumTrackVolume(trackIndex: number, drumIndex: number, velocity: number) {
+            const trackVolume = this.getMelodicTrackVolume(trackIndex, velocity);
             let drumVolume = 1024;
 
             if (trackIndex < this.drumTrackVolumes.length && drumIndex < this.drumTrackVolumes[trackIndex].length) {
@@ -200,12 +200,14 @@ namespace pxsim.music {
                 for (const noteEvent of track.notes) {
                     if (noteEvent.startTick === this._currentTick) {
                         for (const note of noteEvent.notes) {
+                            const velocity = noteEvent.velocity ?? 128;
+
                             if (this.spatialAudioPlayer) {
                                 if (track.drums) {
                                     playDrumAtSpatialAudioPlayerAsync(
                                         this.spatialAudioPlayer,
                                         track.drums[note.note],
-                                        this.getDrumTrackVolume(i, note.note)
+                                        this.getDrumTrackVolume(i, note.note, velocity)
                                     );
                                 }
                                 else {
@@ -214,16 +216,16 @@ namespace pxsim.music {
                                         note.note,
                                         track.instrument,
                                         tickToMs(this.currentlyPlaying.beatsPerMinute, this.currentlyPlaying.ticksPerBeat, noteEvent.endTick - noteEvent.startTick),
-                                        this.getMelodicTrackVolume(i)
+                                        this.getMelodicTrackVolume(i, velocity)
                                     );
                                 }
                             }
                             else {
                                 if (track.drums) {
-                                    playDrumAsync(track.drums[note.note], () => currentToken.cancelled, this.getDrumTrackVolume(i, note.note));
+                                    playDrumAsync(track.drums[note.note], () => currentToken.cancelled, this.getDrumTrackVolume(i, note.note, velocity));
                                 }
                                 else {
-                                    playNoteAsync(note.note, track.instrument, tickToMs(this.currentlyPlaying.beatsPerMinute, this.currentlyPlaying.ticksPerBeat, noteEvent.endTick - noteEvent.startTick), () => currentToken.cancelled, this.getMelodicTrackVolume(i));
+                                    playNoteAsync(note.note, track.instrument, tickToMs(this.currentlyPlaying.beatsPerMinute, this.currentlyPlaying.ticksPerBeat, noteEvent.endTick - noteEvent.startTick), () => currentToken.cancelled, this.getMelodicTrackVolume(i, velocity));
                                 }
                             }
                         }

pxtsim/sound/song.ts

@@ -272,14 +272,19 @@ namespace pxsim.music {
             tracks: []
         };
 
+        const numTracks = buf[6];
         let current = 7;
 
-        while (current < buf.length) {
+        for (let i = 0; i < numTracks; i++) {
             const [track, pointer] = decodeTrack(buf, current);
             current = pointer;
             res.tracks.push(track);
         }
 
+        while (current < buf.length) {
+            current = decodeTrackVelocity(buf, res.tracks, current);
+        }
+
         return res;
     }
 
@@ -320,6 +325,17 @@ namespace pxsim.music {
         return decodeMelodicTrack(buf, offset);
     }
 
+    function decodeTrackVelocity(buf: Uint8Array, tracks: pxt.assets.music.Track[], offset: number): number {
+        const trackId = buf[offset];
+        const track = tracks.find(t => t.id === trackId);
+        if (!track) return buf.length;
+
+        for (let i = 0; i < track.notes.length; i++) {
+            track.notes[i].velocity = buf[offset + i + 1];
+        }
+        return offset + track.notes.length + 1;
+    }
+
     function decodeDrumInstrument(buf: Uint8Array, offset: number): pxt.assets.music.DrumInstrument {
         const res: pxt.assets.music.DrumInstrument = {
             startFrequency: get16BitNumber(buf, offset + 1),