Imported RTP changes 2 from userconcept.

Author: mo-g

Libraries/ecma-rtp/codec.mjs

@@ -15,3 +15,83 @@
  * along with ecma-rtp.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+
+import alawmulaw from 'alawmulaw';
+import waveResampler from 'wave-resampler';
+
+/**
+ * Superclass to represent an audio Codec.
+ */
+class Codec {
+    constructor ({clientRate = 16000,
+                  clientDepth = 16,
+                  serverRate = 8000,
+                  serverDepth = 8} = {}) {
+        
+        this.clientRate = clientRate;
+        this.clientDepth = clientDepth;
+        this.serverRate = serverRate;
+        this.serverDepth = serverDepth;
+
+        this.samples = []
+    }
+
+    transcode () {
+
+    }
+
+        /**
+     * We need to do some testing to see how resampling options affect STT because
+     * 
+     * Cubic without Low Pass Filter:
+     * generate: 0.283ms
+     * transcode: 0.144ms
+     * resample: 0.442ms
+     * 
+     * Cubic with LPF:
+     * generate: 0.375ms
+     * transcode: 0.123ms
+     * resample: 6.605ms
+     * 
+     * That's on 20ms audio on an i5 9500. Interestingly it scales x2 per x10 duration,
+     * so it's not terrible - but if we can shave time off, so much the better
+     * 
+     * First test - LPF Off doesn't seem to hurt anything!
+     */
+    static resampleAudio (samples) {
+        var wavSamples = alawmulaw.mulaw.decode(samples);
+        var floatSamples=Float32Array.from(Float32Array.from(wavSamples).map(x=>x/0x8000));
+        var newSamples = waveResampler.resample(floatSamples, 8000, 16000, {method: "cubic", LPF: false}); // RETURNS A FLOAT64 NOT AN INT16 READ THE DOCS
+        var samples1616=Int16Array.from(newSamples.map(x => (x>0 ? x*0x7FFF : x*0x8000)));
+        return Buffer.from(samples1616.buffer);
+    }
+
+    set samples (samples) {
+        this.samples.push(this.transcode(samples));
+    }
+
+    get samples () {
+        return this.samples.pop;
+    }
+}
+
+/**
+ * Represents mulaw (as used by Combadges)
+ */
+class muLaw8K extends Codec {
+    constructor ({} = {}) {
+        super({clientRate: 8000,
+            clientDepth: 16,
+            serverRate: 16000,
+            serverDepth: 8});
+    }
+
+    decode (samples, callback) {
+        wavSamples = samples;
+        //var wavSamples = alawmulaw.mulaw.decode(samples);
+        var reSamples = Codec.resample(wavSamples);
+        callback(reSamples);
+    }
+}
+
+export { Codec }

Libraries/ecma-rtp/package.json

@@ -1,5 +1,5 @@
 {
-    "name": "ecma-rtp",
+    "name": "@combadge/ecma-rtp",
     "version": "0.0.5",
     "type": "module",
     "description": "An RTP Library for Spin Doctor, written in ECMAScript",

Libraries/ecma-rtp/rtp.mjs

@@ -22,7 +22,8 @@
  */
 
 
-import alawmulaw from 'alawmulaw'; // For now, move to codec.mjs later.
+import dgram from 'dgram';
+import { Codec } from './codec.mjs';
 
 const Payload = {
     Mulaw8K: 0,
@@ -136,10 +137,9 @@ class RTPHeader {
 
         // var payloadType = Payload[findPayload(payloadType)] // Brutish validation to check supported Payload type. Not sure we're there yet.
         var payloadSize = (packet.slice(lengthOfHeader).length);
-
-        var sequenceNo = parseInt(packet.slice(2,4));
-        var timeStamp = parseInt(packet.slice(4,8));
-        var ssrc = parseInt(packet.slice(8,12));
+        var sequenceNo = packet.slice(2,4).readUInt16BE();
+        var timeStamp = packet.slice(4,8).readUInt16BE();
+        var ssrc = packet.slice(8,12).readUInt16BE();
 
         var header = new RTPHeader(payloadSize, payloadType, ssrc);
         header.forceHeader(sequenceNo, timeStamp);
@@ -217,13 +217,7 @@ class RTPHeader {
  class RTPPacket {
     constructor(header, payload = undefined) {
         this.header = header;
-
-        if (this.header.payloadType == Payload.Mulaw8K) {
-            this._payload = alawmulaw.mulaw.decode(payload); // For now, brute force the transcode, since we know we're working in mulaw.
-        } else {
-            console.log(this.header);
-            throw "Not Mulaw, unsure what to do!";
-        }
+        this._payload = payload;
     }
 
     /**
@@ -275,14 +269,92 @@ class RTPHeader {
 }
 
 /**
- * Temporary class - replace with RTPPacket instances when you work out how.
+ * Should provide a source and sink for RTP Packets. For now, hardcode to the correct values for Combadge audio.
+ * 
+ * Down the line, we should generalise this to allow the library to be reused elsewhere -
+ * (since I wouldn't be writing it if there was another approachable JS RTP library)
  */
-class RTPStream {
-    constructor() {
+class RTPServer {
+    constructor(listenAddress, portNumber, consumer = undefined) {
         this.sampleCount = new Number(160); // At 8 bits/sample, this can be used for both incrementing the timestamp AND counting bytes.
         this.header = new RTPHeader(this.sampleCount);
+        this._consumer = undefined;
+
+        if (portNumber % 2 != 0) {
+            throw `RTP Ports must be even. Odd-numbered ports are reserved for RTCP. Invalid port ${portNumber} passed.`;
+        }
+
+        if (consumer) {
+            this._consumer = consumer;
+        }
+
+        this.udpServer = dgram.createSocket('udp4');
+        this.udpServer.bind(portNumber, listenAddress)
+        this.udpServer.on('listening', () => {
+            const address = this.udpServer.address();
+            console.log(`RTP Server spawned at ${address.address}:${address.port}`);
+        })
+        this.udpServer.on('message', (message, clientInfo) => {  
+            var packet = RTPPacket.from(message);
+            this.receivePacket(packet);
+        })
+
+        /*
+        this.transcoding = true;
+        var floatSamples=Float32Array.from(Float32Array.from(this.recSamples).map(x=>x/0x8000));
+        var newSamples = waveResampler.resample(floatSamples, 8000, 16000); // RETURNS A FLOAT64 NOT AN INT16 READ THE DOCS
+        var samples1616=Int16Array.from(newSamples.map(x => (x>0 ? x*0x7FFF : x*0x8000)));
+        var wav16buffer = new Buffer.from(samples1616.buffer);
+        console.log("Result:", model.stt(wav16buffer));
+        this.transcoding = false;
+        */
     }
 
+    /**
+     * Function to pass in a function to receive audio from the server.
+     */
+    set consumer (consumer) {
+        this._consumer = consumer;
+    }
+
+    get consumer () {
+        return this._consumer;
+    }
+
+    /** Do something with the packet, then forward it to the Consumer if set, or cache it if not. */
+    receivePacket (packet) {
+        var samples = Codec.resampleAudio(packet.payload);
+        this.consumer.receiveSamples(samples);
+    }
+
+    /**
+     * Queue audio to be sent to a remote UDP target. 
+     */
+    queueAudio (audio) {
+
+    }
+
+    /**
+     * Register a recipient to receive audio that's passed to queueAudio - basically, a remote target for an RTP stream.
+     * Address should be one of my fancy IP address objects, which I need to move outside the cccp protocol I think.
+     * 
+     * @param {*} address 
+     */
+    registerRemote (address) {
+
+    }
+
+
+
+    /**
+     * Send audio to a remote device. We'll be explicit about target, rather than just using send() from the responder
+     * because otherwise I can already see an exploit where a snooper sends 0-length packets to ensure they're always the most recent address.
+     * socket.send(msg[, offset, length][, port][, address][, callback])
+     */
+    sendPacket (packet, address, port) {
+        packetData = packet.toBuffer()
+        this.udpServer.send(packetData, 0, packetData.length, port, address)
+    }
 
     /**
      * Return a completed media packet without transcoding.
@@ -308,25 +380,4 @@ class RTPStream {
     }
 }
 
-class MediaQueue {
-    constructor() {
-        this.inQueue = new Array();
-        this.outQueue = new Array();
-    }
-
-    /**
-     * Add a packet to the queue for RTP decoding
-     */
-    addReceivedPacket (packet, callback) {
-        return null;
-    }
-
-    /**
-     * Add a media stream for RTP encoding
-     */
-    addTransmitMedia (samples, callback) {
-        return null;
-    }
-}
-
-export { Payload, RTPHeader, RTPStream, RTPPacket, MediaQueue };
+export { Payload, RTPHeader, RTPPacket, RTPServer };

Libraries/robin-agent/non_verbal.mjs

@@ -0,0 +1,92 @@
+/**
+ * Robin-agent - an interactive voice agent for Spin Doctor
+ * Copyright (C) 2021-2022 The Combadge Project by mo-g
+ *
+ * Robin-agent is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * Robin-agent is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Robin-agent.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * tone({
+  freq: Tones["E♭6"],
+  lengthInSecs: 0.2,
+  volume: tone.MAX_16/2,
+  rate: 16000,
+  shape: 'sine'
+})
+ */
+
+
+import { Tones, FixedPitch, Silence} from '@combadge/ecma-tonegenerator'
+
+
+/**
+ * Superclass for all nonverbal sound generators. By default, we should generate audio in the same codec and profile as Robin uses internally.
+ * 
+ * 
+ */
+class Sound {
+    constructor (notes) {
+        this.tones = notes.map(function (note) {
+            var tone = new FixedPitch({frequency:Tones[note],
+                            bitDepth: 16,
+                            sampleRate: 16000});
+            var silence = new Silence({bitDepth: 16,
+                                       sampleRate: 16000});
+            
+            return [tone.approximate({duration:500}), silence.accurate({duration:500})];
+        });
+
+    }
+}
+
+/**
+ * Click, B5, Click, B5, Click, D6
+ */
+class ListeningChirp extends Sound {
+    constructor () {
+        notes = ["B5", "B5", "D6"];
+        super(notes);
+    }
+
+    get output () {
+        output = [];
+        this.tones.forEach(function (tone) {
+            tone.forEach(function (sample) {
+                output.push(sample);
+            })
+        });
+        return new Int16Array(output);
+    }
+}
+
+/**
+ * 
+ */
+class OfflineChirp extends Sound {
+    constructor () {
+        notes = ["B5", "B5"];
+        super(notes);
+    }
+}
+
+/**
+ * B5, D6, B5
+ */
+class BoatswainCall extends Sound {
+    constructor () {
+        notes = ["B5", "D6", "B5"];
+        super(notes);
+    }
+}
+
+export { ListeningChirp, OfflineChirp, BotswainWhistle };

Libraries/robin-agent/package.json

@@ -1,5 +1,5 @@
 {
-    "name": "robin-agent",
+    "name": "@combadge/robin-agent",
     "version": "0.0.1",
     "type": "module",
     "description": "An interactive voice agent for Spin Doctor.",