feat: audio panning

Tested with opus and fluidsynth
Closes #2585

Author: Desdaemon

src/audio.cpp

@@ -33,7 +33,7 @@ AudioInterface& Audio() {
 	return default_;
 }
 
-void EmptyAudio::BGM_Play(Filesystem_Stream::InputStream, int, int, int) {
+void EmptyAudio::BGM_Play(Filesystem_Stream::InputStream, int, int, int, int) {
 	bgm_starttick = Player::GetFrames();
 	playing = true;
 }

src/audio.h

@@ -61,8 +61,9 @@ struct AudioInterface {
 	 * @param volume volume.
 	 * @param pitch pitch.
 	 * @param fadein fadein.
+	 * @param balance balance (0 - 100)
 	 */
-	virtual void BGM_Play(Filesystem_Stream::InputStream stream, int volume, int pitch, int fadein) = 0;
+	virtual void BGM_Play(Filesystem_Stream::InputStream stream, int volume, int pitch, int fadein, int balance) = 0;
 
 	/**
 	 * Stops the currently playing background music.
@@ -129,8 +130,9 @@ struct AudioInterface {
 	 * @param se se to play.
 	 * @param volume volume.
 	 * @param pitch pitch.
+	 * @param balance balance (0 - 100)
 	 */
-	virtual void SE_Play(std::unique_ptr<AudioSeCache> se, int volume, int pitch) = 0;
+	virtual void SE_Play(std::unique_ptr<AudioSeCache> se, int volume, int pitch, int balance) = 0;
 
 	/**
 	 * Stops the currently playing sound effect.
@@ -162,7 +164,7 @@ struct AudioInterface {
 struct EmptyAudio : public AudioInterface {
 public:
 	explicit EmptyAudio(const Game_ConfigAudio& cfg);
-	void BGM_Play(Filesystem_Stream::InputStream, int, int, int) override;
+	void BGM_Play(Filesystem_Stream::InputStream, int, int, int, int) override;
 	void BGM_Pause() override {}
 	void BGM_Resume() override {}
 	void BGM_Stop() override;
@@ -173,7 +175,7 @@ struct EmptyAudio : public AudioInterface {
 	void BGM_Volume(int) override {}
 	void BGM_Pitch(int) override {};
 	std::string BGM_GetType() const override { return {}; };
-	void SE_Play(std::unique_ptr<AudioSeCache>, int, int) override {}
+	void SE_Play(std::unique_ptr<AudioSeCache>, int, int, int) override {}
 	void SE_Stop() override {}
 	void Update() override {}
 	void vGetConfig(Game_ConfigAudio& cfg) const override;

src/audio_decoder.cpp

@@ -175,16 +175,17 @@ void AudioDecoder::Update(std::chrono::microseconds delta) {
 
 	volume += static_cast<float>(std::chrono::duration_cast<std::chrono::microseconds>(delta).count()) * delta_volume_step;
 	volume = Utils::Clamp(static_cast<float>(volume), 0.0f, 100.0f);
-	log_volume = AdjustVolume(volume);
+	SetLogVolume();
 }
 
-int AudioDecoder::GetVolume() const {
-	return static_cast<int>(log_volume);
+std::pair<int, int> AudioDecoder::GetVolume() const {
+	auto& [left, right] = log_volume;
+	return std::pair(static_cast<int>(left), static_cast<int>(right));
 }
 
 void AudioDecoder::SetVolume(int new_volume) {
 	volume = Utils::Clamp(static_cast<float>(new_volume), 0.0f, 100.0f);
-	log_volume = AdjustVolume(volume);
+	SetLogVolume();
 }
 
 void AudioDecoder::SetFade(int end, std::chrono::milliseconds duration) {
@@ -200,6 +201,11 @@ void AudioDecoder::SetFade(int end, std::chrono::milliseconds duration) {
 	delta_volume_step = (static_cast<float>(fade_volume_end) - volume) / fade_time.count();
 }
 
+void AudioDecoder::SetBalance(int balance) {
+	AudioDecoderBase::SetBalance(balance);
+	SetLogVolume();
+}
+
 int AudioDecoder::GetSamplesizeForFormat(AudioDecoderBase::Format format) {
 	switch (format) {
 		case AudioDecoderBase::Format::S8:
@@ -217,3 +223,15 @@ int AudioDecoder::GetSamplesizeForFormat(AudioDecoderBase::Format format) {
 	assert(false && "Bad format");
 	return -1;
 }
+
+void AudioDecoder::SetLogVolume() {
+	int balance = GetBalance();
+	float left_gain = 1.f, right_gain = 1.f;
+	if (balance <= 50) {
+		right_gain = balance / 50.f;
+	} else {
+		left_gain = (100 - balance) / 50.f;
+	}
+	log_volume.first = AdjustVolume(volume * left_gain);
+	log_volume.second = AdjustVolume(volume * right_gain);
+}

src/audio_decoder.h

@@ -71,7 +71,7 @@ class AudioDecoder : public AudioDecoderBase {
 	 *
 	 * @return current volume (from 0 - 100)
 	 */
-	int GetVolume() const final override;
+	std::pair<int, int> GetVolume() const final override;
 
 	/**
 	 * Sets the volume of the audio decoder.
@@ -100,14 +100,21 @@ class AudioDecoder : public AudioDecoderBase {
 	 * @param delta Time in us since the last call of this function.
 	 */
 	void Update(std::chrono::microseconds delta) final override;
+
+	/**
+	 * Sets the pan/balance of the audio decoder.
+	 */
+	void SetBalance(int new_balance) override;
 private:
 	bool paused = false;
 	float volume = 0.0f;
-	float log_volume = 0.0f; // as used by RPG_RT
+	std::pair<float, float> log_volume = {0.0f, 0.0f}; // as used by RPG_RT
 
 	int fade_volume_end = 0;
 	std::chrono::microseconds fade_time = std::chrono::microseconds(0);
 	float delta_volume_step = 0.0f;
+
+	void SetLogVolume();
 };
 
 #endif

src/audio_decoder_base.cpp

@@ -73,6 +73,14 @@ int AudioDecoderBase::GetLoopCount() const {
 	return loop_count;
 }
 
+int AudioDecoderBase::GetBalance() const {
+	return balance;
+}
+
+void AudioDecoderBase::SetBalance(int new_balance) {
+	balance = Utils::Clamp(new_balance, 0, 100);
+}
+
 bool AudioDecoderBase::WasInited() const {
 	return true;
 }

src/audio_decoder_base.h

@@ -107,6 +107,18 @@ class AudioDecoderBase {
 	 */
 	virtual int GetLoopCount() const;
 
+	/**
+	 * Gets the current balance of the audio decoder.
+	 * The value is in the range 0 - 100, where 0 is full left, 100 is full right
+	 */
+	virtual int GetBalance() const;
+
+	/**
+	 * Sets the balance of the audio decoder.
+	 * The value is clamped to the range 0 - 100.
+	 */
+	virtual void SetBalance(int new_balance);
+
 	// Functions to be implemented by the audio decoder
 	/**
 	 * Assigns a stream to the audio decoder.
@@ -131,9 +143,9 @@ class AudioDecoderBase {
 	 * Gets the current volume of the audio decoder.
 	 * Fades are considered.
 	 *
-	 * @return current volume (from 0 - 100)
+	 * @return left-right pair of current volume (from 0 - 100)
 	 */
-	virtual int GetVolume() const = 0;
+	virtual std::pair<int, int> GetVolume() const = 0;
 
 	/**
 	 * Sets the current volume of the audio decoder.
@@ -282,6 +294,7 @@ class AudioDecoderBase {
 
 	bool looping = false;
 	int loop_count = 0;
+	int balance = 50;
 
 	int Decode(uint8_t* buffer, int length, int recursion_depth);
 };

src/audio_decoder_midi.cpp

@@ -21,6 +21,7 @@
 #include "audio_decoder_midi.h"
 #include "midisequencer.h"
 #include "output.h"
+#include "utils.h"
 
 using namespace std::chrono_literals;
 
@@ -38,6 +39,7 @@ constexpr int samples_per_play = 64 / sample_divider;
 
 static const uint8_t midi_set_reg_param_upper = 0x6;
 static const uint8_t midi_control_volume = 0x7;
+static const uint8_t midi_control_pan = 0xA;
 static const uint8_t midi_event_control_change = 0xB;
 static const uint8_t midi_set_reg_param_lower = 0x26;
 static const uint8_t midi_control_all_sound_off = 0x78;
@@ -66,6 +68,10 @@ static uint32_t midimsg_volume(uint8_t channel, uint8_t volume) {
 	return midimsg_make(midi_event_control_change, channel, midi_control_volume, volume);
 }
 
+static uint32_t midimsg_pan(uint8_t channel, uint8_t pan) {
+	return midimsg_make(midi_event_control_change, channel, midi_control_pan, pan);
+}
+
 static uint32_t midimsg_reset_all_controller(uint8_t channel) {
 	return midimsg_make(midi_event_control_change, channel, midi_control_reset_all_controller, 0);
 }
@@ -175,14 +181,15 @@ void AudioDecoderMidi::Resume() {
 	}
 }
 
-int AudioDecoderMidi::GetVolume() const {
+std::pair<int, int> AudioDecoderMidi::GetVolume() const {
 	// When handled by Midi messages fake a 100 otherwise the volume is adjusted twice
 
 	if (!mididec->SupportsMidiMessages()) {
-		return static_cast<int>(log_volume);
+		const auto& [left, right] = log_volume;
+		return std::pair(static_cast<int>(left), static_cast<int>(right));
 	}
 
-	return 100;
+	return {100, 100};
 }
 
 void AudioDecoderMidi::SetVolume(int new_volume) {
@@ -195,9 +202,7 @@ void AudioDecoderMidi::SetVolume(int new_volume) {
 		mididec->SendMidiMessage(msg);
 	}
 
-	if (!mididec->SupportsMidiMessages()) {
-		log_volume = AdjustVolume(volume * 100.0f);
-	}
+	SetLogVolume();
 }
 
 void AudioDecoderMidi::SetFade(int end, std::chrono::milliseconds duration) {
@@ -214,6 +219,19 @@ void AudioDecoderMidi::SetFade(int end, std::chrono::milliseconds duration) {
 	delta_volume_step = (fade_volume_end - volume) / fade_steps;
 }
 
+void AudioDecoderMidi::SetBalance(int new_balance) {
+	AudioDecoderBase::SetBalance(new_balance);
+	SetLogVolume();
+
+	uint8_t pan = Utils::Clamp(last_known_midi_pan + ((GetBalance() - 50) * 2), 0, 127);
+	for (int channel = 0; channel < 16; channel++) {
+		uint32_t msg = midimsg_pan(channel, pan);
+		mididec->SendMidiMessage(msg);
+	}
+
+	Output::Warning("setting balance to {}", new_balance);
+}
+
 bool AudioDecoderMidi::Seek(std::streamoff offset, std::ios_base::seekdir origin) {
 	assert(!tempo.empty());
 
@@ -251,9 +269,7 @@ void AudioDecoderMidi::Update(std::chrono::microseconds delta) {
 	}
 	if (fade_steps > 0 && mtime - last_fade_mtime > 0.1s) {
 		volume = Utils::Clamp<float>(volume + delta_volume_step, 0.0f, 1.0f);
-		if (!mididec->SupportsMidiMessages()) {
-			log_volume = AdjustVolume(volume * 100.0f);
-		}
+		SetLogVolume();
 		for (int i = 0; i < 16; i++) {
 			uint32_t msg = midimsg_volume(i, static_cast<uint8_t>(channel_volumes[i] * volume * global_volume));
 			mididec->SendMidiMessage(msg);
@@ -392,6 +408,12 @@ void AudioDecoderMidi::midi_message(int, uint_least32_t message) {
 		channel_volumes[channel] = value2;
 		// Send the modified volume to midiout
 		message = midimsg_volume(channel, static_cast<uint8_t>(value2 * volume * global_volume));
+	} else if (event_type == midi_event_control_change && value1 == midi_control_pan) {
+		// See #2585 for details on MIDI values processed by RPG_RT
+		last_known_midi_pan = value2;
+		uint8_t pan = Utils::Clamp(value2 + ((GetBalance() - 50) * 2), 0, 127);
+		Output::Warning("want={} changed={} balance={}", value2, pan, GetBalance());
+		message = midimsg_pan(channel, pan);
 	}
 	if (midimsg_validate(message)) {
 		mididec->SendMidiMessage(message);
@@ -480,3 +502,17 @@ int AudioDecoderMidi::MidiTempoData::GetSamples(std::chrono::microseconds mtime_
 	int ticks_since_last = static_cast<int>(ticks_per_us * delta.count());
 	return samples + static_cast<int>(ticks_since_last * samples_per_tick);
 }
+
+void AudioDecoderMidi::SetLogVolume() {
+	if (!mididec->SupportsMidiMessages()) {
+		int balance = GetBalance();
+		float left_gain = 1.f, right_gain = 1.f;
+		if (balance <= 50) {
+			right_gain = balance / 50.f;
+		} else {
+			left_gain = (100 - balance) / 50.f;
+		}
+		log_volume.first = AdjustVolume(volume * left_gain);
+		log_volume.second = AdjustVolume(volume * right_gain);
+	}
+}

src/audio_decoder_midi.h

@@ -56,7 +56,7 @@ class AudioDecoderMidi final : public AudioDecoderBase, public midisequencer::ou
 	 *
 	 * @return current volume (from 0 - 100)
 	 */
-	int GetVolume() const override;
+	std::pair<int, int> GetVolume() const override;
 
 	/**
 	 * Sets the volume of the MIDI devices by sending MIDI messages
@@ -156,6 +156,11 @@ class AudioDecoderMidi final : public AudioDecoderBase, public midisequencer::ou
 	 */
 	bool IsPaused() const;
 
+	/**
+	 * Sets the balance of the MIDI decoder.
+	 */
+	void SetBalance(int new_balance) override;
+
 	std::vector<uint8_t> file_buffer;
 	size_t file_buffer_pos = 0;
 private:
@@ -177,7 +182,7 @@ class AudioDecoderMidi final : public AudioDecoderBase, public midisequencer::ou
 	bool paused = false;
 	float volume = 0.0f;
 	float global_volume = 1.0f; // only used by midiout
-	float log_volume = 0.0f; // as used by RPG_RT, for Midi decoder without event support
+	std::pair<float, float> log_volume = {0.0f, 0.0f}; // as used by RPG_RT, for Midi decoder without event support
 	bool loops_to_end = false;
 
 	int fade_steps = 0;
@@ -210,6 +215,9 @@ class AudioDecoderMidi final : public AudioDecoderBase, public midisequencer::ou
 	// Contains one entry per tempo change (latest on top)
 	// When looping all entries after the loop point are dropped
 	std::vector<MidiTempoData> tempo;
+
+	void SetLogVolume();
+	int last_known_midi_pan = 64;
 };
 
 #endif