Merge pull request #20542 from hrydgard/more-audio-refactor

Fix UWP audio glitches, minor audio cleanups

Core/Debugger/WebSocket/GPUStatsSubscriber.cpp

@@ -26,8 +26,8 @@ struct CollectedStats {
 	float fps;
 	float actual_fps;
 	char statbuf[4096];
-	std::vector<double> frameTimes;
-	std::vector<double> sleepTimes;
+	std::vector<float> frameTimes;
+	std::vector<float> sleepTimes;
 	int frameTimePos;
 };
 
@@ -123,8 +123,8 @@ void WebSocketGPUStatsState::FlipListener() {
 	__DisplayGetDebugStats(stats.statbuf, sizeof(stats.statbuf));
 
 	int valid;
-	double *sleepHistory;
-	double *history = __DisplayGetFrameTimes(&valid, &stats.frameTimePos, &sleepHistory);
+	float *sleepHistory;
+	float *history = __DisplayGetFrameTimes(&valid, &stats.frameTimePos, &sleepHistory);
 
 	stats.frameTimes.resize(valid);
 	stats.sleepTimes.resize(valid);

Core/HW/Display.cpp

@@ -63,8 +63,8 @@ static int fpsHistoryPos = 0;
 static int fpsHistoryValid = 0;
 
 // Frame time stats.
-static double frameTimeHistory[600];
-static double frameSleepHistory[600];
+static float frameTimeHistory[600];
+static float frameSleepHistory[600];
 static constexpr int frameTimeHistorySize = (int)ARRAY_SIZE(frameTimeHistory);
 static int frameTimeHistoryPos = 0;
 static int frameTimeHistoryValid = 0;
@@ -93,13 +93,13 @@ static void CalculateFPS() {
 	}
 
 	if ((DebugOverlay)g_Config.iDebugOverlay == DebugOverlay::FRAME_GRAPH || coreCollectDebugStats) {
-		frameTimeHistory[frameTimeHistoryPos++] = now - lastFrameTimeHistory;
+		frameTimeHistory[frameTimeHistoryPos++] = (float)(now - lastFrameTimeHistory);
 		lastFrameTimeHistory = now;
 		frameTimeHistoryPos = frameTimeHistoryPos % frameTimeHistorySize;
 		if (frameTimeHistoryValid < frameTimeHistorySize) {
 			++frameTimeHistoryValid;
 		}
-		frameSleepHistory[frameTimeHistoryPos] = 0.0;
+		frameSleepHistory[frameTimeHistoryPos] = 0.0f;
 	}
 }
 
@@ -169,7 +169,7 @@ void DisplayAdjustAccumulatedHcount(uint32_t diff) {
 	hCountBase += diff;
 }
 
-double *__DisplayGetFrameTimes(int *out_valid, int *out_pos, double **out_sleep) {
+float *__DisplayGetFrameTimes(int *out_valid, int *out_pos, float **out_sleep) {
 	*out_valid = frameTimeHistoryValid;
 	*out_pos = frameTimeHistoryPos;
 	*out_sleep = frameSleepHistory;

Core/HW/Display.h

@@ -42,7 +42,7 @@ void __DisplayGetDebugStats(char *stats, size_t bufsize);
 void __DisplayGetAveragedFPS(float *out_vps, float *out_fps);
 void __DisplayGetFPS(float *out_vps, float *out_fps, float *out_actual_fps);
 void __DisplayGetVPS(float *out_vps);
-double *__DisplayGetFrameTimes(int *out_valid, int *out_pos, double **out_sleep);
+float *__DisplayGetFrameTimes(int *out_valid, int *out_pos, float **out_sleep);
 int DisplayGetSleepPos();
 void DisplayNotifySleep(double t, int pos = -1);
 bool DisplayIsRunningSlow();

Core/HW/StereoResampler.cpp

@@ -50,10 +50,10 @@
 #include "Core/System.h"
 
 StereoResampler::StereoResampler() noexcept
-		: m_maxBufsize(MAX_BUFSIZE_DEFAULT)
-	  , m_targetBufsize(TARGET_BUFSIZE_DEFAULT) {
+		: maxBufsize_(MAX_BUFSIZE_DEFAULT)
+	  , targetBufsize_(TARGET_BUFSIZE_DEFAULT) {
 	// Need to have space for the worst case in case it changes.
-	m_buffer = new int16_t[MAX_BUFSIZE_EXTRA * 2]();
+	buffer_ = new int16_t[MAX_BUFSIZE_EXTRA * 2]();
 
 	// Some Android devices are v-synced to non-60Hz framerates. We simply timestretch audio to fit.
 	// TODO: should only do this if auto frameskip is off?
@@ -63,30 +63,30 @@ StereoResampler::StereoResampler() noexcept
 	if (refresh != 60.0f && refresh > 50.0f && refresh < 70.0f) {
 		int input_sample_rate = (int)(44100 * (refresh / 60.0f));
 		INFO_LOG(Log::Audio, "StereoResampler: Adjusting target sample rate to %dHz", input_sample_rate);
-		m_input_sample_rate = input_sample_rate;
+		inputSampleRateHz_ = input_sample_rate;
 	}
 
 	UpdateBufferSize();
 }
 
 StereoResampler::~StereoResampler() {
-	delete[] m_buffer;
-	m_buffer = nullptr;
+	delete[] buffer_;
+	buffer_ = nullptr;
 }
 
 void StereoResampler::UpdateBufferSize() {
 	if (g_Config.bExtraAudioBuffering) {
-		m_maxBufsize = MAX_BUFSIZE_EXTRA;
-		m_targetBufsize = TARGET_BUFSIZE_EXTRA;
+		maxBufsize_ = MAX_BUFSIZE_EXTRA;
+		targetBufsize_ = TARGET_BUFSIZE_EXTRA;
 	} else {
-		m_maxBufsize = MAX_BUFSIZE_DEFAULT;
-		m_targetBufsize = TARGET_BUFSIZE_DEFAULT;
+		maxBufsize_ = MAX_BUFSIZE_DEFAULT;
+		targetBufsize_ = TARGET_BUFSIZE_DEFAULT;
 
 		int systemBufsize = System_GetPropertyInt(SYSPROP_AUDIO_FRAMES_PER_BUFFER);
-		if (systemBufsize > 0 && m_targetBufsize < systemBufsize + TARGET_BUFSIZE_MARGIN) {
-			m_targetBufsize = std::min(4096, systemBufsize + TARGET_BUFSIZE_MARGIN);
-			if (m_targetBufsize * 2 > MAX_BUFSIZE_DEFAULT)
-				m_maxBufsize = MAX_BUFSIZE_EXTRA;
+		if (systemBufsize > 0 && targetBufsize_ < systemBufsize + TARGET_BUFSIZE_MARGIN) {
+			targetBufsize_ = std::min(4096, systemBufsize + TARGET_BUFSIZE_MARGIN);
+			if (targetBufsize_ * 2 > MAX_BUFSIZE_DEFAULT)
+				maxBufsize_ = MAX_BUFSIZE_EXTRA;
 		}
 	}
 }
@@ -146,7 +146,7 @@ inline void ClampBufferToS16WithVolume(s16 *out, const s32 *in, size_t size, int
 }
 
 void StereoResampler::Clear() {
-	memset(m_buffer, 0, m_maxBufsize * 2 * sizeof(int16_t));
+	memset(buffer_, 0, maxBufsize_ * 2 * sizeof(int16_t));
 }
 
 inline int16_t MixSingleSample(int16_t s1, int16_t s2, uint16_t frac) {
@@ -173,10 +173,10 @@ unsigned int StereoResampler::Mix(short* samples, unsigned int numSamples, bool
 	// so we will just ignore new written data while interpolating (until it wraps...).
 	// Without this cache, the compiler wouldn't be allowed to optimize the
 	// interpolation loop.
-	u32 indexR = m_indexR.load();
-	u32 indexW = m_indexW.load();
+	u32 indexR = indexR_.load();
+	u32 indexW = indexW_.load();
 
-	const int INDEX_MASK = (m_maxBufsize * 2 - 1);
+	const int INDEX_MASK = (maxBufsize_ * 2 - 1);
 
 	// This is only for debug visualization, not used for anything.
 	lastBufSize_ = ((indexW - indexR) & INDEX_MASK) / 2;
@@ -189,24 +189,24 @@ unsigned int StereoResampler::Mix(short* samples, unsigned int numSamples, bool
 	numLeft -= droppedSamples_;
 	droppedSamples_ = 0;
 
-	// m_numLeftI here becomes a lowpass filtered version of numLeft.
-	m_numLeftI = (numLeft + m_numLeftI * (CONTROL_AVG - 1.0f)) / CONTROL_AVG;
+	// numLeftI_ here becomes a lowpass filtered version of numLeft.
+	numLeftI_ = (numLeft + numLeftI_ * (CONTROL_AVG - 1.0f)) / CONTROL_AVG;
 
 	// Here we try to keep the buffer size around m_lowwatermark (which is
 	// really now more like desired_buffer_size) by adjusting the speed.
 	// Note that the speed of adjustment here does not take the buffer size into
 	// account. Since this is called once per "output frame", the frame size
 	// will affect how fast this algorithm reacts, which can't be a good thing.
-	float offset = (m_numLeftI - (float)m_targetBufsize) * CONTROL_FACTOR;
+	float offset = (numLeftI_ - (float)targetBufsize_) * CONTROL_FACTOR;
 	if (offset > MAX_FREQ_SHIFT) offset = MAX_FREQ_SHIFT;
 	if (offset < -MAX_FREQ_SHIFT) offset = -MAX_FREQ_SHIFT;
 
-	output_sample_rate_ = (float)(m_input_sample_rate + offset);
-	const u32 ratio = (u32)(65536.0 * output_sample_rate_ / (double)sample_rate);
+	outputSampleRateHz_ = (float)(inputSampleRateHz_ + offset);
+	const u32 ratio = (u32)(65536.0 * outputSampleRateHz_ / (double)sample_rate);
 	ratio_ = ratio;
 	// TODO: consider a higher-quality resampling algorithm.
 	// TODO: Add a fast path for 1:1.
-	u32 frac = m_frac;
+	u32 frac = frac_;
 	for (currentSample = 0; currentSample < numSamples * 2; currentSample += 2) {
 		if (((indexW - indexR) & INDEX_MASK) <= 2) {
 			// Ran out!
@@ -216,32 +216,32 @@ unsigned int StereoResampler::Mix(short* samples, unsigned int numSamples, bool
 			break;
 		}
 		u32 indexR2 = indexR + 2; //next sample
-		s16 l1 = m_buffer[indexR & INDEX_MASK]; //current
-		s16 r1 = m_buffer[(indexR + 1) & INDEX_MASK]; //current
-		s16 l2 = m_buffer[indexR2 & INDEX_MASK]; //next
-		s16 r2 = m_buffer[(indexR2 + 1) & INDEX_MASK]; //next
+		s16 l1 = buffer_[indexR & INDEX_MASK]; //current
+		s16 r1 = buffer_[(indexR + 1) & INDEX_MASK]; //current
+		s16 l2 = buffer_[indexR2 & INDEX_MASK]; //next
+		s16 r2 = buffer_[(indexR2 + 1) & INDEX_MASK]; //next
 		samples[currentSample] = MixSingleSample(l1, l2, (u16)frac);
 		samples[currentSample + 1] = MixSingleSample(r1, r2, (u16)frac);
 		frac += ratio;
 		indexR += 2 * (frac >> 16);
 		frac &= 0xffff;
 	}
-	m_frac = frac;
+	frac_ = frac;
 
 	// Let's not count the underrun padding here.
 	outputSampleCount_ += currentSample / 2;
 
 	// Padding with the last value to reduce clicking
 	short s[2];
-	s[0] = clamp_s16(m_buffer[(indexR - 1) & INDEX_MASK]);
-	s[1] = clamp_s16(m_buffer[(indexR - 2) & INDEX_MASK]);
+	s[0] = clamp_s16(buffer_[(indexR - 1) & INDEX_MASK]);
+	s[1] = clamp_s16(buffer_[(indexR - 2) & INDEX_MASK]);
 	for (; currentSample < numSamples * 2; currentSample += 2) {
 		samples[currentSample] = s[0];
 		samples[currentSample + 1] = s[1];
 	}
 
 	// Flush cached variable
-	m_indexR.store(indexR);
+	indexR_.store(indexR);
 
 	// TODO: What should we actually return here?
 	return currentSample / 2;
@@ -252,21 +252,21 @@ void StereoResampler::PushSamples(const s32 *samples, unsigned int numSamples, f
 	inputSampleCount_ += numSamples;
 
 	UpdateBufferSize();
-	const int INDEX_MASK = (m_maxBufsize * 2 - 1);
+	const int INDEX_MASK = (maxBufsize_ * 2 - 1);
 	// Cache access in non-volatile variable
 	// indexR isn't allowed to cache in the audio throttling loop as it
 	// needs to get updates to not deadlock.
-	u32 indexW = m_indexW.load();
+	u32 indexW = indexW_.load();
 
-	u32 cap = m_maxBufsize * 2;
+	u32 cap = maxBufsize_ * 2;
 	// If fast-forwarding, no need to fill up the entire buffer, just screws up timing after releasing the fast-forward button.
 	if (PSP_CoreParameter().fastForward) {
-		cap = m_targetBufsize * 2;
+		cap = targetBufsize_ * 2;
 	}
 
 	// Check if we have enough free space
-	// indexW == m_indexR results in empty buffer, so indexR must always be smaller than indexW
-	if (numSamples * 2 + ((indexW - m_indexR.load()) & INDEX_MASK) >= cap) {
+	// indexW == indexR_ results in empty buffer, so indexR must always be smaller than indexW
+	if (numSamples * 2 + ((indexW - indexR_.load()) & INDEX_MASK) >= cap) {
 		if (!PSP_CoreParameter().fastForward) {
 			overrunCount_++;
 		}
@@ -278,15 +278,15 @@ void StereoResampler::PushSamples(const s32 *samples, unsigned int numSamples, f
 	int volume = (int)(multiplier * 4096.0f);
 
 	// Check if we need to roll over to the start of the buffer during the copy.
-	unsigned int indexW_left_samples = m_maxBufsize * 2 - (indexW & INDEX_MASK);
+	unsigned int indexW_left_samples = maxBufsize_ * 2 - (indexW & INDEX_MASK);
 	if (numSamples * 2 > indexW_left_samples) {
-		ClampBufferToS16WithVolume(&m_buffer[indexW & INDEX_MASK], samples, indexW_left_samples, volume);
-		ClampBufferToS16WithVolume(&m_buffer[0], samples + indexW_left_samples, numSamples * 2 - indexW_left_samples, volume);
+		ClampBufferToS16WithVolume(&buffer_[indexW & INDEX_MASK], samples, indexW_left_samples, volume);
+		ClampBufferToS16WithVolume(&buffer_[0], samples + indexW_left_samples, numSamples * 2 - indexW_left_samples, volume);
 	} else {
-		ClampBufferToS16WithVolume(&m_buffer[indexW & INDEX_MASK], samples, numSamples * 2, volume);
+		ClampBufferToS16WithVolume(&buffer_[indexW & INDEX_MASK], samples, numSamples * 2, volume);
 	}
 
-	m_indexW += numSamples * 2;
+	indexW_ += numSamples * 2;
 	lastPushSize_ = numSamples;
 }
 
@@ -295,8 +295,10 @@ void StereoResampler::GetAudioDebugStats(char *buf, size_t bufSize) {
 
 	double effective_input_sample_rate = (double)inputSampleCount_ / elapsed;
 	double effective_output_sample_rate = (double)outputSampleCount_ / elapsed;
+
+	double bufferLatencyMs = 1000.0 * (double)lastBufSize_ / (double)inputSampleRateHz_;
 	snprintf(buf, bufSize,
-		"Audio buffer: %d/%d (target: %d)\n"
+		"Audio buffer: %d/%d (%0.1fms, target: %d)\n"
 		"Filtered: %0.2f\n"
 		"Underruns: %d\n"
 		"Overruns: %d\n"
@@ -306,13 +308,14 @@ void StereoResampler::GetAudioDebugStats(char *buf, size_t bufSize) {
 		"Push size: %d\n"
 		"Ratio: %0.6f\n",
 		lastBufSize_,
-		m_maxBufsize,
-		m_targetBufsize,
-		m_numLeftI,
+		maxBufsize_,
+		bufferLatencyMs,
+		targetBufsize_,
+		numLeftI_,
 		underrunCountTotal_,
 		overrunCountTotal_,
-		(int)output_sample_rate_,
-		m_input_sample_rate,
+		(int)outputSampleRateHz_,
+		inputSampleRateHz_,
 		effective_input_sample_rate,
 		effective_output_sample_rate,
 		lastPushSize_,

Core/HW/StereoResampler.h

@@ -47,17 +47,19 @@ public:
 private:
 	void UpdateBufferSize();
 
-	int m_maxBufsize;
-	int m_targetBufsize;
+	int maxBufsize_;
+	int targetBufsize_;
 
-	unsigned int m_input_sample_rate = 44100;
-	int16_t *m_buffer;
-	std::atomic<u32> m_indexW;
-	std::atomic<u32> m_indexR;
-	float m_numLeftI = 0.0f;
+	// This can be adjusted, for the case of non-60hz output (a few hz off).
+	int inputSampleRateHz_ = 44100;
 
-	u32 m_frac = 0;
-	float output_sample_rate_ = 0.0;
+	int16_t *buffer_;
+	std::atomic<u32> indexW_;
+	std::atomic<u32> indexR_;
+	float numLeftI_ = 0.0f;
+
+	u32 frac_ = 0;
+	float outputSampleRateHz_ = 0.0;
 	int lastBufSize_ = 0;
 	int lastPushSize_ = 0;
 	u32 ratio_ = 0;

UI/DebugOverlay.cpp

@@ -97,9 +97,9 @@ static void DrawControlDebug(UIContext *ctx, const ControlMapper &mapper, const
 
 static void DrawFrameTimes(UIContext *ctx, const Bounds &bounds) {
 	FontID ubuntu24("UBUNTU24");
-	double *sleepHistory;
+	float *sleepHistory;
 	int valid, pos;
-	double *history = __DisplayGetFrameTimes(&valid, &pos, &sleepHistory);
+	float *history = __DisplayGetFrameTimes(&valid, &pos, &sleepHistory);
 	int scale = 7000;
 	int width = 600;
 

UI/NativeApp.cpp

@@ -1226,7 +1226,7 @@ void NativeFrame(GraphicsContext *graphicsContext) {
 
 		float refreshRate = System_GetPropertyFloat(SYSPROP_DISPLAY_REFRESH_RATE);
 		// Simple throttling to not burn the GPU in the menu.
-		// TODO: This should move into NativeFrame. Also, it's only necessary in MAILBOX or IMMEDIATE presentation modes.
+		// TODO: This is only necessary in MAILBOX or IMMEDIATE presentation modes.
 		double diffTime = time_now_d() - startTime;
 		int sleepTime = (int)(1000.0 / refreshRate) - (int)(diffTime * 1000.0);
 		if (sleepTime > 0)

UWP/XAudioSoundStream.cpp

@@ -53,7 +53,8 @@ private:
 		bufferCount = 3,
 		channels = 2,
 	};
-	float audioBuffer_[samplesPerBuffer * channels];
+	float audioBuffer_[bufferCount][samplesPerBuffer * channels];
+	int curBuffer_ = 0;
 
 	uint32_t cursor_ = 0;
 
@@ -135,13 +136,17 @@ bool XAudioBackend::InitOutputDevice(std::string_view uniqueId, LatencyMode late
 			sourceVoice_->GetState(&state);
 			if (state.BuffersQueued < bufferCount) {
 				// Fill buffer with audio
-				callback_(audioBuffer_, samplesPerBuffer, channels, format_.nSamplesPerSec, userdata_);
+				callback_(audioBuffer_[curBuffer_], samplesPerBuffer, channels, format_.nSamplesPerSec, userdata_);
 
 				XAUDIO2_BUFFER buf = {};
 				buf.AudioBytes = samplesPerBuffer * channels * sizeof(float);
-				buf.pAudioData = reinterpret_cast<BYTE*>(audioBuffer_);
+				buf.pAudioData = reinterpret_cast<BYTE*>(audioBuffer_[curBuffer_]);
 				buf.Flags = 0;
 				sourceVoice_->SubmitSourceBuffer(&buf);
+				curBuffer_ += 1;
+				if (curBuffer_ >= bufferCount) {
+					curBuffer_ = 0;
+				}
 			} else {
 				std::this_thread::sleep_for(std::chrono::milliseconds(1));
 			}