diff --git a/libaegisub/audio/provider_convert.cpp b/libaegisub/audio/provider_convert.cpp
index 88617cd1cc43c82e01bf9fef86a1e5a20cd0ae35..644cb37f26a58d27e074ec192eda1d945e9235de 100644
--- a/libaegisub/audio/provider_convert.cpp
+++ b/libaegisub/audio/provider_convert.cpp
@@ -140,25 +140,27 @@ public:
 	}
 
 	void FillBuffer(void *buf, int64_t start, int64_t count) const override {
-		bool not_end = start + count < num_samples;
-		int64_t src_count = count / 2;
-		source->GetAudio(buf, start / 2, src_count + not_end);
+		int16_t *src, *dst = static_cast<int16_t *>(buf);
 
-		auto buf16 = reinterpret_cast<int16_t*>(buf);
-
-		if (!not_end) {
-			// We weren't able to request a sample past the end so just
-			// duplicate the last sample
-			buf16[src_count] = buf16[src_count + 1];
+		// We need to always get at least two samples to be able to interpolate
+		int16_t srcbuf[2];
+		if (count == 1) {
+			source->GetAudio(srcbuf, start / 2, 2);
+			src = srcbuf;
+		}
+		else {
+			source->GetAudio(buf, start / 2, (start + count) / 2 - start / 2 + 1);
+			src = dst;
 		}
-
-		if (count % 2)
-			buf16[count - 1] = buf16[src_count];
 
 		// walking backwards so that the conversion can be done in place
-		for (int64_t i = src_count - 1; i >= 0; --i) {
-			buf16[i * 2] = buf16[i];
-			buf16[i * 2 + 1] = (int16_t)(((int32_t)buf16[i] + buf16[i + 1]) / 2);
+		for (; count > 0; --count) {
+			auto src_index = (start + count - 1) / 2 - start / 2;
+			auto i = count - 1;
+			if ((start + i) & 1)
+				dst[i] = (int16_t)(((int32_t)src[src_index] + src[src_index + 1]) / 2);
+			else
+				dst[i] = src[src_index];
 		}
 	}
 };
diff --git a/tests/tests/audio.cpp b/tests/tests/audio.cpp
index d3342954db5f6b411bf82ae221d5e1714df76d73..e486c70849873f77f73cd21bd5a8ed1b07ed0986 100644
--- a/tests/tests/audio.cpp
+++ b/tests/tests/audio.cpp
@@ -200,6 +200,42 @@ TEST(lagi_audio, convert_32bit) {
 	EXPECT_EQ(SHRT_MAX, sample);
 }
 
+TEST(lagi_audio, sample_doubling) {
+	struct AudioProvider : agi::AudioProvider {
+		AudioProvider() {
+			channels = 1;
+			num_samples = 90 * 20000;
+			decoded_samples = num_samples;
+			sample_rate = 20000;
+			bytes_per_sample = 2;
+			float_samples = false;
+		}
+
+		void FillBuffer(void *buf, int64_t start, int64_t count) const override {
+			auto out = static_cast<int16_t *>(buf);
+			for (int64_t end = start + count; start < end; ++start)
+				*out++ = (int16_t)(start * 2);
+		}
+	};
+
+	auto provider = agi::CreateConvertAudioProvider(agi::make_unique<AudioProvider>());
+	EXPECT_EQ(40000, provider->GetSampleRate());
+
+	int16_t samples[6];
+	for (int k = 0; k < 6; ++k) {
+		SCOPED_TRACE(k);
+		for (int i = k; i < 6; ++i) {
+			SCOPED_TRACE(i);
+			memset(samples, 0, sizeof(samples));
+			provider->GetAudio(samples, k, i - k);
+			for (int j = 0; j < i - k; ++j)
+				EXPECT_EQ(j + k, samples[j]);
+			for (int j = i - k; j < 6 - k; ++j)
+				EXPECT_EQ(0, samples[j]);
+		}
+	}
+}
+
 TEST(lagi_audio, pcm_simple) {
 	auto path = agi::Path().Decode("?temp/pcm_simple");
 	{