diff --git a/Core/HLE/sceAtrac.cpp b/Core/HLE/sceAtrac.cpp
index f8c312e5a6..d0608cb8df 100644
--- a/Core/HLE/sceAtrac.cpp
+++ b/Core/HLE/sceAtrac.cpp
@@ -1258,8 +1258,12 @@ u32 _AtracDecodeData(int atracID, u8 *outbuf, u32 outbufPtr, u32 *SamplesNum, u3
 				}
 				if ((atrac->bufferState_ & ATRAC_STATUS_STREAMED_MASK) == ATRAC_STATUS_STREAMED_MASK) {
 					// Whatever bytes we have left were added from the loop.
-					atrac->first_.fileoffset = atrac->FileOffsetBySample(atrac->loopStartSample_ - atrac->FirstOffsetExtra() - atrac->firstSampleOffset_ - atrac->SamplesPerFrame() * 2);
-					// Skip the initial frame at the start.
+					u32 loopOffset = atrac->FileOffsetBySample(atrac->loopStartSample_ - atrac->FirstOffsetExtra() - atrac->firstSampleOffset_ - atrac->SamplesPerFrame() * 2);
+					// TODO: Hmm, need to manage the buffer better.  But don't move fileoffset if we already have valid data.
+					if (loopOffset > atrac->first_.fileoffset || loopOffset + atrac->bufferValidBytes_ < atrac->first_.fileoffset) {
+						// Skip the initial frame at the start.
+						atrac->first_.fileoffset = atrac->FileOffsetBySample(atrac->loopStartSample_ - atrac->FirstOffsetExtra() - atrac->firstSampleOffset_ - atrac->SamplesPerFrame() * 2);
+					}
 				}
 			} else if (hitEnd) {
 				finishFlag = 1;