diff --git a/libavcodec/h264_mp4toannexb_bsf.c b/libavcodec/h264_mp4toannexb_bsf.c
index a4cbf87310..03eb956caf 100644
--- a/libavcodec/h264_mp4toannexb_bsf.c
+++ b/libavcodec/h264_mp4toannexb_bsf.c
@@ -41,8 +41,8 @@ static void alloc_and_copy(uint8_t **poutbuf,          int *poutbuf_size,
     if (!offset)
         AV_WB32(*poutbuf+sps_pps_size, 1);
     else {
-        (*poutbuf+offset)[0] = (*poutbuf+offset)[1] = 0;
-        (*poutbuf+offset)[2] = 1;
+        (*poutbuf+offset+sps_pps_size)[0] = (*poutbuf+offset+sps_pps_size)[1] = 0;
+        (*poutbuf+offset+sps_pps_size)[2] = 1;
     }
 }