diff --git a/libavcodec/h264_mp4toannexb_bsf.c b/libavcodec/h264_mp4toannexb_bsf.c
index cbbbc27ff7..aa45f31c81 100644
--- a/libavcodec/h264_mp4toannexb_bsf.c
+++ b/libavcodec/h264_mp4toannexb_bsf.c
@@ -49,7 +49,7 @@ typedef struct H264BSFContext {
 static void count_or_copy(uint8_t **out, uint64_t *out_size,
                           const uint8_t *in, int in_size, int ps, int copy)
 {
-    uint8_t start_code_size = ps < 0 ? 0 : *out_size == 0 || ps ? 4 : 3;
+    uint8_t start_code_size = ps < 0 ? 0 : 4;
 
     if (copy) {
         memcpy(*out + start_code_size, in, in_size);