diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index 98d47e5bef..21bd248adf 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -725,14 +725,6 @@ static int mkv_write_header(AVFormatContext *s)
     ret = mkv_write_chapters(s);
     if (ret < 0) return ret;
 
-    ret = mkv_add_seekhead_entry(mkv->cluster_seekhead, MATROSKA_ID_CLUSTER, url_ftell(pb));
-    if (ret < 0) return ret;
-
-    mkv->cluster_pos = url_ftell(pb);
-    mkv->cluster = start_ebml_master(pb, MATROSKA_ID_CLUSTER, 0);
-    put_ebml_uint(pb, MATROSKA_ID_CLUSTERTIMECODE, 0);
-    mkv->cluster_pts = 0;
-
     if (url_is_streamed(s->pb))
         mkv_write_seekhead(pb, mkv->main_seekhead);
 
@@ -849,12 +841,7 @@ static int mkv_write_packet(AVFormatContext *s, AVPacket *pkt)
     int duration = pkt->duration;
     int ret;
 
-    // start a new cluster every 5 MB or 5 sec
-    if (url_ftell(pb) > mkv->cluster_pos + 5*1024*1024 || pkt->pts > mkv->cluster_pts + 5000) {
-        av_log(s, AV_LOG_DEBUG, "Starting new cluster at offset %" PRIu64
-               " bytes, pts %" PRIu64 "\n", url_ftell(pb), pkt->pts);
-        end_ebml_master(pb, mkv->cluster);
-
+    if (!mkv->cluster_pos) {
         ret = mkv_add_seekhead_entry(mkv->cluster_seekhead, MATROSKA_ID_CLUSTER, url_ftell(pb));
         if (ret < 0) return ret;
 
@@ -882,6 +869,14 @@ static int mkv_write_packet(AVFormatContext *s, AVPacket *pkt)
         if (ret < 0) return ret;
     }
 
+    // start a new cluster every 5 MB or 5 sec
+    if (url_ftell(pb) > mkv->cluster_pos + 5*1024*1024 || pkt->pts > mkv->cluster_pts + 5000) {
+        av_log(s, AV_LOG_DEBUG, "Starting new cluster at offset %" PRIu64
+               " bytes, pts %" PRIu64 "\n", url_ftell(pb), pkt->pts);
+        end_ebml_master(pb, mkv->cluster);
+        mkv->cluster_pos = 0;
+    }
+
     mkv->duration = FFMAX(mkv->duration, pkt->pts + duration);
     return 0;
 }
diff --git a/tests/ref/acodec/pcm b/tests/ref/acodec/pcm
index 6c826dd1ba..e9e3fa1610 100644
--- a/tests/ref/acodec/pcm
+++ b/tests/ref/acodec/pcm
@@ -22,11 +22,11 @@ stddev:    0.00 PSNR:999.99 bytes:  1058444/  1058444
 1058444 ./tests/data/acodec/pcm_s16le.wav
 95e54b261530a1bcf6de6fe3b21dc5f6 *./tests/data/pcm.acodec.out.wav
 stddev:    0.00 PSNR:999.99 bytes:  1058444/  1058444
-2f3e0884e96b984436b1d01200d68871 *./tests/data/acodec/pcm_s16be.mkv
+33c64392615a70aa1132b6f87215b8a9 *./tests/data/acodec/pcm_s16be.mkv
 1060705 ./tests/data/acodec/pcm_s16be.mkv
 95e54b261530a1bcf6de6fe3b21dc5f6 *./tests/data/pcm.acodec.out.wav
 stddev:    0.00 PSNR:999.99 bytes:  1058444/  1058444
-c98158732635effefffa006251b9a6c9 *./tests/data/acodec/pcm_s16le.mkv
+ff09423d5ead01dee128b5875682de2a *./tests/data/acodec/pcm_s16le.mkv
 1060705 ./tests/data/acodec/pcm_s16le.mkv
 95e54b261530a1bcf6de6fe3b21dc5f6 *./tests/data/pcm.acodec.out.wav
 stddev:    0.00 PSNR:999.99 bytes:  1058444/  1058444