diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index 29f0386d46..5f59b6c702 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -85,11 +85,12 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q)
           does depend by this.
         */
         q->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_FIELD_TFF;
-        q->param.mfx.FrameInfo.Height    = FFALIGN(avctx->height, 32);
+        q->height_align = 32;
     } else {
         q->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
-        q->param.mfx.FrameInfo.Height    = FFALIGN(avctx->height, avctx->codec_id == AV_CODEC_ID_HEVC ? 32 : 16);
+        q->height_align = 16;
     }
+   q->param.mfx.FrameInfo.Height    = FFALIGN(avctx->height, q->height_align);
 
     if (avctx->framerate.den > 0 && avctx->framerate.num > 0) {
         q->param.mfx.FrameInfo.FrameRateExtN = avctx->framerate.num;
@@ -323,8 +324,9 @@ static int submit_frame(QSVEncContext *q, const AVFrame *frame,
     }
 
     /* make a copy if the input is not padded as libmfx requires */
-    if (frame->height & 31 || frame->linesize[0] & (q->width_align - 1)) {
-        qf->frame->height = FFALIGN(frame->height, 32);
+    if (     frame->height & (q->height_align - 1) ||
+        frame->linesize[0] & (q->width_align - 1)) {
+        qf->frame->height = FFALIGN(frame->height, q->height_align);
         qf->frame->width  = FFALIGN(frame->width, q->width_align);
 
         ret = ff_get_buffer(q->avctx, qf->frame, AV_GET_BUFFER_FLAG_REF);
diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
index 530754cf1e..8195d9b049 100644
--- a/libavcodec/qsvenc.h
+++ b/libavcodec/qsvenc.h
@@ -43,6 +43,7 @@ typedef struct QSVEncContext {
 
     int packet_size;
     int width_align;
+    int height_align;
 
     mfxVideoParam param;
     mfxFrameAllocRequest req;