diff --git a/ffmpeg.c b/ffmpeg.c
index 41c709d681..1a0e909fb0 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -47,6 +47,7 @@
 #include "libavutil/parseutils.h"
 #include "libavutil/samplefmt.h"
 #include "libavutil/fifo.h"
+#include "libavutil/hwcontext.h"
 #include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/dict.h"
@@ -3421,7 +3422,9 @@ static int init_output_stream(OutputStream *ost, char *error, int error_len)
             !av_dict_get(ost->encoder_opts, "ab", NULL, 0))
             av_dict_set(&ost->encoder_opts, "b", "128000", 0);
 
-        if (ost->filter && av_buffersink_get_hw_frames_ctx(ost->filter->filter)) {
+        if (ost->filter && av_buffersink_get_hw_frames_ctx(ost->filter->filter) &&
+            ((AVHWFramesContext*)av_buffersink_get_hw_frames_ctx(ost->filter->filter)->data)->format ==
+            av_buffersink_get_format(ost->filter->filter)) {
             ost->enc_ctx->hw_frames_ctx = av_buffer_ref(av_buffersink_get_hw_frames_ctx(ost->filter->filter));
             if (!ost->enc_ctx->hw_frames_ctx)
                 return AVERROR(ENOMEM);
diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
index b431990edc..ecfb872ed8 100644
--- a/libavfilter/avfilter.c
+++ b/libavfilter/avfilter.c
@@ -355,14 +355,12 @@ int avfilter_config_links(AVFilterContext *filter)
             }
 
             if (link->src->nb_inputs && link->src->inputs[0]->hw_frames_ctx &&
-                !link->hw_frames_ctx) {
-                AVHWFramesContext *input_ctx = (AVHWFramesContext*)link->src->inputs[0]->hw_frames_ctx->data;
-
-                if (input_ctx->format == link->format) {
-                    link->hw_frames_ctx = av_buffer_ref(link->src->inputs[0]->hw_frames_ctx);
-                    if (!link->hw_frames_ctx)
-                        return AVERROR(ENOMEM);
-                }
+                !(link->src->filter->flags_internal & FF_FILTER_FLAG_HWFRAME_AWARE)) {
+                av_assert0(!link->hw_frames_ctx &&
+                           "should not be set by non-hwframe-aware filter");
+                link->hw_frames_ctx = av_buffer_ref(link->src->inputs[0]->hw_frames_ctx);
+                if (!link->hw_frames_ctx)
+                    return AVERROR(ENOMEM);
             }
 
             if ((config_link = link->dstpad->config_props))
diff --git a/libavfilter/avfilter.h b/libavfilter/avfilter.h
index ac6dca4fc0..60662c19ac 100644
--- a/libavfilter/avfilter.h
+++ b/libavfilter/avfilter.h
@@ -268,6 +268,8 @@ typedef struct AVFilter {
 
     int priv_size;      ///< size of private data to allocate for the filter
 
+    int flags_internal; ///< Additional flags for avfilter internal use only.
+
     /**
      * Used by the filter registration system. Must not be touched by any other
      * code.
diff --git a/libavfilter/internal.h b/libavfilter/internal.h
index 460d75eb83..1070f18376 100644
--- a/libavfilter/internal.h
+++ b/libavfilter/internal.h
@@ -386,6 +386,12 @@ int ff_filter_activate(AVFilterContext *filter);
  */
 void ff_filter_graph_remove_filter(AVFilterGraph *graph, AVFilterContext *filter);
 
+/**
+ * The filter is aware of hardware frames, and any hardware frame context
+ * should not be automatically propagated through it.
+ */
+#define FF_FILTER_FLAG_HWFRAME_AWARE (1 << 0)
+
 /**
  * Run one round of processing on a filter graph.
  */
diff --git a/libavfilter/vf_deinterlace_qsv.c b/libavfilter/vf_deinterlace_qsv.c
index 2fe74c1a75..bd43ef7bba 100644
--- a/libavfilter/vf_deinterlace_qsv.c
+++ b/libavfilter/vf_deinterlace_qsv.c
@@ -570,4 +570,6 @@ AVFilter ff_vf_deinterlace_qsv = {
 
     .inputs    = qsvdeint_inputs,
     .outputs   = qsvdeint_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };
diff --git a/libavfilter/vf_hwdownload.c b/libavfilter/vf_hwdownload.c
index f3138f366a..33af30cf40 100644
--- a/libavfilter/vf_hwdownload.c
+++ b/libavfilter/vf_hwdownload.c
@@ -214,4 +214,5 @@ AVFilter ff_vf_hwdownload = {
     .priv_class    = &hwdownload_class,
     .inputs        = hwdownload_inputs,
     .outputs       = hwdownload_outputs,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };
diff --git a/libavfilter/vf_hwupload.c b/libavfilter/vf_hwupload.c
index 9237253f23..157686b7b3 100644
--- a/libavfilter/vf_hwupload.c
+++ b/libavfilter/vf_hwupload.c
@@ -231,4 +231,5 @@ AVFilter ff_vf_hwupload = {
     .priv_class    = &hwupload_class,
     .inputs        = hwupload_inputs,
     .outputs       = hwupload_outputs,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };
diff --git a/libavfilter/vf_hwupload_cuda.c b/libavfilter/vf_hwupload_cuda.c
index 1e47ada242..ef98233d12 100644
--- a/libavfilter/vf_hwupload_cuda.c
+++ b/libavfilter/vf_hwupload_cuda.c
@@ -187,4 +187,6 @@ AVFilter ff_vf_hwupload_cuda = {
 
     .inputs    = cudaupload_inputs,
     .outputs   = cudaupload_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };
diff --git a/libavfilter/vf_scale_npp.c b/libavfilter/vf_scale_npp.c
index 4b45174742..b5acce653b 100644
--- a/libavfilter/vf_scale_npp.c
+++ b/libavfilter/vf_scale_npp.c
@@ -592,4 +592,6 @@ AVFilter ff_vf_scale_npp = {
 
     .inputs    = nppscale_inputs,
     .outputs   = nppscale_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };
diff --git a/libavfilter/vf_scale_qsv.c b/libavfilter/vf_scale_qsv.c
index 52e3ef9c95..8b262b68af 100644
--- a/libavfilter/vf_scale_qsv.c
+++ b/libavfilter/vf_scale_qsv.c
@@ -628,4 +628,6 @@ AVFilter ff_vf_scale_qsv = {
 
     .inputs    = qsvscale_inputs,
     .outputs   = qsvscale_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };
diff --git a/libavfilter/vf_scale_vaapi.c b/libavfilter/vf_scale_vaapi.c
index c4334c7664..5f049a5d7b 100644
--- a/libavfilter/vf_scale_vaapi.c
+++ b/libavfilter/vf_scale_vaapi.c
@@ -477,4 +477,5 @@ AVFilter ff_vf_scale_vaapi = {
     .inputs        = scale_vaapi_inputs,
     .outputs       = scale_vaapi_outputs,
     .priv_class    = &scale_vaapi_class,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };