diff --git a/libavfilter/vf_colormatrix.c b/libavfilter/vf_colormatrix.c
index e1b48fa1c3..6b38763d45 100644
--- a/libavfilter/vf_colormatrix.c
+++ b/libavfilter/vf_colormatrix.c
@@ -164,8 +164,8 @@ static av_cold int init(AVFilterContext *ctx)
 {
     ColorMatrixContext *color = ctx->priv;
 
-    if (color->source == COLOR_MODE_NONE || color->dest == COLOR_MODE_NONE) {
-        av_log(ctx, AV_LOG_ERROR, "Unspecified source or destination color space\n");
+    if (color->dest == COLOR_MODE_NONE) {
+        av_log(ctx, AV_LOG_ERROR, "Unspecified destination color space\n");
         return AVERROR(EINVAL);
     }
 
@@ -174,10 +174,6 @@ static av_cold int init(AVFilterContext *ctx)
         return AVERROR(EINVAL);
     }
 
-    color->mode = color->source * 4 + color->dest;
-
-    calc_coefficients(ctx);
-
     return 0;
 }
 
@@ -346,6 +342,25 @@ static int filter_frame(AVFilterLink *link, AVFrame *in)
     }
     av_frame_copy_props(out, in);
 
+    if (color->source == COLOR_MODE_NONE) {
+        enum AVColorSpace cs = av_frame_get_colorspace(in);
+        enum ColorMode source;
+
+        switch(cs) {
+        case AVCOL_SPC_BT709     : source = COLOR_MODE_BT709     ; break;
+        case AVCOL_SPC_FCC       : source = COLOR_MODE_FCC       ; break;
+        case AVCOL_SPC_SMPTE240M : source = COLOR_MODE_SMPTE240M ; break;
+        case AVCOL_SPC_BT470BG   : source = COLOR_MODE_BT601     ; break;
+        default :
+            av_log(ctx, AV_LOG_ERROR, "Input frame does not specify a supported colorspace, and none has been specified as source either\n");
+            return AVERROR(EINVAL);
+        }
+        color->mode = source * 4 + color->dest;
+    } else
+        color->mode = color->source * 4 + color->dest;
+
+    calc_coefficients(ctx);
+
     if (in->format == AV_PIX_FMT_YUV422P)
         process_frame_yuv422p(color, out, in);
     else if (in->format == AV_PIX_FMT_YUV420P)