diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index cc6e01861f..fc429aee80 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -89,7 +89,8 @@ static void fill_colmap(H264Context *h, int map[2][16+32], int list, int field,
             for(j=start; j<end; j++){
                 if (4 * h->ref_list[0][j].frame_num + (h->ref_list[0][j].f.reference & 3) == poc) {
                     int cur_ref= mbafi ? (j-16)^field : j;
-                    map[list][2*old_ref + (rfield^field) + 16] = cur_ref;
+                    if (ref1->mbaff)
+                        map[list][2 * old_ref + (rfield^field) + 16] = cur_ref;
                     if(rfield == field || !interl)
                         map[list][old_ref] = cur_ref;
                     break;