diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 0135348d66..522a9cfddd 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -950,7 +950,7 @@ static void fill_decode_caches(H264Context *h, int mb_type){
     if(IS_INTER(mb_type) || (IS_DIRECT(mb_type) && h->direct_spatial_mv_pred)){
         int list;
         for(list=0; list<h->list_count; list++){
-            if(!USES_LIST(mb_type, list) && !IS_DIRECT(mb_type)){
+            if(!USES_LIST(mb_type, list)){
                 /*if(!h->mv_cache_clean[list]){
                     memset(h->mv_cache [list],  0, 8*5*2*sizeof(int16_t)); //FIXME clean only input? clean at all?
                     memset(h->ref_cache[list], PART_NOT_AVAILABLE, 8*5*sizeof(int8_t));
@@ -1449,7 +1449,7 @@ static void decode_mb_skip(H264Context *h){
     if( h->slice_type_nos == FF_B_TYPE )
     {
         // just for fill_caches. pred_direct_motion will set the real mb_type
-        mb_type|= MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2|MB_TYPE_SKIP;
+        mb_type|= MB_TYPE_L0L1|MB_TYPE_DIRECT2|MB_TYPE_SKIP;
 
         fill_decode_caches(h, mb_type); //FIXME check what is needed and what not ...
         ff_h264_pred_direct_motion(h, &mb_type);
diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index bf944a1f39..cf354c836a 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -159,7 +159,6 @@ static void pred_spatial_direct_motion(H264Context * const h, int *mb_type){
 
 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM)
 
-    *mb_type |= MB_TYPE_L0L1;
 
     /* ref = min(neighbors) */
     for(list=0; list<2; list++){
diff --git a/libavcodec/h264data.h b/libavcodec/h264data.h
index 8f93e42653..101633ad93 100644
--- a/libavcodec/h264data.h
+++ b/libavcodec/h264data.h
@@ -197,7 +197,7 @@ static const PMbInfo p_sub_mb_type_info[4]={
 };
 
 static const PMbInfo b_mb_type_info[23]={
-{MB_TYPE_DIRECT2                                                   , 1, },
+{MB_TYPE_DIRECT2|MB_TYPE_L0L1                                      , 1, },
 {MB_TYPE_16x16|MB_TYPE_P0L0                                       , 1, },
 {MB_TYPE_16x16             |MB_TYPE_P0L1                          , 1, },
 {MB_TYPE_16x16|MB_TYPE_P0L0|MB_TYPE_P0L1                          , 1, },