diff --git a/libavcodec/dxva2_h264.c b/libavcodec/dxva2_h264.c
index 8518978343..451d8a04cb 100644
--- a/libavcodec/dxva2_h264.c
+++ b/libavcodec/dxva2_h264.c
@@ -240,7 +240,7 @@ static void fill_slice_long(AVCodecContext *avctx, DXVA_Slice_H264_Long *slice,
         unsigned i;
         for (i = 0; i < FF_ARRAY_ELEMS(slice->RefPicList[list]); i++) {
             if (list < sl->list_count && i < sl->ref_count[list]) {
-                const H264Picture *r = &sl->ref_list[list][i];
+                const H264Picture *r = sl->ref_list[list][i].parent;
                 unsigned plane;
                 unsigned index;
                 if (ctx->workaround & FF_DXVA2_WORKAROUND_INTEL_CLEARVIDEO)
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 1974f6611c..18e5a4dd8b 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -77,7 +77,7 @@ static void h264_er_decode_mb(void *opaque, int ref, int mv_dir, int mv_type,
      * practice then correct remapping should be added. */
     if (ref >= sl->ref_count[0])
         ref = 0;
-    if (!sl->ref_list[0][ref].f.data[0]) {
+    if (!sl->ref_list[0][ref].data[0]) {
         av_log(h->avctx, AV_LOG_DEBUG, "Reference not available for error concealing\n");
         ref = 0;
     }
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 5ecff73f54..2c38a246b6 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -333,6 +333,17 @@ typedef struct H264Picture {
     int crop_top;
 } H264Picture;
 
+typedef struct H264Ref {
+    uint8_t *data[3];
+    int linesize[3];
+
+    int reference;
+    int poc;
+    int pic_id;
+
+    H264Picture *parent;
+} H264Ref;
+
 typedef struct H264SliceContext {
     struct H264Context *h264;
     GetBitContext gb;
@@ -430,7 +441,7 @@ typedef struct H264SliceContext {
      */
     unsigned int ref_count[2];          ///< counts frames or fields, depending on current mb mode
     unsigned int list_count;
-    H264Picture ref_list[2][48];        /**< 0..15: frame refs, 16..47: mbaff field refs.
+    H264Ref ref_list[2][48];        /**< 0..15: frame refs, 16..47: mbaff field refs.
                                          *   Reordered version of default_ref_list
                                          *   according to picture reordering in slice header */
     int ref2frm[MAX_SLICES][2][64];     ///< reference to frame number lists, used in the loop filter, the first 2 are for -2,-1
@@ -628,7 +639,7 @@ typedef struct H264Context {
      */
     int max_pic_num;
 
-    H264Picture default_ref_list[2][32]; ///< base reference list for all slices of a coded picture
+    H264Ref default_ref_list[2][32]; ///< base reference list for all slices of a coded picture
     H264Picture *short_ref[32];
     H264Picture *long_ref[32];
     H264Picture *delayed_pic[MAX_DELAYED_PIC_COUNT + 2]; // FIXME size?
diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index 08cab2b849..5756a7ba66 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -39,7 +39,7 @@ static int get_scale_factor(H264SliceContext *sl,
 {
     int poc0 = sl->ref_list[0][i].poc;
     int td = av_clip_int8(poc1 - poc0);
-    if (td == 0 || sl->ref_list[0][i].long_ref) {
+    if (td == 0 || sl->ref_list[0][i].parent->long_ref) {
         return 256;
     } else {
         int tb = av_clip_int8(poc - poc0);
@@ -59,7 +59,7 @@ void ff_h264_direct_dist_scale_factor(const H264Context *const h,
     if (FRAME_MBAFF(h))
         for (field = 0; field < 2; field++) {
             const int poc  = h->cur_pic_ptr->field_poc[field];
-            const int poc1 = sl->ref_list[1][0].field_poc[field];
+            const int poc1 = sl->ref_list[1][0].parent->field_poc[field];
             for (i = 0; i < 2 * sl->ref_count[0]; i++)
                 sl->dist_scale_factor_field[field][i ^ field] =
                     get_scale_factor(sl, poc, poc1, i + 16);
@@ -73,7 +73,7 @@ static void fill_colmap(const H264Context *h, H264SliceContext *sl,
                         int map[2][16 + 32], int list,
                         int field, int colfield, int mbafi)
 {
-    H264Picture *const ref1 = &sl->ref_list[1][0];
+    H264Picture *const ref1 = sl->ref_list[1][0].parent;
     int j, old_ref, rfield;
     int start  = mbafi ? 16                       : 0;
     int end    = mbafi ? 16 + 2 * sl->ref_count[0] : sl->ref_count[0];
@@ -93,7 +93,7 @@ static void fill_colmap(const H264Context *h, H264SliceContext *sl,
                 poc = (poc & ~3) + rfield + 1;
 
             for (j = start; j < end; j++) {
-                if (4 * sl->ref_list[0][j].frame_num +
+                if (4 * sl->ref_list[0][j].parent->frame_num +
                     (sl->ref_list[0][j].reference & 3) == poc) {
                     int cur_ref = mbafi ? (j - 16) ^ field : j;
                     if (ref1->mbaff)
@@ -109,16 +109,16 @@ static void fill_colmap(const H264Context *h, H264SliceContext *sl,
 
 void ff_h264_direct_ref_list_init(const H264Context *const h, H264SliceContext *sl)
 {
-    H264Picture *const ref1 = &sl->ref_list[1][0];
+    H264Ref *const ref1 = &sl->ref_list[1][0];
     H264Picture *const cur = h->cur_pic_ptr;
     int list, j, field;
     int sidx     = (h->picture_structure & 1) ^ 1;
     int ref1sidx = (ref1->reference      & 1) ^ 1;
 
-    for (list = 0; list < 2; list++) {
+    for (list = 0; list < sl->list_count; list++) {
         cur->ref_count[sidx][list] = sl->ref_count[list];
         for (j = 0; j < sl->ref_count[list]; j++)
-            cur->ref_poc[sidx][list][j] = 4 * sl->ref_list[list][j].frame_num +
+            cur->ref_poc[sidx][list][j] = 4 * sl->ref_list[list][j].parent->frame_num +
                                           (sl->ref_list[list][j].reference & 3);
     }
 
@@ -130,16 +130,20 @@ void ff_h264_direct_ref_list_init(const H264Context *const h, H264SliceContext *
     cur->mbaff = FRAME_MBAFF(h);
 
     sl->col_fieldoff = 0;
+
+    if (sl->list_count != 2 || !sl->ref_count[1])
+        return;
+
     if (h->picture_structure == PICT_FRAME) {
         int cur_poc  = h->cur_pic_ptr->poc;
-        int *col_poc = sl->ref_list[1]->field_poc;
+        int *col_poc = sl->ref_list[1][0].parent->field_poc;
         sl->col_parity = (FFABS(col_poc[0] - cur_poc) >=
                           FFABS(col_poc[1] - cur_poc));
         ref1sidx =
         sidx     = sl->col_parity;
     // FL -> FL & differ parity
     } else if (!(h->picture_structure & sl->ref_list[1][0].reference) &&
-               !sl->ref_list[1][0].mbaff) {
+               !sl->ref_list[1][0].parent->mbaff) {
         sl->col_fieldoff = 2 * sl->ref_list[1][0].reference - 3;
     }
 
@@ -155,11 +159,11 @@ void ff_h264_direct_ref_list_init(const H264Context *const h, H264SliceContext *
     }
 }
 
-static void await_reference_mb_row(const H264Context *const h, H264Picture *ref,
+static void await_reference_mb_row(const H264Context *const h, H264Ref *ref,
                                    int mb_y)
 {
     int ref_field         = ref->reference - 1;
-    int ref_field_picture = ref->field_picture;
+    int ref_field_picture = ref->parent->field_picture;
     int ref_height        = 16 * h->mb_height >> ref_field_picture;
 
     if (!HAVE_THREADS || !(h->avctx->active_thread_type & FF_THREAD_FRAME))
@@ -168,7 +172,7 @@ static void await_reference_mb_row(const H264Context *const h, H264Picture *ref,
     /* FIXME: It can be safe to access mb stuff
      * even if pixels aren't deblocked yet. */
 
-    ff_thread_await_progress(&ref->tf,
+    ff_thread_await_progress(&ref->parent->tf,
                              FFMIN(16 * mb_y >> ref_field_picture,
                                    ref_height - 1),
                              ref_field_picture && ref_field);
@@ -261,7 +265,7 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
         return;
     }
 
-    if (IS_INTERLACED(sl->ref_list[1][0].mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
+    if (IS_INTERLACED(sl->ref_list[1][0].parent->mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
         if (!IS_INTERLACED(*mb_type)) {                    //     AFR/FR    -> AFL/FL
             mb_y  = (sl->mb_y & ~1) + sl->col_parity;
             mb_xy = sl->mb_x +
@@ -276,8 +280,8 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
         if (IS_INTERLACED(*mb_type)) {                   // AFL       /FL -> AFR/FR
             mb_y           =  sl->mb_y & ~1;
             mb_xy          = (sl->mb_y & ~1) * h->mb_stride + sl->mb_x;
-            mb_type_col[0] = sl->ref_list[1][0].mb_type[mb_xy];
-            mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy + h->mb_stride];
+            mb_type_col[0] = sl->ref_list[1][0].parent->mb_type[mb_xy];
+            mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy + h->mb_stride];
             b8_stride      = 2 + 4 * h->mb_stride;
             b4_stride     *= 6;
             if (IS_INTERLACED(mb_type_col[0]) !=
@@ -297,7 +301,7 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
         } else {                                         //     AFR/FR    -> AFR/FR
 single_col:
             mb_type_col[0] =
-            mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy];
+            mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy];
 
             sub_mb_type |= MB_TYPE_16x16 | MB_TYPE_DIRECT2; /* B_SUB_8x8 */
             if (!is_b8x8 && (mb_type_col[0] & MB_TYPE_16x16_OR_INTRA)) {
@@ -319,10 +323,10 @@ single_col:
 
     await_reference_mb_row(h, &sl->ref_list[1][0], mb_y);
 
-    l1mv0  = (void*)&sl->ref_list[1][0].motion_val[0][h->mb2b_xy[mb_xy]];
-    l1mv1  = (void*)&sl->ref_list[1][0].motion_val[1][h->mb2b_xy[mb_xy]];
-    l1ref0 = &sl->ref_list[1][0].ref_index[0][4 * mb_xy];
-    l1ref1 = &sl->ref_list[1][0].ref_index[1][4 * mb_xy];
+    l1mv0  = (void*)&sl->ref_list[1][0].parent->motion_val[0][h->mb2b_xy[mb_xy]];
+    l1mv1  = (void*)&sl->ref_list[1][0].parent->motion_val[1][h->mb2b_xy[mb_xy]];
+    l1ref0 = &sl->ref_list[1][0].parent->ref_index[0][4 * mb_xy];
+    l1ref1 = &sl->ref_list[1][0].parent->ref_index[1][4 * mb_xy];
     if (!b8_stride) {
         if (sl->mb_y & 1) {
             l1ref0 += 2;
@@ -349,7 +353,7 @@ single_col:
                            (uint8_t)ref[0], 1);
             fill_rectangle(&sl->ref_cache[1][scan8[i8 * 4]], 2, 2, 8,
                            (uint8_t)ref[1], 1);
-            if (!IS_INTRA(mb_type_col[y8]) && !sl->ref_list[1][0].long_ref &&
+            if (!IS_INTRA(mb_type_col[y8]) && !sl->ref_list[1][0].parent->long_ref &&
                 ((l1ref0[xy8] == 0 &&
                   FFABS(l1mv0[xy4][0]) <= 1 &&
                   FFABS(l1mv0[xy4][1]) <= 1) ||
@@ -380,7 +384,7 @@ single_col:
 
         fill_rectangle(&sl->ref_cache[0][scan8[0]], 4, 4, 8, (uint8_t)ref[0], 1);
         fill_rectangle(&sl->ref_cache[1][scan8[0]], 4, 4, 8, (uint8_t)ref[1], 1);
-        if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].long_ref &&
+        if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].parent->long_ref &&
             ((l1ref0[0] == 0 &&
               FFABS(l1mv0[0][0]) <= 1 &&
               FFABS(l1mv0[0][1]) <= 1) ||
@@ -418,7 +422,7 @@ single_col:
 
             assert(b8_stride == 2);
             /* col_zero_flag */
-            if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].long_ref &&
+            if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].parent->long_ref &&
                 (l1ref0[i8] == 0 ||
                  (l1ref0[i8] < 0 &&
                   l1ref1[i8] == 0 &&
@@ -479,7 +483,7 @@ static void pred_temp_direct_motion(const H264Context *const h, H264SliceContext
     await_reference_mb_row(h, &sl->ref_list[1][0],
                            sl->mb_y + !!IS_INTERLACED(*mb_type));
 
-    if (IS_INTERLACED(sl->ref_list[1][0].mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
+    if (IS_INTERLACED(sl->ref_list[1][0].parent->mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
         if (!IS_INTERLACED(*mb_type)) {                    //     AFR/FR    -> AFL/FL
             mb_y  = (sl->mb_y & ~1) + sl->col_parity;
             mb_xy = sl->mb_x +
@@ -494,8 +498,8 @@ static void pred_temp_direct_motion(const H264Context *const h, H264SliceContext
         if (IS_INTERLACED(*mb_type)) {              // AFL       /FL -> AFR/FR
             mb_y           = sl->mb_y & ~1;
             mb_xy          = sl->mb_x + (sl->mb_y & ~1) * h->mb_stride;
-            mb_type_col[0] = sl->ref_list[1][0].mb_type[mb_xy];
-            mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy + h->mb_stride];
+            mb_type_col[0] = sl->ref_list[1][0].parent->mb_type[mb_xy];
+            mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy + h->mb_stride];
             b8_stride      = 2 + 4 * h->mb_stride;
             b4_stride     *= 6;
             if (IS_INTERLACED(mb_type_col[0]) !=
@@ -518,7 +522,7 @@ static void pred_temp_direct_motion(const H264Context *const h, H264SliceContext
         } else {                                    //     AFR/FR    -> AFR/FR
 single_col:
             mb_type_col[0]     =
-                mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy];
+                mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy];
 
             sub_mb_type = MB_TYPE_16x16 | MB_TYPE_P0L0 | MB_TYPE_P0L1 |
                           MB_TYPE_DIRECT2;                  /* B_SUB_8x8 */
@@ -543,10 +547,10 @@ single_col:
 
     await_reference_mb_row(h, &sl->ref_list[1][0], mb_y);
 
-    l1mv0  = (void*)&sl->ref_list[1][0].motion_val[0][h->mb2b_xy[mb_xy]];
-    l1mv1  = (void*)&sl->ref_list[1][0].motion_val[1][h->mb2b_xy[mb_xy]];
-    l1ref0 = &sl->ref_list[1][0].ref_index[0][4 * mb_xy];
-    l1ref1 = &sl->ref_list[1][0].ref_index[1][4 * mb_xy];
+    l1mv0  = (void*)&sl->ref_list[1][0].parent->motion_val[0][h->mb2b_xy[mb_xy]];
+    l1mv1  = (void*)&sl->ref_list[1][0].parent->motion_val[1][h->mb2b_xy[mb_xy]];
+    l1ref0 = &sl->ref_list[1][0].parent->ref_index[0][4 * mb_xy];
+    l1ref1 = &sl->ref_list[1][0].parent->ref_index[1][4 * mb_xy];
     if (!b8_stride) {
         if (sl->mb_y & 1) {
             l1ref0 += 2;
@@ -567,7 +571,7 @@ single_col:
             map_col_to_list0[1] = sl->map_col_to_list0_field[sl->mb_y & 1][1];
             dist_scale_factor   = sl->dist_scale_factor_field[sl->mb_y & 1];
         }
-        ref_offset = (sl->ref_list[1][0].mbaff << 4) & (mb_type_col[0] >> 3);
+        ref_offset = (sl->ref_list[1][0].parent->mbaff << 4) & (mb_type_col[0] >> 3);
 
         if (IS_INTERLACED(*mb_type) != IS_INTERLACED(mb_type_col[0])) {
             int y_shift = 2 * !IS_INTERLACED(*mb_type);
diff --git a/libavcodec/h264_mb.c b/libavcodec/h264_mb.c
index f2892710d7..b1764a099f 100644
--- a/libavcodec/h264_mb.c
+++ b/libavcodec/h264_mb.c
@@ -60,12 +60,12 @@ static inline void get_lowest_part_y(const H264Context *h, H264SliceContext *sl,
 
     if (list0) {
         int ref_n = sl->ref_cache[0][scan8[n]];
-        H264Picture *ref = &sl->ref_list[0][ref_n];
+        H264Ref *ref = &sl->ref_list[0][ref_n];
 
         // Error resilience puts the current picture in the ref list.
         // Don't try to wait on these as it will cause a deadlock.
         // Fields can wait on each other, though.
-        if (ref->tf.progress->data != h->cur_pic.tf.progress->data ||
+        if (ref->parent->tf.progress->data != h->cur_pic.tf.progress->data ||
             (ref->reference & 3) != h->picture_structure) {
             my = get_lowest_part_list_y(sl, n, height, y_offset, 0);
             if (refs[0][ref_n] < 0)
@@ -76,9 +76,9 @@ static inline void get_lowest_part_y(const H264Context *h, H264SliceContext *sl,
 
     if (list1) {
         int ref_n    = sl->ref_cache[1][scan8[n]];
-        H264Picture *ref = &sl->ref_list[1][ref_n];
+        H264Ref *ref = &sl->ref_list[1][ref_n];
 
-        if (ref->tf.progress->data != h->cur_pic.tf.progress->data ||
+        if (ref->parent->tf.progress->data != h->cur_pic.tf.progress->data ||
             (ref->reference & 3) != h->picture_structure) {
             my = get_lowest_part_list_y(sl, n, height, y_offset, 1);
             if (refs[1][ref_n] < 0)
@@ -167,33 +167,33 @@ static void await_references(const H264Context *h, H264SliceContext *sl)
         for (ref = 0; ref < 48 && nrefs[list]; ref++) {
             int row = refs[list][ref];
             if (row >= 0) {
-                H264Picture *ref_pic  = &sl->ref_list[list][ref];
+                H264Ref *ref_pic  = &sl->ref_list[list][ref];
                 int ref_field         = ref_pic->reference - 1;
-                int ref_field_picture = ref_pic->field_picture;
+                int ref_field_picture = ref_pic->parent->field_picture;
                 int pic_height        = 16 * h->mb_height >> ref_field_picture;
 
                 row <<= MB_MBAFF(sl);
                 nrefs[list]--;
 
                 if (!FIELD_PICTURE(h) && ref_field_picture) { // frame referencing two fields
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN((row >> 1) - !(row & 1),
                                                    pic_height - 1),
                                              1);
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN((row >> 1), pic_height - 1),
                                              0);
                 } else if (FIELD_PICTURE(h) && !ref_field_picture) { // field referencing one field of a frame
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN(row * 2 + ref_field,
                                                    pic_height - 1),
                                              0);
                 } else if (FIELD_PICTURE(h)) {
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN(row, pic_height - 1),
                                              ref_field);
                 } else {
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN(row, pic_height - 1),
                                              0);
                 }
@@ -202,7 +202,7 @@ static void await_references(const H264Context *h, H264SliceContext *sl)
 }
 
 static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext *sl,
-                                         H264Picture *pic,
+                                         H264Ref *pic,
                                          int n, int square, int height,
                                          int delta, int list,
                                          uint8_t *dest_y, uint8_t *dest_cb,
@@ -216,7 +216,7 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
     int my            = sl->mv_cache[list][scan8[n]][1] + src_y_offset * 8;
     const int luma_xy = (mx & 3) + ((my & 3) << 2);
     ptrdiff_t offset  = (mx >> 2) * (1 << pixel_shift) + (my >> 2) * sl->mb_linesize;
-    uint8_t *src_y    = pic->f.data[0] + offset;
+    uint8_t *src_y    = pic->data[0] + offset;
     uint8_t *src_cb, *src_cr;
     int extra_width  = 0;
     int extra_height = 0;
@@ -253,7 +253,7 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
         return;
 
     if (chroma_idc == 3 /* yuv444 */) {
-        src_cb = pic->f.data[1] + offset;
+        src_cb = pic->data[1] + offset;
         if (emu) {
             h->vdsp.emulated_edge_mc(sl->edge_emu_buffer,
                                      src_cb - (2 << pixel_shift) - 2 * sl->mb_linesize,
@@ -267,7 +267,7 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
         if (!square)
             qpix_op[luma_xy](dest_cb + delta, src_cb + delta, sl->mb_linesize);
 
-        src_cr = pic->f.data[2] + offset;
+        src_cr = pic->data[2] + offset;
         if (emu) {
             h->vdsp.emulated_edge_mc(sl->edge_emu_buffer,
                                      src_cr - (2 << pixel_shift) - 2 * sl->mb_linesize,
@@ -290,9 +290,9 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
         emu |= (my >> 3) < 0 || (my >> 3) + 8 >= (pic_height >> 1);
     }
 
-    src_cb = pic->f.data[1] + ((mx >> 3) * (1 << pixel_shift)) +
+    src_cb = pic->data[1] + ((mx >> 3) * (1 << pixel_shift)) +
              (my >> ysh) * sl->mb_uvlinesize;
-    src_cr = pic->f.data[2] + ((mx >> 3) * (1 << pixel_shift)) +
+    src_cr = pic->data[2] + ((mx >> 3) * (1 << pixel_shift)) +
              (my >> ysh) * sl->mb_uvlinesize;
 
     if (emu) {
@@ -348,7 +348,7 @@ static av_always_inline void mc_part_std(const H264Context *h, H264SliceContext
     y_offset += 8 * (sl->mb_y >> MB_FIELD(sl));
 
     if (list0) {
-        H264Picture *ref = &sl->ref_list[0][sl->ref_cache[0][scan8[n]]];
+        H264Ref *ref = &sl->ref_list[0][sl->ref_cache[0][scan8[n]]];
         mc_dir_part(h, sl, ref, n, square, height, delta, 0,
                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
                     qpix_op, chroma_op, pixel_shift, chroma_idc);
@@ -358,7 +358,7 @@ static av_always_inline void mc_part_std(const H264Context *h, H264SliceContext
     }
 
     if (list1) {
-        H264Picture *ref = &sl->ref_list[1][sl->ref_cache[1][scan8[n]]];
+        H264Ref *ref = &sl->ref_list[1][sl->ref_cache[1][scan8[n]]];
         mc_dir_part(h, sl, ref, n, square, height, delta, 1,
                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
                     qpix_op, chroma_op, pixel_shift, chroma_idc);
@@ -455,7 +455,7 @@ static av_always_inline void mc_part_weighted(const H264Context *h, H264SliceCon
     } else {
         int list     = list1 ? 1 : 0;
         int refn     = sl->ref_cache[list][scan8[n]];
-        H264Picture *ref = &sl->ref_list[list][refn];
+        H264Ref *ref = &sl->ref_list[list][refn];
         mc_dir_part(h, sl, ref, n, square, height, delta, list,
                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
                     qpix_put, chroma_put, pixel_shift, chroma_idc);
@@ -489,7 +489,7 @@ static av_always_inline void prefetch_motion(const H264Context *h, H264SliceCont
     if (refn >= 0) {
         const int mx  = (sl->mv_cache[list][scan8[0]][0] >> 2) + 16 * sl->mb_x + 8;
         const int my  = (sl->mv_cache[list][scan8[0]][1] >> 2) + 16 * sl->mb_y;
-        uint8_t **src = sl->ref_list[list][refn].f.data;
+        uint8_t **src = sl->ref_list[list][refn].data;
         int off       =  mx * (1<< pixel_shift) +
                         (my + (sl->mb_x & 3) * 4) * sl->mb_linesize +
                         (64 << pixel_shift);
diff --git a/libavcodec/h264_picture.c b/libavcodec/h264_picture.c
index d63a12e1f2..127bc45c88 100644
--- a/libavcodec/h264_picture.c
+++ b/libavcodec/h264_picture.c
@@ -203,20 +203,23 @@ int ff_h264_field_end(H264Context *h, H264SliceContext *sl, int in_setup)
 
         if (use_last_pic) {
             ff_h264_set_erpic(&sl->er.last_pic, &h->last_pic_for_ec);
-            COPY_PICTURE(&sl->ref_list[0][0], &h->last_pic_for_ec);
+            sl->ref_list[0][0].parent = &h->last_pic_for_ec;
+            memcpy(sl->ref_list[0][0].data, h->last_pic_for_ec.f.data, sizeof(sl->ref_list[0][0].data));
+            memcpy(sl->ref_list[0][0].linesize, h->last_pic_for_ec.f.linesize, sizeof(sl->ref_list[0][0].linesize));
+            sl->ref_list[0][0].reference = h->last_pic_for_ec.reference;
         } else if (sl->ref_count[0]) {
-            ff_h264_set_erpic(&sl->er.last_pic, &sl->ref_list[0][0]);
+            ff_h264_set_erpic(&sl->er.last_pic, sl->ref_list[0][0].parent);
         } else
             ff_h264_set_erpic(&sl->er.last_pic, NULL);
 
         if (sl->ref_count[1])
-            ff_h264_set_erpic(&sl->er.next_pic, &sl->ref_list[1][0]);
+            ff_h264_set_erpic(&sl->er.next_pic, sl->ref_list[1][0].parent);
 
         sl->er.ref_count = sl->ref_count[0];
 
         ff_er_frame_end(&sl->er);
         if (use_last_pic)
-            memset(&sl->ref_list[0][0], 0, sizeof(h->last_pic_for_ec));
+            memset(&sl->ref_list[0][0], 0, sizeof(sl->ref_list[0][0]));
     }
 #endif /* CONFIG_ERROR_RESILIENCE */
 
diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 6c7ec60e68..6d72d2ae19 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -36,24 +36,34 @@
 
 #include <assert.h>
 
-
-static void pic_as_field(H264Picture *pic, const int parity){
+static void pic_as_field(H264Ref *pic, const int parity)
+{
     int i;
-    for (i = 0; i < 4; ++i) {
+    for (i = 0; i < FF_ARRAY_ELEMS(pic->data); ++i) {
         if (parity == PICT_BOTTOM_FIELD)
-            pic->f.data[i] += pic->f.linesize[i];
+            pic->data[i]   += pic->linesize[i];
         pic->reference      = parity;
-        pic->f.linesize[i] *= 2;
+        pic->linesize[i] *= 2;
     }
-    pic->poc= pic->field_poc[parity == PICT_BOTTOM_FIELD];
+    pic->poc = pic->parent->field_poc[parity == PICT_BOTTOM_FIELD];
 }
 
-static int split_field_copy(H264Picture *dest, H264Picture *src, int parity, int id_add)
+static void ref_from_h264pic(H264Ref *dst, H264Picture *src)
+{
+    memcpy(dst->data,     src->f.data,     sizeof(dst->data));
+    memcpy(dst->linesize, src->f.linesize, sizeof(dst->linesize));
+    dst->reference = src->reference;
+    dst->poc       = src->poc;
+    dst->pic_id    = src->pic_id;
+    dst->parent = src;
+}
+
+static int split_field_copy(H264Ref *dest, H264Picture *src, int parity, int id_add)
 {
     int match = !!(src->reference & parity);
 
     if (match) {
-        COPY_PICTURE(dest, src);
+        ref_from_h264pic(dest, src);
         if (parity != PICT_FRAME) {
             pic_as_field(dest, parity);
             dest->pic_id *= 2;
@@ -64,7 +74,7 @@ static int split_field_copy(H264Picture *dest, H264Picture *src, int parity, int
     return match;
 }
 
-static int build_def_list(H264Picture *def, int def_len,
+static int build_def_list(H264Ref *def, int def_len,
                           H264Picture **in, int len, int is_long, int sel)
 {
     int  i[2] = { 0 };
@@ -139,19 +149,16 @@ int ff_h264_fill_default_ref_list(H264Context *h, H264SliceContext *sl)
             av_assert0(len <= 32);
 
             if (len < sl->ref_count[list])
-                memset(&h->default_ref_list[list][len], 0, sizeof(H264Picture) * (sl->ref_count[list] - len));
+                memset(&h->default_ref_list[list][len], 0, sizeof(H264Ref) * (sl->ref_count[list] - len));
             lens[list] = len;
         }
 
         if (lens[0] == lens[1] && lens[1] > 1) {
             for (i = 0; i < lens[0] &&
-                        h->default_ref_list[0][i].f.buf[0]->buffer ==
-                        h->default_ref_list[1][i].f.buf[0]->buffer; i++);
+                        h->default_ref_list[0][i].parent->f.buf[0]->buffer ==
+                        h->default_ref_list[1][i].parent->f.buf[0]->buffer; i++);
             if (i == lens[0]) {
-                H264Picture tmp;
-                COPY_PICTURE(&tmp, &h->default_ref_list[1][0]);
-                COPY_PICTURE(&h->default_ref_list[1][0], &h->default_ref_list[1][1]);
-                COPY_PICTURE(&h->default_ref_list[1][1], &tmp);
+                FFSWAP(H264Ref, h->default_ref_list[1][0], h->default_ref_list[1][1]);
             }
         }
     } else {
@@ -163,7 +170,7 @@ int ff_h264_fill_default_ref_list(H264Context *h, H264SliceContext *sl)
         av_assert0(len <= 32);
 
         if (len < sl->ref_count[0])
-            memset(&h->default_ref_list[0][len], 0, sizeof(H264Picture) * (sl->ref_count[0] - len));
+            memset(&h->default_ref_list[0][len], 0, sizeof(H264Ref) * (sl->ref_count[0] - len));
     }
 #ifdef TRACE
     for (i = 0; i < sl->ref_count[0]; i++) {
@@ -212,14 +219,13 @@ static int pic_num_extract(H264Context *h, int pic_num, int *structure)
 
 int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
 {
-    int list, index, pic_structure, i;
+    int list, index, pic_structure;
 
     print_short_term(h);
     print_long_term(h);
 
     for (list = 0; list < sl->list_count; list++) {
-        for (i = 0; i < sl->ref_count[list]; i++)
-            COPY_PICTURE(&sl->ref_list[list][i], &h->default_ref_list[list][i]);
+        memcpy(sl->ref_list[list], h->default_ref_list[list], sl->ref_count[list] * sizeof(sl->ref_list[0][0]));
 
         if (get_bits1(&sl->gb)) {    // ref_pic_list_modification_flag_l[01]
             int pred = h->curr_pic_num;
@@ -302,17 +308,18 @@ int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
                 if (i < 0) {
                     av_log(h->avctx, AV_LOG_ERROR,
                            "reference picture missing during reorder\n");
-                    memset(&sl->ref_list[list][index], 0, sizeof(H264Picture)); // FIXME
+                    memset(&sl->ref_list[list][index], 0, sizeof(sl->ref_list[0][0])); // FIXME
                 } else {
                     for (i = index; i + 1 < sl->ref_count[list]; i++) {
-                        if (ref->long_ref == sl->ref_list[list][i].long_ref &&
+                        if (sl->ref_list[list][i].parent &&
+                            ref->long_ref == sl->ref_list[list][i].parent->long_ref &&
                             ref->pic_id   == sl->ref_list[list][i].pic_id)
                             break;
                     }
                     for (; i > index; i--) {
-                        COPY_PICTURE(&sl->ref_list[list][i], &sl->ref_list[list][i - 1]);
+                        sl->ref_list[list][i] = sl->ref_list[list][i - 1];
                     }
-                    COPY_PICTURE(&sl->ref_list[list][index], ref);
+                    ref_from_h264pic(&sl->ref_list[list][index], ref);
                     if (FIELD_PICTURE(h)) {
                         pic_as_field(&sl->ref_list[list][index], pic_structure);
                     }
@@ -322,19 +329,19 @@ int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
     }
     for (list = 0; list < sl->list_count; list++) {
         for (index = 0; index < sl->ref_count[list]; index++) {
-            if (   !sl->ref_list[list][index].f.buf[0]
+            if (   !sl->ref_list[list][index].parent
                 || (!FIELD_PICTURE(h) && (sl->ref_list[list][index].reference&3) != 3)) {
                 int i;
                 av_log(h->avctx, AV_LOG_ERROR, "Missing reference picture, default is %d\n", h->default_ref_list[list][0].poc);
                 for (i = 0; i < FF_ARRAY_ELEMS(h->last_pocs); i++)
                     h->last_pocs[i] = INT_MIN;
-                if (h->default_ref_list[list][0].f.buf[0]
+                if (h->default_ref_list[list][0].parent
                     && !(!FIELD_PICTURE(h) && (h->default_ref_list[list][0].reference&3) != 3))
-                    COPY_PICTURE(&sl->ref_list[list][index], &h->default_ref_list[list][0]);
+                    sl->ref_list[list][index] = h->default_ref_list[list][0];
                 else
                     return -1;
             }
-            av_assert0(av_buffer_get_ref_count(sl->ref_list[list][index].f.buf[0]) > 0);
+            av_assert0(av_buffer_get_ref_count(sl->ref_list[list][index].parent->f.buf[0]) > 0);
         }
     }
 
@@ -346,18 +353,22 @@ void ff_h264_fill_mbaff_ref_list(H264Context *h, H264SliceContext *sl)
     int list, i, j;
     for (list = 0; list < sl->list_count; list++) {
         for (i = 0; i < sl->ref_count[list]; i++) {
-            H264Picture *frame = &sl->ref_list[list][i];
-            H264Picture *field = &sl->ref_list[list][16 + 2 * i];
-            COPY_PICTURE(field, frame);
+            H264Ref *frame = &sl->ref_list[list][i];
+            H264Ref *field = &sl->ref_list[list][16 + 2 * i];
+
+            field[0] = *frame;
+
             for (j = 0; j < 3; j++)
-                field[0].f.linesize[j] <<= 1;
+                field[0].linesize[j] <<= 1;
             field[0].reference = PICT_TOP_FIELD;
-            field[0].poc       = field[0].field_poc[0];
-            COPY_PICTURE(field + 1, field);
+            field[0].poc       = field[0].parent->field_poc[0];
+
+            field[1] = field[0];
+
             for (j = 0; j < 3; j++)
-                field[1].f.data[j] += frame->f.linesize[j];
+                field[1].data[j] += frame->parent->f.linesize[j];
             field[1].reference = PICT_BOTTOM_FIELD;
-            field[1].poc       = field[1].field_poc[1];
+            field[1].poc       = field[1].parent->field_poc[1];
 
             sl->luma_weight[16 + 2 * i][list][0] = sl->luma_weight[16 + 2 * i + 1][list][0] = sl->luma_weight[i][list][0];
             sl->luma_weight[16 + 2 * i][list][1] = sl->luma_weight[16 + 2 * i + 1][list][1] = sl->luma_weight[i][list][1];
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 39b3722d45..3af77ad71d 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -893,7 +893,7 @@ static void implicit_weight_table(const H264Context *h, H264SliceContext *sl, in
         int poc0 = sl->ref_list[0][ref0].poc;
         for (ref1 = ref_start; ref1 < ref_count1; ref1++) {
             int w = 32;
-            if (!sl->ref_list[0][ref0].long_ref && !sl->ref_list[1][ref1].long_ref) {
+            if (!sl->ref_list[0][ref0].parent->long_ref && !sl->ref_list[1][ref1].parent->long_ref) {
                 int poc1 = sl->ref_list[1][ref1].poc;
                 int td   = av_clip_int8(poc1 - poc0);
                 if (td) {
@@ -1869,9 +1869,9 @@ int ff_h264_decode_slice_header(H264Context *h, H264SliceContext *sl)
         for (i = 0; i < 16; i++) {
             id_list[i] = 60;
             if (j < sl->list_count && i < sl->ref_count[j] &&
-                sl->ref_list[j][i].f.buf[0]) {
+                sl->ref_list[j][i].parent->f.buf[0]) {
                 int k;
-                AVBuffer *buf = sl->ref_list[j][i].f.buf[0]->buffer;
+                AVBuffer *buf = sl->ref_list[j][i].parent->f.buf[0]->buffer;
                 for (k = 0; k < h->short_ref_count; k++)
                     if (h->short_ref[k]->f.buf[0]->buffer == buf) {
                         id_list[i] = k;
diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c
index 7e01e01960..9794a96036 100644
--- a/libavcodec/vaapi_h264.c
+++ b/libavcodec/vaapi_h264.c
@@ -156,13 +156,13 @@ static int fill_vaapi_ReferenceFrames(VAPictureParameterBufferH264 *pic_param,
  * @param[in]  ref_count   The number of reference pictures in ref_list
  */
 static void fill_vaapi_RefPicList(VAPictureH264 RefPicList[32],
-                                  H264Picture  *ref_list,
+                                  H264Ref  *ref_list,
                                   unsigned int  ref_count)
 {
     unsigned int i, n = 0;
     for (i = 0; i < ref_count; i++)
         if (ref_list[i].reference)
-            fill_vaapi_pic(&RefPicList[n++], &ref_list[i], 0);
+            fill_vaapi_pic(&RefPicList[n++], ref_list[i].parent, 0);
 
     for (; n < 32; n++)
         init_vaapi_pic(&RefPicList[n]);