xenia-project/FFmpeg
1
0
Fork 0
mirror of https://github.com/xenia-project/FFmpeg.git synced 2024-11-28 22:10:34 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

h264: disallow constrained intra prediction modes for luma.

Browse Source 
Conversion of the luma intra prediction mode to one of the constrained
("alzheimer") ones can happen by crafting special bitstreams, causing
a crash because we'll call a NULL function pointer for 16x16 block intra
prediction, since constrained intra prediction functions are only
implemented for chroma (8x8 blocks).

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
This commit is contained in:
Ronald S. Bultje 2012-02-09 22:57:01 -08:00
parent 81749f30cd
commit 45b7bd7c53
5 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libavcodec/h264.c
View File

@ -105,7 +105,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h){
* Check if the top & left blocks are available if needed and * Check if the top & left blocks are available if needed and
* change the dc mode so it only uses the available blocks. * change the dc mode so it only uses the available blocks.
*/ */
int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma){
MpegEncContext * const s = &h->s; MpegEncContext * const s = &h->s;
static const int8_t top [7]= {LEFT_DC_PRED8x8, 1,-1,-1}; static const int8_t top [7]= {LEFT_DC_PRED8x8, 1,-1,-1};
static const int8_t left[7]= { TOP_DC_PRED8x8,-1, 2,-1,DC_128_PRED8x8}; static const int8_t left[7]= { TOP_DC_PRED8x8,-1, 2,-1,DC_128_PRED8x8};
@ -125,7 +125,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode){
if((h->left_samples_available&0x8080) != 0x8080){ if((h->left_samples_available&0x8080) != 0x8080){
mode= left[ mode ]; mode= left[ mode ];
if(h->left_samples_available&0x8080){ //mad cow disease mode, aka MBAFF + constrained_intra_pred if(is_chroma && (h->left_samples_available&0x8080)){ //mad cow disease mode, aka MBAFF + constrained_intra_pred
mode= ALZHEIMER_DC_L0T_PRED8x8 + (!(h->left_samples_available&0x8000)) + 2*(mode == DC_128_PRED8x8); mode= ALZHEIMER_DC_L0T_PRED8x8 + (!(h->left_samples_available&0x8000)) + 2*(mode == DC_128_PRED8x8);
} }
if(mode<0){ if(mode<0){

2
libavcodec/h264.h
View File

@ -657,7 +657,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h);
/** /**
* Check if the top & left blocks are available if needed & change the dc mode so it only uses the available blocks. * Check if the top & left blocks are available if needed & change the dc mode so it only uses the available blocks.
*/ */
int ff_h264_check_intra_pred_mode(H264Context *h, int mode); int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma);
void ff_h264_hl_decode_mb(H264Context *h); void ff_h264_hl_decode_mb(H264Context *h);
int ff_h264_frame_start(H264Context *h); int ff_h264_frame_start(H264Context *h);

4
libavcodec/h264_cabac.c
View File

@ -2040,14 +2040,14 @@ decode_intra_mb:
write_back_intra_pred_mode(h); write_back_intra_pred_mode(h);
if( ff_h264_check_intra4x4_pred_mode(h) < 0 ) return -1; if( ff_h264_check_intra4x4_pred_mode(h) < 0 ) return -1;
} else { } else {
h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode ); h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode, 0 );
if( h->intra16x16_pred_mode < 0 ) return -1; if( h->intra16x16_pred_mode < 0 ) return -1;
} }
if(decode_chroma){ if(decode_chroma){
h->chroma_pred_mode_table[mb_xy] = h->chroma_pred_mode_table[mb_xy] =
pred_mode = decode_cabac_mb_chroma_pre_mode( h ); pred_mode = decode_cabac_mb_chroma_pre_mode( h );
pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode ); pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode, 1 );
if( pred_mode < 0 ) return -1; if( pred_mode < 0 ) return -1;
h->chroma_pred_mode= pred_mode; h->chroma_pred_mode= pred_mode;
} else { } else {

4
libavcodec/h264_cavlc.c
View File

@ -822,12 +822,12 @@ decode_intra_mb:
if( ff_h264_check_intra4x4_pred_mode(h) < 0) if( ff_h264_check_intra4x4_pred_mode(h) < 0)
return -1; return -1;
}else{ }else{
h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode); h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode, 0);
if(h->intra16x16_pred_mode < 0) if(h->intra16x16_pred_mode < 0)
return -1; return -1;
} }
if(decode_chroma){ if(decode_chroma){
pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb)); pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb), 1);
if(pred_mode < 0) if(pred_mode < 0)
return -1; return -1;
h->chroma_pred_mode= pred_mode; h->chroma_pred_mode= pred_mode;

4
libavcodec/svq3.c
View File

@ -612,7 +612,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type)
dir = i_mb_type_info[mb_type - 8].pred_mode; dir = i_mb_type_info[mb_type - 8].pred_mode;
dir = (dir >> 1) ^ 3*(dir & 1) ^ 1; dir = (dir >> 1) ^ 3*(dir & 1) ^ 1;
if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir)) == -1){ if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1){
av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n"); av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n");
return -1; return -1;
} }
@ -711,7 +711,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type)
s->current_picture.f.mb_type[mb_xy] = mb_type; s->current_picture.f.mb_type[mb_xy] = mb_type;
if (IS_INTRA(mb_type)) { if (IS_INTRA(mb_type)) {
h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8); h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8, 1);
} }
return 0; return 0;