From 6781b5315e5c12c0b223f96f0bf17c3291dc9c29 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 07:45:05 +0100 Subject: [PATCH 1/3] rl2: cosmetics, reformat --- libavcodec/rl2.c | 82 +++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/libavcodec/rl2.c b/libavcodec/rl2.c index a368e6b470..9973c28736 100644 --- a/libavcodec/rl2.c +++ b/libavcodec/rl2.c @@ -66,60 +66,60 @@ static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size, int stride_adj = stride - s->avctx->width; int i; const uint8_t *back_frame = s->back_frame; - const uint8_t *in_end = in + size; - const uint8_t *out_end = out + stride * s->avctx->height; + const uint8_t *in_end = in + size; + const uint8_t *out_end = out + stride * s->avctx->height; uint8_t *line_end; /** copy start of the background frame */ - for(i=0;i<=base_y;i++){ - if(s->back_frame) - memcpy(out,back_frame,s->avctx->width); - out += stride; + for (i = 0; i <= base_y; i++) { + if (s->back_frame) + memcpy(out, back_frame, s->avctx->width); + out += stride; back_frame += s->avctx->width; } back_frame += base_x - s->avctx->width; - line_end = out - stride_adj; - out += base_x - stride; + line_end = out - stride_adj; + out += base_x - stride; /** decode the variable part of the frame */ - while(in < in_end){ + while (in < in_end) { uint8_t val = *in++; - int len = 1; - if(val >= 0x80){ - if(in >= in_end) + int len = 1; + if (val >= 0x80) { + if (in >= in_end) break; len = *in++; - if(!len) + if (!len) break; } - if(len >= out_end - out) + if (len >= out_end - out) break; - if(s->back_frame) + if (s->back_frame) val |= 0x80; else val &= ~0x80; - while(len--){ - *out++ = (val == 0x80)? *back_frame:val; + while (len--) { + *out++ = (val == 0x80) ? *back_frame : val; back_frame++; - if(out == line_end){ - out += stride_adj; + if (out == line_end) { + out += stride_adj; line_end += stride; - if(len >= out_end - out) + if (len >= out_end - out) break; } } } /** copy the rest from the background frame */ - if(s->back_frame){ - while(out < out_end){ + if (s->back_frame) { + while (out < out_end) { memcpy(out, back_frame, line_end - out); back_frame += line_end - out; - out = line_end + stride_adj; - line_end += stride; + out = line_end + stride_adj; + line_end += stride; } } } @@ -135,37 +135,38 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx) Rl2Context *s = avctx->priv_data; int back_size; int i; - s->avctx = avctx; + + s->avctx = avctx; avctx->pix_fmt = AV_PIX_FMT_PAL8; /** parse extra data */ - if(!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE){ + if (!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE) { av_log(avctx, AV_LOG_ERROR, "invalid extradata size\n"); return AVERROR(EINVAL); } /** get frame_offset */ s->video_base = AV_RL16(&avctx->extradata[0]); - s->clr_count = AV_RL32(&avctx->extradata[2]); + s->clr_count = AV_RL32(&avctx->extradata[2]); - if(s->video_base >= avctx->width * avctx->height){ + if (s->video_base >= avctx->width * avctx->height) { av_log(avctx, AV_LOG_ERROR, "invalid video_base\n"); return AVERROR_INVALIDDATA; } /** initialize palette */ - for(i=0;ipalette[i] = AV_RB24(&avctx->extradata[6 + i * 3]); /** decode background frame if present */ back_size = avctx->extradata_size - EXTRADATA1_SIZE; - if(back_size > 0){ + if (back_size > 0) { uint8_t *back_frame = av_mallocz(avctx->width*avctx->height); - if(!back_frame) + if (!back_frame) return AVERROR(ENOMEM); - rl2_rle_decode(s,avctx->extradata + EXTRADATA1_SIZE,back_size, - back_frame,avctx->width,0); + rl2_rle_decode(s, avctx->extradata + EXTRADATA1_SIZE, back_size, + back_frame, avctx->width, 0); s->back_frame = back_frame; } return 0; @@ -173,25 +174,26 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx) static int rl2_decode_frame(AVCodecContext *avctx, - void *data, int *got_frame, - AVPacket *avpkt) + void *data, int *got_frame, + AVPacket *avpkt) { const uint8_t *buf = avpkt->data; - int ret, buf_size = avpkt->size; + int ret, buf_size = avpkt->size; Rl2Context *s = avctx->priv_data; - if(s->frame.data[0]) + if (s->frame.data[0]) avctx->release_buffer(avctx, &s->frame); /** get buffer */ - s->frame.reference= 0; + s->frame.reference = 0; if ((ret = ff_get_buffer(avctx, &s->frame)) < 0) { av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed\n"); return ret; } /** run length decode */ - rl2_rle_decode(s,buf,buf_size,s->frame.data[0],s->frame.linesize[0],s->video_base); + rl2_rle_decode(s, buf, buf_size, s->frame.data[0], s->frame.linesize[0], + s->video_base); /** make the palette available on the way out */ memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE); @@ -213,7 +215,7 @@ static av_cold int rl2_decode_end(AVCodecContext *avctx) { Rl2Context *s = avctx->priv_data; - if(s->frame.data[0]) + if (s->frame.data[0]) avctx->release_buffer(avctx, &s->frame); av_free(s->back_frame); From 6ea2c9a4cf9fb0abd085825724e54b691f773251 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 08:06:19 +0100 Subject: [PATCH 2/3] indeo2: return meaningful error codes --- libavcodec/indeo2.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libavcodec/indeo2.c b/libavcodec/indeo2.c index 1dd151dfbe..83e1f604b2 100644 --- a/libavcodec/indeo2.c +++ b/libavcodec/indeo2.c @@ -57,7 +57,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst int t; if(width&1) - return -1; + return AVERROR_INVALIDDATA; /* first line contain absolute values, other lines contain deltas */ while (out < width){ @@ -65,7 +65,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst if(c >= 0x80) { /* we have a run */ c -= 0x7F; if(out + c*2 > width) - return -1; + return AVERROR_INVALIDDATA; for (i = 0; i < c * 2; i++) dst[out++] = 0x80; } else { /* copy two values from table */ @@ -82,7 +82,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst if(c >= 0x80) { /* we have a skip */ c -= 0x7F; if(out + c*2 > width) - return -1; + return AVERROR_INVALIDDATA; for (i = 0; i < c * 2; i++) { dst[out] = dst[out - stride]; out++; @@ -112,7 +112,7 @@ static int ir2_decode_plane_inter(Ir2Context *ctx, int width, int height, uint8_ int t; if(width&1) - return -1; + return AVERROR_INVALIDDATA; for (j = 0; j < height; j++){ out = 0; @@ -146,16 +146,16 @@ static int ir2_decode_frame(AVCodecContext *avctx, Ir2Context * const s = avctx->priv_data; AVFrame *picture = data; AVFrame * const p = &s->picture; - int start; + int start, ret; if(p->data[0]) avctx->release_buffer(avctx, p); p->reference = 1; p->buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (avctx->reget_buffer(avctx, p)) { + if ((ret = avctx->reget_buffer(avctx, p)) < 0) { av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); - return -1; + return ret; } start = 48; /* hardcoded for now */ From 7b1fbd4729a52dd7c02622dbe7bb81a6a7ed12f8 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 08:08:40 +0100 Subject: [PATCH 3/3] indeo2: check decoding errors. --- libavcodec/indeo2.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/libavcodec/indeo2.c b/libavcodec/indeo2.c index 83e1f604b2..46d582f7af 100644 --- a/libavcodec/indeo2.c +++ b/libavcodec/indeo2.c @@ -176,21 +176,34 @@ static int ir2_decode_frame(AVCodecContext *avctx, init_get_bits(&s->gb, buf + start, (buf_size - start) * 8); if (s->decode_delta) { /* intraframe */ - ir2_decode_plane(s, avctx->width, avctx->height, - s->picture.data[0], s->picture.linesize[0], ir2_luma_table); + if ((ret = ir2_decode_plane(s, avctx->width, avctx->height, + s->picture.data[0], s->picture.linesize[0], + ir2_luma_table)) < 0) + return ret; + /* swapped U and V */ - ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2, - s->picture.data[2], s->picture.linesize[2], ir2_luma_table); - ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2, - s->picture.data[1], s->picture.linesize[1], ir2_luma_table); + if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2, + s->picture.data[2], s->picture.linesize[2], + ir2_luma_table)) < 0) + return ret; + if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2, + s->picture.data[1], s->picture.linesize[1], + ir2_luma_table)) < 0) + return ret; } else { /* interframe */ - ir2_decode_plane_inter(s, avctx->width, avctx->height, - s->picture.data[0], s->picture.linesize[0], ir2_luma_table); + if ((ret = ir2_decode_plane_inter(s, avctx->width, avctx->height, + s->picture.data[0], s->picture.linesize[0], + ir2_luma_table)) < 0) + return ret; /* swapped U and V */ - ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2, - s->picture.data[2], s->picture.linesize[2], ir2_luma_table); - ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2, - s->picture.data[1], s->picture.linesize[1], ir2_luma_table); + if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2, + s->picture.data[2], s->picture.linesize[2], + ir2_luma_table)) < 0) + return ret; + if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2, + s->picture.data[1], s->picture.linesize[1], + ir2_luma_table)) < 0) + return ret; } *picture = s->picture;