third_party_ffmpeg/libavcodec/rl2.c
Michael Niedermayer 965e766e48 avcodec/rl2: set dimensions
The dimensions are always 320x200 they are hardcoded in the demuxer.
Hardcode them instead in the decoder.

Fixes: Timeout (16sec -> 400ms)
Fixes: 15574/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_RL2_fuzzer-5158614072819712

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
2019-08-23 22:24:07 +02:00

232 lines
6.4 KiB
C

/*
* RL2 Video Decoder
* Copyright (C) 2008 Sascha Sommer (saschasommer@freenet.de)
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* RL2 Video Decoder
* @author Sascha Sommer (saschasommer@freenet.de)
* @see http://wiki.multimedia.cx/index.php?title=RL2
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "libavutil/internal.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/mem.h"
#include "avcodec.h"
#include "internal.h"
#define EXTRADATA1_SIZE (6 + 256 * 3) ///< video base, clr count, palette
typedef struct Rl2Context {
AVCodecContext *avctx;
uint16_t video_base; ///< initial drawing offset
uint32_t clr_count; ///< number of used colors (currently unused)
uint8_t *back_frame; ///< background frame
uint32_t palette[AVPALETTE_COUNT];
} Rl2Context;
/**
* Run Length Decode a single 320x200 frame
* @param s rl2 context
* @param in input buffer
* @param size input buffer size
* @param out output buffer
* @param stride stride of the output buffer
* @param video_base offset of the rle data inside the frame
*/
static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
uint8_t *out, int stride, int video_base)
{
int base_x = video_base % s->avctx->width;
int base_y = video_base / s->avctx->width;
int stride_adj = stride - s->avctx->width;
int i;
const uint8_t *back_frame = s->back_frame;
const uint8_t *in_end = in + size;
const uint8_t *out_end = out + stride * s->avctx->height;
uint8_t *line_end;
/** copy start of the background frame */
for (i = 0; i <= base_y; i++) {
if (s->back_frame)
memcpy(out, back_frame, s->avctx->width);
out += stride;
back_frame += s->avctx->width;
}
back_frame += base_x - s->avctx->width;
line_end = out - stride_adj;
out += base_x - stride;
/** decode the variable part of the frame */
while (in < in_end) {
uint8_t val = *in++;
int len = 1;
if (val >= 0x80) {
if (in >= in_end)
break;
len = *in++;
if (!len)
break;
}
if (len >= out_end - out)
break;
if (s->back_frame)
val |= 0x80;
else
val &= ~0x80;
while (len--) {
*out++ = (val == 0x80) ? *back_frame : val;
back_frame++;
if (out == line_end) {
out += stride_adj;
line_end += stride;
if (len >= out_end - out)
break;
}
}
}
/** copy the rest from the background frame */
if (s->back_frame) {
while (out < out_end) {
memcpy(out, back_frame, line_end - out);
back_frame += line_end - out;
out = line_end + stride_adj;
line_end += stride;
}
}
}
/**
* Initialize the decoder
* @param avctx decoder context
* @return 0 success, -1 on error
*/
static av_cold int rl2_decode_init(AVCodecContext *avctx)
{
Rl2Context *s = avctx->priv_data;
int back_size;
int i;
int ret;
s->avctx = avctx;
avctx->pix_fmt = AV_PIX_FMT_PAL8;
ret = ff_set_dimensions(avctx, 320, 200);
if (ret < 0)
return ret;
/** parse extra data */
if (!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE) {
av_log(avctx, AV_LOG_ERROR, "invalid extradata size\n");
return AVERROR(EINVAL);
}
/** get frame_offset */
s->video_base = AV_RL16(&avctx->extradata[0]);
s->clr_count = AV_RL32(&avctx->extradata[2]);
if (s->video_base >= avctx->width * avctx->height) {
av_log(avctx, AV_LOG_ERROR, "invalid video_base\n");
return AVERROR_INVALIDDATA;
}
/** initialize palette */
for (i = 0; i < AVPALETTE_COUNT; i++)
s->palette[i] = 0xFFU << 24 | AV_RB24(&avctx->extradata[6 + i * 3]);
/** decode background frame if present */
back_size = avctx->extradata_size - EXTRADATA1_SIZE;
if (back_size > 0) {
uint8_t *back_frame = av_mallocz(avctx->width*avctx->height);
if (!back_frame)
return AVERROR(ENOMEM);
rl2_rle_decode(s, avctx->extradata + EXTRADATA1_SIZE, back_size,
back_frame, avctx->width, 0);
s->back_frame = back_frame;
}
return 0;
}
static int rl2_decode_frame(AVCodecContext *avctx,
void *data, int *got_frame,
AVPacket *avpkt)
{
AVFrame *frame = data;
const uint8_t *buf = avpkt->data;
int ret, buf_size = avpkt->size;
Rl2Context *s = avctx->priv_data;
if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
return ret;
/** run length decode */
rl2_rle_decode(s, buf, buf_size, frame->data[0], frame->linesize[0],
s->video_base);
/** make the palette available on the way out */
memcpy(frame->data[1], s->palette, AVPALETTE_SIZE);
*got_frame = 1;
/** report that the buffer was completely consumed */
return buf_size;
}
/**
* Uninit decoder
* @param avctx decoder context
* @return 0 success, -1 on error
*/
static av_cold int rl2_decode_end(AVCodecContext *avctx)
{
Rl2Context *s = avctx->priv_data;
av_freep(&s->back_frame);
return 0;
}
AVCodec ff_rl2_decoder = {
.name = "rl2",
.long_name = NULL_IF_CONFIG_SMALL("RL2 video"),
.type = AVMEDIA_TYPE_VIDEO,
.id = AV_CODEC_ID_RL2,
.priv_data_size = sizeof(Rl2Context),
.init = rl2_decode_init,
.close = rl2_decode_end,
.decode = rl2_decode_frame,
.capabilities = AV_CODEC_CAP_DR1,
};