avfilter/vf_transpose: Fix rounding error

Fixes out of array access
Fixes: asan_heap-oob_7f875d_3482_cov_1818465256_ssudec.mov

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2015-06-30 16:01:15 +02:00
parent 9295ee137a
commit 0083c16605

View File

@ -151,7 +151,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr,
int hsub = plane == 1 || plane == 2 ? trans->hsub : 0; int hsub = plane == 1 || plane == 2 ? trans->hsub : 0;
int vsub = plane == 1 || plane == 2 ? trans->vsub : 0; int vsub = plane == 1 || plane == 2 ? trans->vsub : 0;
int pixstep = trans->pixsteps[plane]; int pixstep = trans->pixsteps[plane];
int inh = in->height >> vsub; int inh = FF_CEIL_RSHIFT(in->height, vsub);
int outw = FF_CEIL_RSHIFT(out->width, hsub); int outw = FF_CEIL_RSHIFT(out->width, hsub);
int outh = FF_CEIL_RSHIFT(out->height, vsub); int outh = FF_CEIL_RSHIFT(out->height, vsub);
int start = (outh * jobnr ) / nb_jobs; int start = (outh * jobnr ) / nb_jobs;