avfilter/vf_edgedetect: fix undefined behaviour

Fixes #8287
This commit is contained in:
Paul B Mahol 2019-10-16 18:06:47 +02:00
parent 7b94b22261
commit 932913f0b4

View File

@ -216,7 +216,7 @@ static int get_rounded_direction(int gx, int gy)
if (gx < 0)
gx = -gx, gy = -gy;
gy <<= 16;
gy *= (1 << 16);
tanpi8gx = 27146 * gx;
tan3pi8gx = 158218 * gx;
if (gy > -tan3pi8gx && gy < -tanpi8gx) return DIRECTION_45UP;