From 3cec54b7d72bc594b28faa7b8fb6683ef397fe66 Mon Sep 17 00:00:00 2001 From: James Almer Date: Thu, 13 Nov 2014 13:47:55 -0300 Subject: [PATCH] x86/flacdsp: add SSE2 and AVX decorrelate functions Two to four times faster depending on instruction set, block size and channel count. --- libavcodec/arm/flacdsp_init_arm.c | 2 +- libavcodec/flacdec.c | 6 +- libavcodec/flacdsp.c | 6 +- libavcodec/flacdsp.h | 6 +- libavcodec/flacenc.c | 2 +- libavcodec/x86/flacdsp.asm | 255 ++++++++++++++++++++++++++++++ libavcodec/x86/flacdsp_init.c | 63 +++++++- 7 files changed, 328 insertions(+), 12 deletions(-) diff --git a/libavcodec/arm/flacdsp_init_arm.c b/libavcodec/arm/flacdsp_init_arm.c index 9ddb26829d..df1b19c1e8 100644 --- a/libavcodec/arm/flacdsp_init_arm.c +++ b/libavcodec/arm/flacdsp_init_arm.c @@ -24,7 +24,7 @@ void ff_flac_lpc_16_arm(int32_t *samples, const int coeffs[32], int order, int qlevel, int len); -av_cold void ff_flacdsp_init_arm(FLACDSPContext *c, enum AVSampleFormat fmt, +av_cold void ff_flacdsp_init_arm(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps) { if (bps <= 16 && CONFIG_FLAC_DECODER) diff --git a/libavcodec/flacdec.c b/libavcodec/flacdec.c index f1316658e7..9b1e4280b6 100644 --- a/libavcodec/flacdec.c +++ b/libavcodec/flacdec.c @@ -111,7 +111,7 @@ static av_cold int flac_decode_init(AVCodecContext *avctx) if (ret < 0) return ret; flac_set_bps(s); - ff_flacdsp_init(&s->dsp, avctx->sample_fmt, s->bps); + ff_flacdsp_init(&s->dsp, avctx->sample_fmt, s->channels, s->bps); s->got_streaminfo = 1; return 0; @@ -173,7 +173,7 @@ static int parse_streaminfo(FLACContext *s, const uint8_t *buf, int buf_size) if (ret < 0) return ret; flac_set_bps(s); - ff_flacdsp_init(&s->dsp, s->avctx->sample_fmt, s->bps); + ff_flacdsp_init(&s->dsp, s->avctx->sample_fmt, s->channels, s->bps); s->got_streaminfo = 1; return 0; @@ -472,7 +472,7 @@ static int decode_frame(FLACContext *s) ret = allocate_buffers(s); if (ret < 0) return ret; - ff_flacdsp_init(&s->dsp, s->avctx->sample_fmt, s->bps); + ff_flacdsp_init(&s->dsp, s->avctx->sample_fmt, s->channels, s->bps); s->got_streaminfo = 1; dump_headers(s->avctx, (FLACStreaminfo *)s); } diff --git a/libavcodec/flacdsp.c b/libavcodec/flacdsp.c index b15bc7476b..a83eb833b5 100644 --- a/libavcodec/flacdsp.c +++ b/libavcodec/flacdsp.c @@ -85,7 +85,7 @@ static void flac_lpc_32_c(int32_t *decoded, const int coeffs[32], } -av_cold void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, +av_cold void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps) { if (bps > 16) { @@ -127,7 +127,7 @@ av_cold void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, } if (ARCH_ARM) - ff_flacdsp_init_arm(c, fmt, bps); + ff_flacdsp_init_arm(c, fmt, channels, bps); if (ARCH_X86) - ff_flacdsp_init_x86(c, fmt, bps); + ff_flacdsp_init_x86(c, fmt, channels, bps); } diff --git a/libavcodec/flacdsp.h b/libavcodec/flacdsp.h index 14f3466b0b..417381cc70 100644 --- a/libavcodec/flacdsp.h +++ b/libavcodec/flacdsp.h @@ -31,8 +31,8 @@ typedef struct FLACDSPContext { const int32_t coefs[32], int shift); } FLACDSPContext; -void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, int bps); -void ff_flacdsp_init_arm(FLACDSPContext *c, enum AVSampleFormat fmt, int bps); -void ff_flacdsp_init_x86(FLACDSPContext *c, enum AVSampleFormat fmt, int bps); +void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps); +void ff_flacdsp_init_arm(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps); +void ff_flacdsp_init_x86(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps); #endif /* AVCODEC_FLACDSP_H */ diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c index 3b72888966..e66ef3db5e 100644 --- a/libavcodec/flacenc.c +++ b/libavcodec/flacenc.c @@ -428,7 +428,7 @@ static av_cold int flac_encode_init(AVCodecContext *avctx) s->options.max_prediction_order, FF_LPC_TYPE_LEVINSON); ff_bswapdsp_init(&s->bdsp); - ff_flacdsp_init(&s->flac_dsp, avctx->sample_fmt, + ff_flacdsp_init(&s->flac_dsp, avctx->sample_fmt, channels, avctx->bits_per_raw_sample); dprint_compression_options(s); diff --git a/libavcodec/x86/flacdsp.asm b/libavcodec/x86/flacdsp.asm index 37ee87b163..d60a6e4a06 100644 --- a/libavcodec/x86/flacdsp.asm +++ b/libavcodec/x86/flacdsp.asm @@ -2,6 +2,7 @@ ;* FLAC DSP SIMD optimizations ;* ;* Copyright (C) 2014 Loren Merritt +;* Copyright (C) 2014 James Almer ;* ;* This file is part of FFmpeg. ;* @@ -72,3 +73,257 @@ ALIGN 16 LPC_32 xop %endif LPC_32 sse4 + +;---------------------------------------------------------------------------------- +;void ff_flac_decorrelate_[lrm]s_16_sse2(uint8_t **out, int32_t **in, int channels, +; int len, int shift); +;---------------------------------------------------------------------------------- +%macro FLAC_DECORRELATE_16 3-4 +cglobal flac_decorrelate_%1_16, 2, 4, 4, out, in0, in1, len +%if ARCH_X86_32 || WIN64 + movd m3, r4m +%if ARCH_X86_32 + mov lend, lenm +%endif +%else ; UNIX64 + movd m3, r4d +%endif + shl lend, 2 + mov in1q, [in0q + gprsize] + mov in0q, [in0q] + mov outq, [outq] + add in1q, lenq + add in0q, lenq + add outq, lenq + neg lenq + +align 16 +.loop: + mova m0, [in0q + lenq] + mova m1, [in1q + lenq] +%ifidn %1, ms + psrad m2, m1, 1 + psubd m0, m2 +%endif +%ifnidn %1, indep2 + p%4d m2, m0, m1 +%endif + packssdw m%2, m%2 + packssdw m%3, m%3 + punpcklwd m%2, m%3 + psllw m%2, m3 + mova [outq + lenq], m%2 + add lenq, 16 + jl .loop + REP_RET +%endmacro + +INIT_XMM sse2 +FLAC_DECORRELATE_16 ls, 0, 2, sub +FLAC_DECORRELATE_16 rs, 2, 1, add +FLAC_DECORRELATE_16 ms, 2, 0, add + +;---------------------------------------------------------------------------------- +;void ff_flac_decorrelate_[lrm]s_32_sse2(uint8_t **out, int32_t **in, int channels, +; int len, int shift); +;---------------------------------------------------------------------------------- +%macro FLAC_DECORRELATE_32 5 +cglobal flac_decorrelate_%1_32, 2, 4, 4, out, in0, in1, len +%if ARCH_X86_32 || WIN64 + movd m3, r4m +%if ARCH_X86_32 + mov lend, lenm +%endif +%else ; UNIX64 + movd m3, r4d +%endif + mov in1q, [in0q + gprsize] + mov in0q, [in0q] + mov outq, [outq] + sub in1q, in0q + +align 16 +.loop: + mova m0, [in0q] + mova m1, [in0q + in1q] +%ifidn %1, ms + psrad m2, m1, 1 + psubd m0, m2 +%endif + p%5d m2, m0, m1 + pslld m%2, m3 + pslld m%3, m3 + + SBUTTERFLY dq, %2, %3, %4 + + mova [outq ], m%2 + mova [outq + mmsize], m%3 + + add in0q, mmsize + add outq, mmsize*2 + sub lend, mmsize/4 + jg .loop + REP_RET +%endmacro + +INIT_XMM sse2 +FLAC_DECORRELATE_32 ls, 0, 2, 1, sub +FLAC_DECORRELATE_32 rs, 2, 1, 0, add +FLAC_DECORRELATE_32 ms, 2, 0, 1, add + +;----------------------------------------------------------------------------------------- +;void ff_flac_decorrelate_indep__(uint8_t **out, int32_t **in, int channels, +; int len, int shift); +;----------------------------------------------------------------------------------------- +%macro TRANSPOSE8x4D 9 + SBUTTERFLY dq, %1, %2, %9 + SBUTTERFLY dq, %3, %4, %9 + SBUTTERFLY dq, %5, %6, %9 + SBUTTERFLY dq, %7, %8, %9 + SBUTTERFLY qdq, %1, %3, %9 + SBUTTERFLY qdq, %2, %4, %9 + SBUTTERFLY qdq, %5, %7, %9 + SBUTTERFLY qdq, %6, %8, %9 + SWAP %2, %5 + SWAP %4, %7 +%endmacro + +;%1 = bps +;%2 = channels +;%3 = last xmm reg used +;%4 = word/dword (shift instruction) +%macro FLAC_DECORRELATE_INDEP 4 +%define REPCOUNT %2/(32/%1) ; 16bits = channels / 2; 32bits = channels +cglobal flac_decorrelate_indep%2_%1, 2, %2+2, %3+1, out, in0, in1, len, in2, in3, in4, in5, in6, in7 +%if ARCH_X86_32 + movd m%3, r4m +%if %2 == 6 + DEFINE_ARGS out, in0, in1, in2, in3, in4, in5 + %define lend dword r3m +%else + mov lend, lenm +%endif +%elif WIN64 + movd m%3, r4m +%else ; UNIX64 + movd m%3, r4d +%endif + +%assign %%i 1 +%rep %2-1 + mov in %+ %%i %+ q, [in0q+%%i*gprsize] +%assign %%i %%i+1 +%endrep + + mov in0q, [in0q] + mov outq, [outq] + +%assign %%i 1 +%rep %2-1 + sub in %+ %%i %+ q, in0q +%assign %%i %%i+1 +%endrep + +align 16 +.loop: + mova m0, [in0q] + +%assign %%i 1 +%rep REPCOUNT-1 + mova m %+ %%i, [in0q + in %+ %%i %+ q] +%assign %%i %%i+1 +%endrep + +%if %1 == 32 + +%if %2 == 8 + TRANSPOSE8x4D 0, 1, 2, 3, 4, 5, 6, 7, 8 +%elif %2 == 6 + SBUTTERFLY dq, 0, 1, 6 + SBUTTERFLY dq, 2, 3, 6 + SBUTTERFLY dq, 4, 5, 6 + + punpcklqdq m6, m0, m2 + punpckhqdq m2, m4 + shufps m4, m0, 0xe4 + punpcklqdq m0, m1, m3 + punpckhqdq m3, m5 + shufps m5, m1, 0xe4 + SWAP 0,6,1,4,5,3 +%elif %2 == 4 + TRANSPOSE4x4D 0, 1, 2, 3, 4 +%else ; %2 == 2 + SBUTTERFLY dq, 0, 1, 2 +%endif + +%else ; %1 == 16 + +%if %2 == 8 + packssdw m0, [in0q + in4q] + packssdw m1, [in0q + in5q] + packssdw m2, [in0q + in6q] + packssdw m3, [in0q + in7q] + TRANSPOSE2x4x4W 0, 1, 2, 3, 4 +%elif %2 == 6 + packssdw m0, [in0q + in3q] + packssdw m1, [in0q + in4q] + packssdw m2, [in0q + in5q] + pshufd m3, m0, q1032 + punpcklwd m0, m1 + punpckhwd m1, m2 + punpcklwd m2, m3 + + shufps m3, m0, m2, q2020 + shufps m0, m1, q2031 + shufps m2, m1, q3131 + shufps m1, m2, m3, q3120 + shufps m3, m0, q0220 + shufps m0, m2, q3113 + SWAP 2, 0, 3 +%else ; %2 == 4 + packssdw m0, [in0q + in2q] + packssdw m1, [in0q + in3q] + SBUTTERFLY wd, 0, 1, 2 + SBUTTERFLY dq, 0, 1, 2 +%endif + +%endif + +%assign %%i 0 +%rep REPCOUNT + psll%4 m %+ %%i, m%3 +%assign %%i %%i+1 +%endrep + +%assign %%i 0 +%rep REPCOUNT + mova [outq + %%i*mmsize], m %+ %%i +%assign %%i %%i+1 +%endrep + + add in0q, mmsize + add outq, mmsize*REPCOUNT + sub lend, mmsize/4 + jg .loop + REP_RET +%endmacro + +INIT_XMM sse2 +FLAC_DECORRELATE_16 indep2, 0, 1 ; Reuse stereo 16bits macro +FLAC_DECORRELATE_INDEP 32, 2, 3, d +FLAC_DECORRELATE_INDEP 16, 4, 3, w +FLAC_DECORRELATE_INDEP 32, 4, 5, d +FLAC_DECORRELATE_INDEP 16, 6, 4, w +FLAC_DECORRELATE_INDEP 32, 6, 7, d +%if ARCH_X86_64 +FLAC_DECORRELATE_INDEP 16, 8, 5, w +FLAC_DECORRELATE_INDEP 32, 8, 9, d +%endif + +INIT_XMM avx +FLAC_DECORRELATE_INDEP 32, 4, 5, d +FLAC_DECORRELATE_INDEP 32, 6, 7, d +%if ARCH_X86_64 +FLAC_DECORRELATE_INDEP 16, 8, 5, w +FLAC_DECORRELATE_INDEP 32, 8, 9, d +%endif diff --git a/libavcodec/x86/flacdsp_init.c b/libavcodec/x86/flacdsp_init.c index 33b31f12e4..d04af4511d 100644 --- a/libavcodec/x86/flacdsp_init.c +++ b/libavcodec/x86/flacdsp_init.c @@ -29,17 +29,78 @@ void ff_flac_lpc_32_xop(int32_t *samples, const int coeffs[32], int order, void ff_flac_enc_lpc_16_sse4(int32_t *, const int32_t *, int, int, const int32_t *,int); -av_cold void ff_flacdsp_init_x86(FLACDSPContext *c, enum AVSampleFormat fmt, +#define DECORRELATE_FUNCS(fmt, opt) \ +void ff_flac_decorrelate_ls_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift); \ +void ff_flac_decorrelate_rs_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift); \ +void ff_flac_decorrelate_ms_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift); \ +void ff_flac_decorrelate_indep2_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift); \ +void ff_flac_decorrelate_indep4_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift); \ +void ff_flac_decorrelate_indep6_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift); \ +void ff_flac_decorrelate_indep8_##fmt##_##opt(uint8_t **out, int32_t **in, int channels, \ + int len, int shift) + +DECORRELATE_FUNCS(16, sse2); +DECORRELATE_FUNCS(16, avx); +DECORRELATE_FUNCS(32, sse2); +DECORRELATE_FUNCS(32, avx); + +av_cold void ff_flacdsp_init_x86(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps) { #if HAVE_YASM int cpu_flags = av_get_cpu_flags(); #if CONFIG_FLAC_DECODER + if (EXTERNAL_SSE2(cpu_flags)) { + if (fmt == AV_SAMPLE_FMT_S16) { + if (channels == 2) + c->decorrelate[0] = ff_flac_decorrelate_indep2_16_sse2; + else if (channels == 4) + c->decorrelate[0] = ff_flac_decorrelate_indep4_16_sse2; + else if (channels == 6) + c->decorrelate[0] = ff_flac_decorrelate_indep6_16_sse2; + else if (ARCH_X86_64 && channels == 8) + c->decorrelate[0] = ff_flac_decorrelate_indep8_16_sse2; + c->decorrelate[1] = ff_flac_decorrelate_ls_16_sse2; + c->decorrelate[2] = ff_flac_decorrelate_rs_16_sse2; + c->decorrelate[3] = ff_flac_decorrelate_ms_16_sse2; + } else if (fmt == AV_SAMPLE_FMT_S32) { + if (channels == 2) + c->decorrelate[0] = ff_flac_decorrelate_indep2_32_sse2; + else if (channels == 4) + c->decorrelate[0] = ff_flac_decorrelate_indep4_32_sse2; + else if (channels == 6) + c->decorrelate[0] = ff_flac_decorrelate_indep6_32_sse2; + else if (ARCH_X86_64 && channels == 8) + c->decorrelate[0] = ff_flac_decorrelate_indep8_32_sse2; + c->decorrelate[1] = ff_flac_decorrelate_ls_32_sse2; + c->decorrelate[2] = ff_flac_decorrelate_rs_32_sse2; + c->decorrelate[3] = ff_flac_decorrelate_ms_32_sse2; + } + } if (EXTERNAL_SSE4(cpu_flags)) { if (bps > 16) c->lpc = ff_flac_lpc_32_sse4; } + if (EXTERNAL_AVX(cpu_flags)) { + if (fmt == AV_SAMPLE_FMT_S16) { + if (ARCH_X86_64 && channels == 8) + c->decorrelate[0] = ff_flac_decorrelate_indep8_16_avx; + } else if (fmt == AV_SAMPLE_FMT_S32) { + if (channels == 4) + c->decorrelate[0] = ff_flac_decorrelate_indep4_32_avx; + else if (channels == 6) + c->decorrelate[0] = ff_flac_decorrelate_indep6_32_avx; + else if (ARCH_X86_64 && channels == 8) + c->decorrelate[0] = ff_flac_decorrelate_indep8_32_avx; + } + } if (EXTERNAL_XOP(cpu_flags)) { if (bps > 16) c->lpc = ff_flac_lpc_32_xop;