diff --git a/include/common/attributes.h b/include/common/attributes.h index 4bd8217d..d5c4ce50 100644 --- a/include/common/attributes.h +++ b/include/common/attributes.h @@ -43,15 +43,18 @@ #endif #if ARCH_X86_64 -/* x86-64 needs 32-byte alignment for AVX2. */ +/* x86-64 needs 32- and 64-byte alignment for AVX2 and AVX-512. */ +#define ALIGN_64_VAL 64 #define ALIGN_32_VAL 32 #define ALIGN_16_VAL 16 #elif ARCH_X86_32 || ARCH_ARM || ARCH_AARCH64 || ARCH_PPC64LE /* ARM doesn't benefit from anything more than 16-byte alignment. */ +#define ALIGN_64_VAL 16 #define ALIGN_32_VAL 16 #define ALIGN_16_VAL 16 #else /* No need for extra alignment on platforms without assembly. */ +#define ALIGN_64_VAL 8 #define ALIGN_32_VAL 8 #define ALIGN_16_VAL 8 #endif @@ -76,9 +79,10 @@ * becomes: * ALIGN_STK_$align(uint8_t, var, 1, [2][3][4]) */ +#define ALIGN_STK_64(type, var, sz1d, sznd) \ + ALIGN(type var[sz1d]sznd, ALIGN_64_VAL) #define ALIGN_STK_32(type, var, sz1d, sznd) \ ALIGN(type var[sz1d]sznd, ALIGN_32_VAL) -// as long as stack is itself 16-byte aligned, this works (win64, gcc) #define ALIGN_STK_16(type, var, sz1d, sznd) \ ALIGN(type var[sz1d]sznd, ALIGN_16_VAL) diff --git a/tests/checkasm/cdef.c b/tests/checkasm/cdef.c index b177de50..f1a8f260 100644 --- a/tests/checkasm/cdef.c +++ b/tests/checkasm/cdef.c @@ -45,9 +45,9 @@ static void init_tmp(pixel *buf, int n, const int bitdepth_max) { } static void check_cdef_filter(const cdef_fn fn, const int w, const int h) { - ALIGN_STK_32(pixel, c_src, 10 * 16 + 8, ), *const c_dst = c_src + 8; - ALIGN_STK_32(pixel, a_src, 10 * 16 + 8, ), *const a_dst = a_src + 8; - ALIGN_STK_32(pixel, top, 16 * 2 + 8, ); + ALIGN_STK_64(pixel, c_src, 10 * 16 + 8, ), *const c_dst = c_src + 8; + ALIGN_STK_64(pixel, a_src, 10 * 16 + 8, ), *const a_dst = a_src + 8; + ALIGN_STK_64(pixel, top, 16 * 2 + 8, ); pixel left[8][2]; pixel *const top_ptrs[2] = { top + 8, top + 24 }; const ptrdiff_t stride = 16 * sizeof(pixel); @@ -103,7 +103,7 @@ static void check_cdef_filter(const cdef_fn fn, const int w, const int h) { } static void check_cdef_direction(const cdef_dir_fn fn) { - ALIGN_STK_32(pixel, src, 8 * 8,); + ALIGN_STK_64(pixel, src, 8 * 8,); declare_func(int, pixel *src, ptrdiff_t dst_stride, unsigned *var HIGHBD_DECL_SUFFIX); diff --git a/tests/checkasm/filmgrain.c b/tests/checkasm/filmgrain.c index 34b9b539..5c9e0bfc 100644 --- a/tests/checkasm/filmgrain.c +++ b/tests/checkasm/filmgrain.c @@ -137,9 +137,9 @@ static void check_gen_grnuv(const Dav1dFilmGrainDSPContext *const dsp) { } static void check_fgy_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { - ALIGN_STK_32(pixel, c_dst, 128 * 32,); - ALIGN_STK_32(pixel, a_dst, 128 * 32,); - ALIGN_STK_32(pixel, src, 128 * 32,); + ALIGN_STK_64(pixel, c_dst, 128 * 32,); + ALIGN_STK_64(pixel, a_dst, 128 * 32,); + ALIGN_STK_64(pixel, src, 128 * 32,); const ptrdiff_t stride = 128 * sizeof(pixel); declare_func(void, pixel *dst_row, const pixel *src_row, ptrdiff_t stride, @@ -207,10 +207,10 @@ static void check_fgy_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { } static void check_fguv_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { - ALIGN_STK_32(pixel, c_dst, 128 * 32,); - ALIGN_STK_32(pixel, a_dst, 128 * 32,); - ALIGN_STK_32(pixel, src, 128 * 32,); - ALIGN_STK_32(pixel, luma_src, 128 * 32,); + ALIGN_STK_64(pixel, c_dst, 128 * 32,); + ALIGN_STK_64(pixel, a_dst, 128 * 32,); + ALIGN_STK_64(pixel, src, 128 * 32,); + ALIGN_STK_64(pixel, luma_src, 128 * 32,); const ptrdiff_t lstride = 128 * sizeof(pixel); declare_func(void, pixel *dst_row, const pixel *src_row, ptrdiff_t stride, diff --git a/tests/checkasm/ipred.c b/tests/checkasm/ipred.c index 7c08a0b3..d5955d20 100644 --- a/tests/checkasm/ipred.c +++ b/tests/checkasm/ipred.c @@ -66,9 +66,9 @@ static const uint8_t z_angles[27] = { }; static void check_intra_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 64 * 64,); - ALIGN_STK_32(pixel, a_dst, 64 * 64,); - ALIGN_STK_32(pixel, topleft_buf, 257,); + ALIGN_STK_64(pixel, c_dst, 64 * 64,); + ALIGN_STK_64(pixel, a_dst, 64 * 64,); + ALIGN_STK_64(pixel, topleft_buf, 257,); pixel *const topleft = topleft_buf + 128; declare_func(void, pixel *dst, ptrdiff_t stride, const pixel *topleft, @@ -132,9 +132,9 @@ static void check_intra_pred(Dav1dIntraPredDSPContext *const c) { } static void check_cfl_ac(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(int16_t, c_dst, 32 * 32,); - ALIGN_STK_32(int16_t, a_dst, 32 * 32,); - ALIGN_STK_32(pixel, luma, 32 * 32,); + ALIGN_STK_64(int16_t, c_dst, 32 * 32,); + ALIGN_STK_64(int16_t, a_dst, 32 * 32,); + ALIGN_STK_64(pixel, luma, 32 * 32,); declare_func(void, int16_t *ac, const pixel *y, ptrdiff_t stride, int w_pad, int h_pad, int cw, int ch); @@ -175,10 +175,10 @@ static void check_cfl_ac(Dav1dIntraPredDSPContext *const c) { } static void check_cfl_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 32 * 32,); - ALIGN_STK_32(pixel, a_dst, 32 * 32,); - ALIGN_STK_32(int16_t, ac, 32 * 32,); - ALIGN_STK_32(pixel, topleft_buf, 257,); + ALIGN_STK_64(pixel, c_dst, 32 * 32,); + ALIGN_STK_64(pixel, a_dst, 32 * 32,); + ALIGN_STK_64(int16_t, ac, 32 * 32,); + ALIGN_STK_64(pixel, topleft_buf, 257,); pixel *const topleft = topleft_buf + 128; declare_func(void, pixel *dst, ptrdiff_t stride, const pixel *topleft, @@ -227,9 +227,9 @@ static void check_cfl_pred(Dav1dIntraPredDSPContext *const c) { } static void check_pal_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 64 * 64,); - ALIGN_STK_32(pixel, a_dst, 64 * 64,); - ALIGN_STK_32(uint8_t, idx, 64 * 64,); + ALIGN_STK_64(pixel, c_dst, 64 * 64,); + ALIGN_STK_64(pixel, a_dst, 64 * 64,); + ALIGN_STK_64(uint8_t, idx, 64 * 64,); ALIGN_STK_16(uint16_t, pal, 8,); declare_func(void, pixel *dst, ptrdiff_t stride, const uint16_t *pal, diff --git a/tests/checkasm/itx.c b/tests/checkasm/itx.c index d738a6ed..9d715c8f 100644 --- a/tests/checkasm/itx.c +++ b/tests/checkasm/itx.c @@ -226,9 +226,9 @@ void bitfn(checkasm_check_itx)(void) { Dav1dInvTxfmDSPContext c; bitfn(dav1d_itx_dsp_init)(&c); - ALIGN_STK_32(coef, coeff, 2, [32 * 32]); - ALIGN_STK_32(pixel, c_dst, 64 * 64,); - ALIGN_STK_32(pixel, a_dst, 64 * 64,); + ALIGN_STK_64(coef, coeff, 2, [32 * 32]); + ALIGN_STK_64(pixel, c_dst, 64 * 64,); + ALIGN_STK_64(pixel, a_dst, 64 * 64,); static const uint8_t txfm_size_order[N_RECT_TX_SIZES] = { TX_4X4, RTX_4X8, RTX_4X16, diff --git a/tests/checkasm/loopfilter.c b/tests/checkasm/loopfilter.c index 00e62db4..aabf54ff 100644 --- a/tests/checkasm/loopfilter.c +++ b/tests/checkasm/loopfilter.c @@ -95,8 +95,8 @@ static void check_lpf_sb(loopfilter_sb_fn fn, const char *const name, const int n_blks, const int lf_idx, const int is_chroma, const int dir) { - ALIGN_STK_32(pixel, c_dst_mem, 128 * 16,); - ALIGN_STK_32(pixel, a_dst_mem, 128 * 16,); + ALIGN_STK_64(pixel, c_dst_mem, 128 * 16,); + ALIGN_STK_64(pixel, a_dst_mem, 128 * 16,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const uint32_t *mask, const uint8_t (*l)[4], ptrdiff_t b4_stride, diff --git a/tests/checkasm/looprestoration.c b/tests/checkasm/looprestoration.c index ed0a9746..9e8eba94 100644 --- a/tests/checkasm/looprestoration.c +++ b/tests/checkasm/looprestoration.c @@ -44,9 +44,9 @@ static void init_tmp(pixel *buf, const ptrdiff_t stride, } static void check_wiener(Dav1dLoopRestorationDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 448 * 64,); - ALIGN_STK_32(pixel, a_dst, 448 * 64,); - ALIGN_STK_32(pixel, h_edge, 448 * 8,); + ALIGN_STK_64(pixel, c_dst, 448 * 64,); + ALIGN_STK_64(pixel, a_dst, 448 * 64,); + ALIGN_STK_64(pixel, h_edge, 448 * 8,); pixel left[64][4]; declare_func(void, pixel *dst, ptrdiff_t dst_stride, @@ -116,9 +116,9 @@ static void check_wiener(Dav1dLoopRestorationDSPContext *const c) { } static void check_sgr(Dav1dLoopRestorationDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 448 * 64,); - ALIGN_STK_32(pixel, a_dst, 448 * 64,); - ALIGN_STK_32(pixel, h_edge, 448 * 8,); + ALIGN_STK_64(pixel, c_dst, 448 * 64,); + ALIGN_STK_64(pixel, a_dst, 448 * 64,); + ALIGN_STK_64(pixel, h_edge, 448 * 8,); pixel left[64][4]; declare_func(void, pixel *dst, ptrdiff_t dst_stride, diff --git a/tests/checkasm/mc.c b/tests/checkasm/mc.c index d4357c8f..e820bda8 100644 --- a/tests/checkasm/mc.c +++ b/tests/checkasm/mc.c @@ -55,9 +55,9 @@ static int mc_h_next(const int h) { } static void check_mc(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 135 * 135,); - ALIGN_STK_32(pixel, c_dst, 128 * 128,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(pixel, src_buf, 135 * 135,); + ALIGN_STK_64(pixel, c_dst, 128 * 128,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); const pixel *src = src_buf + 135 * 3 + 3; const ptrdiff_t src_stride = 135 * sizeof(pixel); @@ -118,9 +118,9 @@ static void generate_mct_input(pixel *const buf, const int bitdepth_max) { } static void check_mct(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 135 * 135,); - ALIGN_STK_32(int16_t, c_tmp, 128 * 128,); - ALIGN_STK_32(int16_t, a_tmp, 128 * 128,); + ALIGN_STK_64(pixel, src_buf, 135 * 135,); + ALIGN_STK_64(int16_t, c_tmp, 128 * 128,); + ALIGN_STK_64(int16_t, a_tmp, 128 * 128,); const pixel *src = src_buf + 135 * 3 + 3; const ptrdiff_t src_stride = 135 * sizeof(pixel); @@ -173,9 +173,9 @@ static void init_tmp(Dav1dMCDSPContext *const c, pixel *const buf, } static void check_avg(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, const int16_t *tmp2, int w, int h HIGHBD_DECL_SUFFIX); @@ -204,9 +204,9 @@ static void check_avg(Dav1dMCDSPContext *const c) { } static void check_w_avg(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, const int16_t *tmp2, int w, int h, int weight HIGHBD_DECL_SUFFIX); @@ -236,10 +236,10 @@ static void check_w_avg(Dav1dMCDSPContext *const c) { } static void check_mask(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); - ALIGN_STK_32(uint8_t, mask, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(uint8_t, mask, 128 * 128,); for (int i = 0; i < 128 * 128; i++) mask[i] = rnd() % 65; @@ -271,11 +271,11 @@ static void check_mask(Dav1dMCDSPContext *const c) { } static void check_w_mask(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); - ALIGN_STK_32(uint8_t, c_mask, 128 * 128,); - ALIGN_STK_32(uint8_t, a_mask, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(uint8_t, c_mask, 128 * 128,); + ALIGN_STK_64(uint8_t, a_mask, 128 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, const int16_t *tmp2, int w, int h, uint8_t *mask, int sign @@ -321,10 +321,10 @@ static void check_w_mask(Dav1dMCDSPContext *const c) { } static void check_blend(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, tmp, 32 * 32,); - ALIGN_STK_32(pixel, c_dst, 32 * 32,); - ALIGN_STK_32(pixel, a_dst, 32 * 32,); - ALIGN_STK_32(uint8_t, mask, 32 * 32,); + ALIGN_STK_64(pixel, tmp, 32 * 32,); + ALIGN_STK_64(pixel, c_dst, 32 * 32,); + ALIGN_STK_64(pixel, a_dst, 32 * 32,); + ALIGN_STK_64(uint8_t, mask, 32 * 32,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h, const uint8_t *mask); @@ -357,9 +357,9 @@ static void check_blend(Dav1dMCDSPContext *const c) { } static void check_blend_v(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, tmp, 32 * 128,); - ALIGN_STK_32(pixel, c_dst, 32 * 128,); - ALIGN_STK_32(pixel, a_dst, 32 * 128,); + ALIGN_STK_64(pixel, tmp, 32 * 128,); + ALIGN_STK_64(pixel, c_dst, 32 * 128,); + ALIGN_STK_64(pixel, a_dst, 32 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h); @@ -391,9 +391,9 @@ static void check_blend_v(Dav1dMCDSPContext *const c) { } static void check_blend_h(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, tmp, 128 * 32,); - ALIGN_STK_32(pixel, c_dst, 128 * 32,); - ALIGN_STK_32(pixel, a_dst, 128 * 32,); + ALIGN_STK_64(pixel, tmp, 128 * 32,); + ALIGN_STK_64(pixel, c_dst, 128 * 32,); + ALIGN_STK_64(pixel, a_dst, 128 * 32,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h); @@ -424,9 +424,9 @@ static void check_blend_h(Dav1dMCDSPContext *const c) { } static void check_warp8x8(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 15 * 15,); - ALIGN_STK_32(pixel, c_dst, 8 * 8,); - ALIGN_STK_32(pixel, a_dst, 8 * 8,); + ALIGN_STK_64(pixel, src_buf, 15 * 15,); + ALIGN_STK_64(pixel, c_dst, 8 * 8,); + ALIGN_STK_64(pixel, a_dst, 8 * 8,); int16_t abcd[4]; const pixel *src = src_buf + 15 * 3 + 3; const ptrdiff_t dst_stride = 8 * sizeof(pixel); @@ -462,9 +462,9 @@ static void check_warp8x8(Dav1dMCDSPContext *const c) { } static void check_warp8x8t(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 15 * 15,); - ALIGN_STK_32(int16_t, c_tmp, 8 * 8,); - ALIGN_STK_32(int16_t, a_tmp, 8 * 8,); + ALIGN_STK_64(pixel, src_buf, 15 * 15,); + ALIGN_STK_64(int16_t, c_tmp, 8 * 8,); + ALIGN_STK_64(int16_t, a_tmp, 8 * 8,); int16_t abcd[4]; const pixel *src = src_buf + 15 * 3 + 3; const ptrdiff_t src_stride = 15 * sizeof(pixel); @@ -534,9 +534,9 @@ static void random_offset_for_edge(int *const x, int *const y, } static void check_emuedge(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 135 * 192,); - ALIGN_STK_32(pixel, a_dst, 135 * 192,); - ALIGN_STK_32(pixel, src, 160 * 160,); + ALIGN_STK_64(pixel, c_dst, 135 * 192,); + ALIGN_STK_64(pixel, a_dst, 135 * 192,); + ALIGN_STK_64(pixel, src, 160 * 160,); for (int i = 0; i < 160 * 160; i++) src[i] = rnd() & ((1U << BITDEPTH) - 1);