diff --git a/include/common/intops.h b/include/common/intops.h index 2d21998b..089da5e1 100644 --- a/include/common/intops.h +++ b/include/common/intops.h @@ -65,11 +65,11 @@ static inline int apply_sign64(const int v, const int64_t s) { } static inline int ulog2(const unsigned v) { - return 31 - clz(v); + return 31 ^ clz(v); } static inline int u64log2(const uint64_t v) { - return 63 - clzll(v); + return 63 ^ clzll(v); } static inline unsigned inv_recenter(const unsigned r, const unsigned v) { diff --git a/src/ctx.c b/src/ctx.c new file mode 100644 index 00000000..0a0fe54c --- /dev/null +++ b/src/ctx.c @@ -0,0 +1,65 @@ +/* + * Copyright © 2024, VideoLAN and dav1d authors + * Copyright © 2024, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "ctx.h" + +static void memset_w1(void *const ptr, const int value) { + set_ctx1((uint8_t *) ptr, 0, value); +} + +static void memset_w2(void *const ptr, const int value) { + set_ctx2((uint8_t *) ptr, 0, value); +} + +static void memset_w4(void *const ptr, const int value) { + set_ctx4((uint8_t *) ptr, 0, value); +} + +static void memset_w8(void *const ptr, const int value) { + set_ctx8((uint8_t *) ptr, 0, value); +} + +static void memset_w16(void *const ptr, const int value) { + set_ctx16((uint8_t *) ptr, 0, value); +} + +static void memset_w32(void *const ptr, const int value) { + set_ctx32((uint8_t *) ptr, 0, value); +} + +const dav1d_memset_pow2_fn dav1d_memset_pow2[6] = { + memset_w1, + memset_w2, + memset_w4, + memset_w8, + memset_w16, + memset_w32 +}; diff --git a/src/ctx.h b/src/ctx.h index d0e1f310..7dea8b68 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -31,61 +31,59 @@ #include #include "common/attributes.h" +#include "common/intops.h" union alias64 { uint64_t u64; uint8_t u8[8]; } ATTR_ALIAS; union alias32 { uint32_t u32; uint8_t u8[4]; } ATTR_ALIAS; union alias16 { uint16_t u16; uint8_t u8[2]; } ATTR_ALIAS; union alias8 { uint8_t u8; } ATTR_ALIAS; -#define set_ctx_rep4(type, var, off, val) do { \ - const uint64_t const_val = val; \ - ((union alias64 *) &var[off + 0])->u64 = const_val; \ - ((union alias64 *) &var[off + 8])->u64 = const_val; \ - ((union alias64 *) &var[off + 16])->u64 = const_val; \ - ((union alias64 *) &var[off + 24])->u64 = const_val; \ +typedef void (*dav1d_memset_pow2_fn)(void *ptr, int value); +EXTERN const dav1d_memset_pow2_fn dav1d_memset_pow2[6]; + +static inline void dav1d_memset_likely_pow2(void *const ptr, const int value, const int n) { + assert(n >= 1 && n <= 32); + if ((n&(n-1)) == 0) { + dav1d_memset_pow2[ulog2(n)](ptr, value); + } else { + memset(ptr, value, n); + } +} + +// For smaller sizes use multiplication to broadcast bytes. memset misbehaves on the smaller sizes. +// For the larger sizes, we want to use memset to get access to vector operations. +#define set_ctx1(var, off, val) \ + ((union alias8 *) &(var)[off])->u8 = (val) * 0x01 +#define set_ctx2(var, off, val) \ + ((union alias16 *) &(var)[off])->u16 = (val) * 0x0101 +#define set_ctx4(var, off, val) \ + ((union alias32 *) &(var)[off])->u32 = (val) * 0x01010101U +#define set_ctx8(var, off, val) \ + ((union alias64 *) &(var)[off])->u64 = (val) * 0x0101010101010101ULL +#define set_ctx16(var, off, val) do { \ + memset(&(var)[off], val, 16); \ } while (0) -#define set_ctx_rep2(type, var, off, val) do { \ - const uint64_t const_val = val; \ - ((union alias64 *) &var[off + 0])->u64 = const_val; \ - ((union alias64 *) &var[off + 8])->u64 = const_val; \ +#define set_ctx32(var, off, val) do { \ + memset(&(var)[off], val, 32); \ } while (0) -#define set_ctx_rep1(typesz, var, off, val) \ - ((union alias##typesz *) &var[off])->u##typesz = val -#define case_set(var, dir, diridx, off) \ +#define case_set(var) \ switch (var) { \ - case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ - case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ - case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ - case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ - case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ - case 32: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep4); break; \ + case 0: set_ctx(set_ctx1); break; \ + case 1: set_ctx(set_ctx2); break; \ + case 2: set_ctx(set_ctx4); break; \ + case 3: set_ctx(set_ctx8); break; \ + case 4: set_ctx(set_ctx16); break; \ + case 5: set_ctx(set_ctx32); break; \ + default: assert(0); \ } -#define case_set_upto16(var, dir, diridx, off) \ +#define case_set_upto16(var) \ switch (var) { \ - case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ - case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ - case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ - case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ - case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ - } -#define case_set_upto32_with_default(var, dir, diridx, off) \ - switch (var) { \ - case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ - case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ - case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ - case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ - case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ - case 32: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep4); break; \ - default: default_memset(dir, diridx, off, var); break; \ - } -#define case_set_upto16_with_default(var, dir, diridx, off) \ - switch (var) { \ - case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ - case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ - case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ - case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ - case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ - default: default_memset(dir, diridx, off, var); break; \ + case 0: set_ctx(set_ctx1); break; \ + case 1: set_ctx(set_ctx2); break; \ + case 2: set_ctx(set_ctx4); break; \ + case 3: set_ctx(set_ctx8); break; \ + case 4: set_ctx(set_ctx16); break; \ + default: assert(0); \ } #endif /* DAV1D_SRC_CTX_H */ diff --git a/src/decode.c b/src/decode.c index ea371324..f5b6db95 100644 --- a/src/decode.c +++ b/src/decode.c @@ -161,14 +161,8 @@ static void read_tx_tree(Dav1dTaskContext *const t, } t->by -= txsh; } else { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir tx, off, is_split ? TX_4X4 : mul * txh) - case_set_upto16(t_dim->h, l., 1, by4); -#undef set_ctx -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir tx, off, is_split ? TX_4X4 : mul * txw) - case_set_upto16(t_dim->w, a->, 0, bx4); -#undef set_ctx + dav1d_memset_pow2[t_dim->lw](&t->a->tx[bx4], is_split ? TX_4X4 : txw); + dav1d_memset_pow2[t_dim->lh](&t->l.tx[by4], is_split ? TX_4X4 : txh); } } @@ -464,19 +458,13 @@ static void read_vartx_tree(Dav1dTaskContext *const t, { b->max_ytx = b->uvtx = TX_4X4; if (f->frame_hdr->txfm_mode == DAV1D_TX_SWITCHABLE) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir tx, off, TX_4X4) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); -#undef set_ctx + dav1d_memset_pow2[b_dim[2]](&t->a->tx[bx4], TX_4X4); + dav1d_memset_pow2[b_dim[3]](&t->l.tx[by4], TX_4X4); } } else if (f->frame_hdr->txfm_mode != DAV1D_TX_SWITCHABLE || b->skip) { if (f->frame_hdr->txfm_mode == DAV1D_TX_SWITCHABLE) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir tx, off, mul * b_dim[2 + diridx]) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); -#undef set_ctx + dav1d_memset_pow2[b_dim[2]](&t->a->tx[bx4], b_dim[2 + 0]); + dav1d_memset_pow2[b_dim[3]](&t->l.tx[by4], b_dim[2 + 1]); } b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; } else { @@ -696,8 +684,7 @@ static int decode_b(Dav1dTaskContext *const t, const enum BlockLevel bl, const enum BlockSize bs, const enum BlockPartition bp, - const enum EdgeFlags intra_edge_flags) -{ + const enum EdgeFlags intra_edge_flags) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; Av1Block b_mem, *const b = t->frame_thread.pass ? @@ -722,11 +709,13 @@ static int decode_b(Dav1dTaskContext *const t, const enum IntraPredMode y_mode_nofilt = b->y_mode == FILTER_PRED ? DC_PRED : b->y_mode; -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir mode, off, mul * y_mode_nofilt); \ - rep_macro(type, t->dir intra, off, mul) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); +#define set_ctx(rep_macro) \ + rep_macro(edge->mode, off, y_mode_nofilt); \ + rep_macro(edge->intra, off, 1) + BlockContext *edge = t->a; + for (int i = 0, off = bx4; i < 2; i++, off = by4, edge = &t->l) { + case_set(b_dim[2 + i]); + } #undef set_ctx if (IS_INTER_OR_SWITCH(f->frame_hdr)) { refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; @@ -742,11 +731,9 @@ static int decode_b(Dav1dTaskContext *const t, } if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + uint8_t uv_mode = b->uv_mode; + dav1d_memset_pow2[ulog2(cbw4)](&t->a->uvmode[cbx4], uv_mode); + dav1d_memset_pow2[ulog2(cbh4)](&t->l.uvmode[cby4], uv_mode); } } else { if (IS_INTER_OR_SWITCH(f->frame_hdr) /* not intrabc */ && @@ -784,13 +771,15 @@ static int decode_b(Dav1dTaskContext *const t, if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; const uint8_t *const filter = dav1d_filter_dir[b->filter2d]; -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir filter[0], off, mul * filter[0]); \ - rep_macro(type, t->dir filter[1], off, mul * filter[1]); \ - rep_macro(type, t->dir intra, off, 0) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); + BlockContext *edge = t->a; + for (int i = 0, off = bx4; i < 2; i++, off = by4, edge = &t->l) { +#define set_ctx(rep_macro) \ + rep_macro(edge->filter[0], off, filter[0]); \ + rep_macro(edge->filter[1], off, filter[1]); \ + rep_macro(edge->intra, off, 0) + case_set(b_dim[2 + i]); #undef set_ctx + } if (IS_INTER_OR_SWITCH(f->frame_hdr)) { refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; @@ -808,11 +797,8 @@ static int decode_b(Dav1dTaskContext *const t, } if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir uvmode, off, mul * DC_PRED) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + dav1d_memset_pow2[ulog2(cbw4)](&t->a->uvmode[cbx4], DC_PRED); + dav1d_memset_pow2[ulog2(cbh4)](&t->l.uvmode[cby4], DC_PRED); } } return 0; @@ -1240,39 +1226,39 @@ static int decode_b(Dav1dTaskContext *const t, has_chroma ? &t->a->tx_lpf_uv[cbx4] : NULL, has_chroma ? &t->l.tx_lpf_uv[cby4] : NULL); } - // update contexts -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir tx_intra, off, mul * (((uint8_t *) &t_dim->lw)[diridx])); \ - rep_macro(type, t->dir tx, off, mul * (((uint8_t *) &t_dim->lw)[diridx])); \ - rep_macro(type, t->dir mode, off, mul * y_mode_nofilt); \ - rep_macro(type, t->dir pal_sz, off, mul * b->pal_sz[0]); \ - rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ - rep_macro(type, t->dir skip_mode, off, 0); \ - rep_macro(type, t->dir intra, off, mul); \ - rep_macro(type, t->dir skip, off, mul * b->skip); \ - /* see aomedia bug 2183 for why we use luma coordinates here */ \ - rep_macro(type, t->pal_sz_uv[diridx], off, mul * (has_chroma ? b->pal_sz[1] : 0)); \ - if (IS_INTER_OR_SWITCH(f->frame_hdr)) { \ - rep_macro(type, t->dir comp_type, off, mul * COMP_INTER_NONE); \ - rep_macro(type, t->dir ref[0], off, mul * ((uint8_t) -1)); \ - rep_macro(type, t->dir ref[1], off, mul * ((uint8_t) -1)); \ - rep_macro(type, t->dir filter[0], off, mul * DAV1D_N_SWITCHABLE_FILTERS); \ - rep_macro(type, t->dir filter[1], off, mul * DAV1D_N_SWITCHABLE_FILTERS); \ - } const enum IntraPredMode y_mode_nofilt = b->y_mode == FILTER_PRED ? DC_PRED : b->y_mode; - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); + BlockContext *edge = t->a; + for (int i = 0, off = bx4; i < 2; i++, off = by4, edge = &t->l) { + int t_lsz = ((uint8_t *) &t_dim->lw)[i]; // lw then lh +#define set_ctx(rep_macro) \ + rep_macro(edge->tx_intra, off, t_lsz); \ + rep_macro(edge->tx, off, t_lsz); \ + rep_macro(edge->mode, off, y_mode_nofilt); \ + rep_macro(edge->pal_sz, off, b->pal_sz[0]); \ + rep_macro(edge->seg_pred, off, seg_pred); \ + rep_macro(edge->skip_mode, off, 0); \ + rep_macro(edge->intra, off, 1); \ + rep_macro(edge->skip, off, b->skip); \ + /* see aomedia bug 2183 for why we use luma coordinates here */ \ + rep_macro(t->pal_sz_uv[i], off, (has_chroma ? b->pal_sz[1] : 0)); \ + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { \ + rep_macro(edge->comp_type, off, COMP_INTER_NONE); \ + rep_macro(edge->ref[0], off, ((uint8_t) -1)); \ + rep_macro(edge->ref[1], off, ((uint8_t) -1)); \ + rep_macro(edge->filter[0], off, DAV1D_N_SWITCHABLE_FILTERS); \ + rep_macro(edge->filter[1], off, DAV1D_N_SWITCHABLE_FILTERS); \ + } + case_set(b_dim[2 + i]); #undef set_ctx + } if (b->pal_sz[0]) f->bd_fn.copy_pal_block_y(t, bx4, by4, bw4, bh4); if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + uint8_t uv_mode = b->uv_mode; + dav1d_memset_pow2[ulog2(cbw4)](&t->a->uvmode[cbx4], uv_mode); + dav1d_memset_pow2[ulog2(cbh4)](&t->l.uvmode[cby4], uv_mode); if (b->pal_sz[1]) f->bd_fn.copy_pal_block_uv(t, bx4, by4, bw4, bh4); } @@ -1374,26 +1360,24 @@ static int decode_b(Dav1dTaskContext *const t, } splat_intrabc_mv(f->c, t, bs, b, bw4, bh4); - -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ - rep_macro(type, t->dir mode, off, mul * DC_PRED); \ - rep_macro(type, t->dir pal_sz, off, 0); \ - /* see aomedia bug 2183 for why this is outside if (has_chroma) */ \ - rep_macro(type, t->pal_sz_uv[diridx], off, 0); \ - rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ - rep_macro(type, t->dir skip_mode, off, 0); \ - rep_macro(type, t->dir intra, off, 0); \ - rep_macro(type, t->dir skip, off, mul * b->skip) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); + BlockContext *edge = t->a; + for (int i = 0, off = bx4; i < 2; i++, off = by4, edge = &t->l) { +#define set_ctx(rep_macro) \ + rep_macro(edge->tx_intra, off, b_dim[2 + i]); \ + rep_macro(edge->mode, off, DC_PRED); \ + rep_macro(edge->pal_sz, off, 0); \ + /* see aomedia bug 2183 for why this is outside if (has_chroma) */ \ + rep_macro(t->pal_sz_uv[i], off, 0); \ + rep_macro(edge->seg_pred, off, seg_pred); \ + rep_macro(edge->skip_mode, off, 0); \ + rep_macro(edge->intra, off, 0); \ + rep_macro(edge->skip, off, b->skip) + case_set(b_dim[2 + i]); #undef set_ctx + } if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir uvmode, off, mul * DC_PRED) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + dav1d_memset_pow2[ulog2(cbw4)](&t->a->uvmode[cbx4], DC_PRED); + dav1d_memset_pow2[ulog2(cbh4)](&t->l.uvmode[cby4], DC_PRED); } } else { // inter-specific mode/mv coding @@ -1922,32 +1906,29 @@ static int decode_b(Dav1dTaskContext *const t, splat_tworef_mv(f->c, t, bs, b, bw4, bh4); else splat_oneref_mv(f->c, t, bs, b, bw4, bh4); - -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ - rep_macro(type, t->dir skip_mode, off, mul * b->skip_mode); \ - rep_macro(type, t->dir intra, off, 0); \ - rep_macro(type, t->dir skip, off, mul * b->skip); \ - rep_macro(type, t->dir pal_sz, off, 0); \ - /* see aomedia bug 2183 for why this is outside if (has_chroma) */ \ - rep_macro(type, t->pal_sz_uv[diridx], off, 0); \ - rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ - rep_macro(type, t->dir comp_type, off, mul * b->comp_type); \ - rep_macro(type, t->dir filter[0], off, mul * filter[0]); \ - rep_macro(type, t->dir filter[1], off, mul * filter[1]); \ - rep_macro(type, t->dir mode, off, mul * b->inter_mode); \ - rep_macro(type, t->dir ref[0], off, mul * b->ref[0]); \ - rep_macro(type, t->dir ref[1], off, mul * ((uint8_t) b->ref[1])) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); + BlockContext *edge = t->a; + for (int i = 0, off = bx4; i < 2; i++, off = by4, edge = &t->l) { +#define set_ctx(rep_macro) \ + rep_macro(edge->seg_pred, off, seg_pred); \ + rep_macro(edge->skip_mode, off, b->skip_mode); \ + rep_macro(edge->intra, off, 0); \ + rep_macro(edge->skip, off, b->skip); \ + rep_macro(edge->pal_sz, off, 0); \ + /* see aomedia bug 2183 for why this is outside if (has_chroma) */ \ + rep_macro(t->pal_sz_uv[i], off, 0); \ + rep_macro(edge->tx_intra, off, b_dim[2 + i]); \ + rep_macro(edge->comp_type, off, b->comp_type); \ + rep_macro(edge->filter[0], off, filter[0]); \ + rep_macro(edge->filter[1], off, filter[1]); \ + rep_macro(edge->mode, off, b->inter_mode); \ + rep_macro(edge->ref[0], off, b->ref[0]); \ + rep_macro(edge->ref[1], off, ((uint8_t) b->ref[1])) + case_set(b_dim[2 + i]); #undef set_ctx - + } if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir uvmode, off, mul * DC_PRED) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + dav1d_memset_pow2[ulog2(cbw4)](&t->a->uvmode[cbx4], DC_PRED); + dav1d_memset_pow2[ulog2(cbh4)](&t->l.uvmode[cby4], DC_PRED); } } @@ -1956,12 +1937,12 @@ static int decode_b(Dav1dTaskContext *const t, f->frame_hdr->segmentation.update_map) { uint8_t *seg_ptr = &f->cur_segmap[t->by * f->b4_stride + t->bx]; -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ +#define set_ctx(rep_macro) \ for (int y = 0; y < bh4; y++) { \ - rep_macro(type, seg_ptr, 0, mul * b->seg_id); \ + rep_macro(seg_ptr, 0, b->seg_id); \ seg_ptr += f->b4_stride; \ } - case_set(bw4, NULL, 0, 0); + case_set(b_dim[2]); #undef set_ctx } if (!b->skip) { @@ -2398,10 +2379,10 @@ static int decode_sb(Dav1dTaskContext *const t, const enum BlockLevel bl, } if (t->frame_thread.pass != 2 && (bp != PARTITION_SPLIT || bl == BL_8X8)) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->a->partition, bx8, mul * dav1d_al_part_ctx[0][bl][bp]); \ - rep_macro(type, t->l.partition, by8, mul * dav1d_al_part_ctx[1][bl][bp]) - case_set_upto16(hsz,,,); +#define set_ctx(rep_macro) \ + rep_macro(t->a->partition, bx8, dav1d_al_part_ctx[0][bl][bp]); \ + rep_macro(t->l.partition, by8, dav1d_al_part_ctx[1][bl][bp]) + case_set_upto16(ulog2(hsz)); #undef set_ctx } diff --git a/src/lf_mask.c b/src/lf_mask.c index 09a5c532..c81bd9b5 100644 --- a/src/lf_mask.c +++ b/src/lf_mask.c @@ -64,18 +64,15 @@ static void decomp_tx(uint8_t (*const txa)[2 /* txsz, step */][32 /* y */][32 /* } else { const int lw = imin(2, t_dim->lw), lh = imin(2, t_dim->lh); -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ +#define set_ctx(rep_macro) \ for (int y = 0; y < t_dim->h; y++) { \ - rep_macro(type, txa[0][0][y], off, mul * lw); \ - rep_macro(type, txa[1][0][y], off, mul * lh); \ + rep_macro(txa[0][0][y], 0, lw); \ + rep_macro(txa[1][0][y], 0, lh); \ txa[0][1][y][0] = t_dim->w; \ } - case_set_upto16(t_dim->w,,, 0); -#undef set_ctx -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, txa[1][1][0], off, mul * t_dim->h) - case_set_upto16(t_dim->w,,, 0); + case_set_upto16(t_dim->lw); #undef set_ctx + dav1d_memset_pow2[t_dim->lw](txa[1][1][0], t_dim->h); } } @@ -196,20 +193,8 @@ static inline void mask_edges_intra(uint16_t (*const masks)[32][3][2], if (inner2) masks[1][by4 + y][thl4c][1] |= inner2; } -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, a, off, mul * thl4c) -#define default_memset(dir, diridx, off, var) \ - memset(a, thl4c, var) - case_set_upto32_with_default(w4,,, 0); -#undef default_memset -#undef set_ctx -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, l, off, mul * twl4c) -#define default_memset(dir, diridx, off, var) \ - memset(l, twl4c, var) - case_set_upto32_with_default(h4,,, 0); -#undef default_memset -#undef set_ctx + dav1d_memset_likely_pow2(a, thl4c, w4); + dav1d_memset_likely_pow2(l, twl4c, h4); } static void mask_edges_chroma(uint16_t (*const masks)[32][2][2], @@ -267,20 +252,8 @@ static void mask_edges_chroma(uint16_t (*const masks)[32][2][2], } } -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, a, off, mul * thl4c) -#define default_memset(dir, diridx, off, var) \ - memset(a, thl4c, var) - case_set_upto32_with_default(cw4,,, 0); -#undef default_memset -#undef set_ctx -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, l, off, mul * twl4c) -#define default_memset(dir, diridx, off, var) \ - memset(l, twl4c, var) - case_set_upto32_with_default(ch4,,, 0); -#undef default_memset -#undef set_ctx + dav1d_memset_likely_pow2(a, thl4c, cw4); + dav1d_memset_likely_pow2(l, twl4c, ch4); } void dav1d_create_lf_mask_intra(Av1Filter *const lflvl, diff --git a/src/meson.build b/src/meson.build index 9f2822b5..7b66517f 100644 --- a/src/meson.build +++ b/src/meson.build @@ -30,6 +30,7 @@ libdav1d_sources = files( 'cdf.c', 'cpu.c', + 'ctx.c', 'data.c', 'decode.c', 'dequant_tables.c', diff --git a/src/recon_tmpl.c b/src/recon_tmpl.c index 6abe6684..426fa406 100644 --- a/src/recon_tmpl.c +++ b/src/recon_tmpl.c @@ -793,21 +793,15 @@ static void read_coef_tree(Dav1dTaskContext *const t, if (DEBUG_BLOCK_INFO) printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", ytx, txtp, eob, ts->msac.rng); -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir lcoef, off, mul * cf_ctx) -#define default_memset(dir, diridx, off, sz) \ - memset(&t->dir lcoef[off], cf_ctx, sz) - case_set_upto16_with_default(imin(txh, f->bh - t->by), l., 1, by4); - case_set_upto16_with_default(imin(txw, f->bw - t->bx), a->, 0, bx4); -#undef default_memset -#undef set_ctx -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + dav1d_memset_likely_pow2(&t->a->lcoef[bx4], cf_ctx, imin(txw, f->bw - t->bx)); + dav1d_memset_likely_pow2(&t->l.lcoef[by4], cf_ctx, imin(txh, f->bh - t->by)); +#define set_ctx(rep_macro) \ for (int y = 0; y < txh; y++) { \ - rep_macro(type, txtp_map, 0, mul * txtp); \ + rep_macro(txtp_map, 0, txtp); \ txtp_map += 32; \ } uint8_t *txtp_map = &t->scratch.txtp_map[by4 * 32 + bx4]; - case_set_upto16(txw,,,); + case_set_upto16(t_dim->lw); #undef set_ctx if (t->frame_thread.pass == 1) *ts->frame_thread[1].cbi++ = eob * (1 << 5) + txtp; @@ -846,18 +840,16 @@ void bytefn(dav1d_read_coef_blocks)(Dav1dTaskContext *const t, (bh4 > ss_ver || t->by & 1); if (b->skip) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir lcoef, off, mul * 0x40) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); -#undef set_ctx + BlockContext *const a = t->a; + dav1d_memset_pow2[b_dim[2]](&a->lcoef[bx4], 0x40); + dav1d_memset_pow2[b_dim[3]](&t->l.lcoef[by4], 0x40); if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir ccoef[0], off, mul * 0x40); \ - rep_macro(type, t->dir ccoef[1], off, mul * 0x40) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + dav1d_memset_pow2_fn memset_cw = dav1d_memset_pow2[ulog2(cbw4)]; + dav1d_memset_pow2_fn memset_ch = dav1d_memset_pow2[ulog2(cbh4)]; + memset_cw(&a->ccoef[0][cbx4], 0x40); + memset_cw(&a->ccoef[1][cbx4], 0x40); + memset_ch(&t->l.ccoef[0][cby4], 0x40); + memset_ch(&t->l.ccoef[1][cby4], 0x40); } return; } @@ -898,16 +890,8 @@ void bytefn(dav1d_read_coef_blocks)(Dav1dTaskContext *const t, b->tx, txtp, eob, ts->msac.rng); *ts->frame_thread[1].cbi++ = eob * (1 << 5) + txtp; ts->frame_thread[1].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir lcoef, off, mul * cf_ctx) -#define default_memset(dir, diridx, off, sz) \ - memset(&t->dir lcoef[off], cf_ctx, sz) - case_set_upto16_with_default(imin(t_dim->h, f->bh - t->by), - l., 1, by4 + y); - case_set_upto16_with_default(imin(t_dim->w, f->bw - t->bx), - a->, 0, bx4 + x); -#undef default_memset -#undef set_ctx + dav1d_memset_likely_pow2(&t->a->lcoef[bx4 + x], cf_ctx, imin(t_dim->w, f->bw - t->bx)); + dav1d_memset_likely_pow2(&t->l.lcoef[by4 + y], cf_ctx, imin(t_dim->h, f->bh - t->by)); } } t->bx -= x; @@ -941,18 +925,10 @@ void bytefn(dav1d_read_coef_blocks)(Dav1dTaskContext *const t, pl, b->uvtx, txtp, eob, ts->msac.rng); *ts->frame_thread[1].cbi++ = eob * (1 << 5) + txtp; ts->frame_thread[1].cf += uv_t_dim->w * uv_t_dim->h * 16; -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) -#define default_memset(dir, diridx, off, sz) \ - memset(&t->dir ccoef[pl][off], cf_ctx, sz) - case_set_upto16_with_default( \ - imin(uv_t_dim->h, (f->bh - t->by + ss_ver) >> ss_ver), - l., 1, cby4 + y); - case_set_upto16_with_default( \ - imin(uv_t_dim->w, (f->bw - t->bx + ss_hor) >> ss_hor), - a->, 0, cbx4 + x); -#undef default_memset -#undef set_ctx + int ctw = imin(uv_t_dim->w, (f->bw - t->bx + ss_hor) >> ss_hor); + int cth = imin(uv_t_dim->h, (f->bh - t->by + ss_ver) >> ss_ver); + dav1d_memset_likely_pow2(&t->a->ccoef[pl][cbx4 + x], cf_ctx, ctw); + dav1d_memset_likely_pow2(&t->l.ccoef[pl][cby4 + y], cf_ctx, cth); } t->bx -= x << ss_hor; } @@ -1337,16 +1313,8 @@ void bytefn(dav1d_recon_b_intra)(Dav1dTaskContext *const t, const enum BlockSize if (DEBUG_BLOCK_INFO) printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", b->tx, txtp, eob, ts->msac.rng); -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir lcoef, off, mul * cf_ctx) -#define default_memset(dir, diridx, off, sz) \ - memset(&t->dir lcoef[off], cf_ctx, sz) - case_set_upto16_with_default(imin(t_dim->h, f->bh - t->by), \ - l., 1, by4 + y); - case_set_upto16_with_default(imin(t_dim->w, f->bw - t->bx), \ - a->, 0, bx4 + x); -#undef default_memset -#undef set_ctx + dav1d_memset_likely_pow2(&t->a->lcoef[bx4 + x], cf_ctx, imin(t_dim->w, f->bw - t->bx)); + dav1d_memset_likely_pow2(&t->l.lcoef[by4 + y], cf_ctx, imin(t_dim->h, f->bh - t->by)); } if (eob >= 0) { if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) @@ -1361,11 +1329,8 @@ void bytefn(dav1d_recon_b_intra)(Dav1dTaskContext *const t, const enum BlockSize t_dim->w * 4, t_dim->h * 4, "recon"); } } else if (!t->frame_thread.pass) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir lcoef, off, mul * 0x40) - case_set_upto16(t_dim->h, l., 1, by4 + y); - case_set_upto16(t_dim->w, a->, 0, bx4 + x); -#undef set_ctx + dav1d_memset_pow2[t_dim->lw](&t->a->lcoef[bx4 + x], 0x40); + dav1d_memset_pow2[t_dim->lh](&t->l.lcoef[by4 + y], 0x40); } dst += 4 * t_dim->w; } @@ -1562,18 +1527,10 @@ void bytefn(dav1d_recon_b_intra)(Dav1dTaskContext *const t, const enum BlockSize printf("Post-uv-cf-blk[pl=%d,tx=%d," "txtp=%d,eob=%d]: r=%d [x=%d,cbx4=%d]\n", pl, b->uvtx, txtp, eob, ts->msac.rng, x, cbx4); -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) -#define default_memset(dir, diridx, off, sz) \ - memset(&t->dir ccoef[pl][off], cf_ctx, sz) - case_set_upto16_with_default( \ - imin(uv_t_dim->h, (f->bh - t->by + ss_ver) >> ss_ver), - l., 1, cby4 + y); - case_set_upto16_with_default( \ - imin(uv_t_dim->w, (f->bw - t->bx + ss_hor) >> ss_hor), - a->, 0, cbx4 + x); -#undef default_memset -#undef set_ctx + int ctw = imin(uv_t_dim->w, (f->bw - t->bx + ss_hor) >> ss_hor); + int cth = imin(uv_t_dim->h, (f->bh - t->by + ss_ver) >> ss_ver); + dav1d_memset_likely_pow2(&t->a->ccoef[pl][cbx4 + x], cf_ctx, ctw); + dav1d_memset_likely_pow2(&t->l.ccoef[pl][cby4 + y], cf_ctx, cth); } if (eob >= 0) { if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) @@ -1587,11 +1544,8 @@ void bytefn(dav1d_recon_b_intra)(Dav1dTaskContext *const t, const enum BlockSize uv_t_dim->h * 4, "recon"); } } else if (!t->frame_thread.pass) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir ccoef[pl], off, mul * 0x40) - case_set_upto16(uv_t_dim->h, l., 1, cby4 + y); - case_set_upto16(uv_t_dim->w, a->, 0, cbx4 + x); -#undef set_ctx + dav1d_memset_pow2[uv_t_dim->lw](&t->a->ccoef[pl][cbx4 + x], 0x40); + dav1d_memset_pow2[uv_t_dim->lh](&t->l.ccoef[pl][cby4 + y], 0x40); } dst += uv_t_dim->w * 4; } @@ -1929,18 +1883,16 @@ int bytefn(dav1d_recon_b_inter)(Dav1dTaskContext *const t, const enum BlockSize if (b->skip) { // reset coef contexts -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir lcoef, off, mul * 0x40) - case_set(bh4, l., 1, by4); - case_set(bw4, a->, 0, bx4); -#undef set_ctx + BlockContext *const a = t->a; + dav1d_memset_pow2[b_dim[2]](&a->lcoef[bx4], 0x40); + dav1d_memset_pow2[b_dim[3]](&t->l.lcoef[by4], 0x40); if (has_chroma) { -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir ccoef[0], off, mul * 0x40); \ - rep_macro(type, t->dir ccoef[1], off, mul * 0x40) - case_set(cbh4, l., 1, cby4); - case_set(cbw4, a->, 0, cbx4); -#undef set_ctx + dav1d_memset_pow2_fn memset_cw = dav1d_memset_pow2[ulog2(cbw4)]; + dav1d_memset_pow2_fn memset_ch = dav1d_memset_pow2[ulog2(cbh4)]; + memset_cw(&a->ccoef[0][cbx4], 0x40); + memset_cw(&a->ccoef[1][cbx4], 0x40); + memset_ch(&t->l.ccoef[0][cby4], 0x40); + memset_ch(&t->l.ccoef[1][cby4], 0x40); } return 0; } @@ -2006,18 +1958,10 @@ int bytefn(dav1d_recon_b_inter)(Dav1dTaskContext *const t, const enum BlockSize printf("Post-uv-cf-blk[pl=%d,tx=%d," "txtp=%d,eob=%d]: r=%d\n", pl, b->uvtx, txtp, eob, ts->msac.rng); -#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ - rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) -#define default_memset(dir, diridx, off, sz) \ - memset(&t->dir ccoef[pl][off], cf_ctx, sz) - case_set_upto16_with_default( \ - imin(uvtx->h, (f->bh - t->by + ss_ver) >> ss_ver), - l., 1, cby4 + y); - case_set_upto16_with_default( \ - imin(uvtx->w, (f->bw - t->bx + ss_hor) >> ss_hor), - a->, 0, cbx4 + x); -#undef default_memset -#undef set_ctx + int ctw = imin(uvtx->w, (f->bw - t->bx + ss_hor) >> ss_hor); + int cth = imin(uvtx->h, (f->bh - t->by + ss_ver) >> ss_ver); + dav1d_memset_likely_pow2(&t->a->ccoef[pl][cbx4 + x], cf_ctx, ctw); + dav1d_memset_likely_pow2(&t->l.ccoef[pl][cby4 + y], cf_ctx, cth); } if (eob >= 0) { if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS)