diff mbox series

[v6,13/35] target/arm: Implement SVE gather loads

Message ID 20180627043328.11531-14-richard.henderson@linaro.org
State New
Headers show
Series target/arm SVE patches | expand

Commit Message

Richard Henderson June 27, 2018, 4:33 a.m. UTC 
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>


---
v6:
  * Finish esz == msz && u==1 decode in sve.decode.
  * Remove duplicate decode in trans_ST1_zprz.
  * Add xs=2 comment.
  * Reformat tables to leave room for ff helpers.
---
 target/arm/helper-sve.h    |  67 +++++++++++++++++++++++++
 target/arm/sve_helper.c    |  77 ++++++++++++++++++++++++++++
 target/arm/translate-sve.c | 100 +++++++++++++++++++++++++++++++++++++
 target/arm/sve.decode      |  57 +++++++++++++++++++++
 4 files changed, 301 insertions(+)

-- 
2.17.1

Comments

Peter Maydell June 28, 2018, 10:37 a.m. UTC | #1 
On 27 June 2018 at 05:33, Richard Henderson
<richard.henderson@linaro.org> wrote:
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

>

> ---

> v6:

>   * Finish esz == msz && u==1 decode in sve.decode.

>   * Remove duplicate decode in trans_ST1_zprz.

>   * Add xs=2 comment.

>   * Reformat tables to leave room for ff helpers.

> ---

>  target/arm/helper-sve.h    |  67 +++++++++++++++++++++++++

>  target/arm/sve_helper.c    |  77 ++++++++++++++++++++++++++++

>  target/arm/translate-sve.c | 100 +++++++++++++++++++++++++++++++++++++

>  target/arm/sve.decode      |  57 +++++++++++++++++++++

>  4 files changed, 301 insertions(+)

>


Reviewed-by: Peter Maydell <peter.maydell@linaro.org>


thanks
-- PMM
Alex Bennée June 28, 2018, 1:18 p.m. UTC | #2 
Richard Henderson <richard.henderson@linaro.org> writes:

> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>


Reviewed-by: Alex Bennée <alex.bennee@linaro.org>


>

> ---

> v6:

>   * Finish esz == msz && u==1 decode in sve.decode.

>   * Remove duplicate decode in trans_ST1_zprz.

>   * Add xs=2 comment.

>   * Reformat tables to leave room for ff helpers.

> ---

>  target/arm/helper-sve.h    |  67 +++++++++++++++++++++++++

>  target/arm/sve_helper.c    |  77 ++++++++++++++++++++++++++++

>  target/arm/translate-sve.c | 100 +++++++++++++++++++++++++++++++++++++

>  target/arm/sve.decode      |  57 +++++++++++++++++++++

>  4 files changed, 301 insertions(+)

>

> diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h

> index 8880128f9c..aeb62afc34 100644

> --- a/target/arm/helper-sve.h

> +++ b/target/arm/helper-sve.h

> @@ -959,6 +959,73 @@ DEF_HELPER_FLAGS_4(sve_st1hd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

>

>  DEF_HELPER_FLAGS_4(sve_st1sd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)

>

> +DEF_HELPER_FLAGS_6(sve_ldbsu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhsu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldssu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldbss_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhss_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +

> +DEF_HELPER_FLAGS_6(sve_ldbsu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhsu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldssu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldbss_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhss_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +

> +DEF_HELPER_FLAGS_6(sve_ldbdu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhdu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldsdu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldddu_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldbds_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhds_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldsds_zsu, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +

> +DEF_HELPER_FLAGS_6(sve_ldbdu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhdu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldsdu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldddu_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldbds_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhds_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldsds_zss, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +

> +DEF_HELPER_FLAGS_6(sve_ldbdu_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhdu_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldsdu_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldddu_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldbds_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldhds_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +DEF_HELPER_FLAGS_6(sve_ldsds_zd, TCG_CALL_NO_WG,

> +                   void, env, ptr, ptr, ptr, tl, i32)

> +

>  DEF_HELPER_FLAGS_6(sve_stbs_zsu, TCG_CALL_NO_WG,

>                     void, env, ptr, ptr, ptr, tl, i32)

>  DEF_HELPER_FLAGS_6(sve_sths_zsu, TCG_CALL_NO_WG,

> diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c

> index 7622bb2af0..24f75a32d3 100644

> --- a/target/arm/sve_helper.c

> +++ b/target/arm/sve_helper.c

> @@ -3714,6 +3714,83 @@ void HELPER(sve_st4dd_r)(CPUARMState *env, void *vg,

>      }

>  }

>

> +/* Loads with a vector index.  */

> +

> +#define DO_LD1_ZPZ_S(NAME, TYPEI, TYPEM, FN)                            \

> +void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm,       \

> +                  target_ulong base, uint32_t desc)                     \

> +{                                                                       \

> +    intptr_t i, oprsz = simd_oprsz(desc);                               \

> +    unsigned scale = simd_data(desc);                                   \

> +    uintptr_t ra = GETPC();                                             \

> +    for (i = 0; i < oprsz; i++) {                                       \

> +        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));                 \

> +        do {                                                            \

> +            TYPEM m = 0;                                                \

> +            if (pg & 1) {                                               \

> +                target_ulong off = *(TYPEI *)(vm + H1_4(i));            \

> +                m = FN(env, base + (off << scale), ra);                 \

> +            }                                                           \

> +            *(uint32_t *)(vd + H1_4(i)) = m;                            \

> +            i += 4, pg >>= 4;                                           \

> +        } while (i & 15);                                               \

> +    }                                                                   \

> +}

> +

> +#define DO_LD1_ZPZ_D(NAME, TYPEI, TYPEM, FN)                            \

> +void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm,       \

> +                  target_ulong base, uint32_t desc)                     \

> +{                                                                       \

> +    intptr_t i, oprsz = simd_oprsz(desc) / 8;                           \

> +    unsigned scale = simd_data(desc);                                   \

> +    uintptr_t ra = GETPC();                                             \

> +    uint64_t *d = vd, *m = vm; uint8_t *pg = vg;                        \

> +    for (i = 0; i < oprsz; i++) {                                       \

> +        TYPEM mm = 0;                                                   \

> +        if (pg[H1(i)] & 1) {                                            \

> +            target_ulong off = (TYPEI)m[i];                             \

> +            mm = FN(env, base + (off << scale), ra);                    \

> +        }                                                               \

> +        d[i] = mm;                                                      \

> +    }                                                                   \

> +}

> +

> +DO_LD1_ZPZ_S(sve_ldbsu_zsu, uint32_t, uint8_t,  cpu_ldub_data_ra)

> +DO_LD1_ZPZ_S(sve_ldhsu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)

> +DO_LD1_ZPZ_S(sve_ldssu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)

> +DO_LD1_ZPZ_S(sve_ldbss_zsu, uint32_t, int8_t,   cpu_ldub_data_ra)

> +DO_LD1_ZPZ_S(sve_ldhss_zsu, uint32_t, int16_t,  cpu_lduw_data_ra)

> +

> +DO_LD1_ZPZ_S(sve_ldbsu_zss, int32_t, uint8_t,  cpu_ldub_data_ra)

> +DO_LD1_ZPZ_S(sve_ldhsu_zss, int32_t, uint16_t, cpu_lduw_data_ra)

> +DO_LD1_ZPZ_S(sve_ldssu_zss, int32_t, uint32_t, cpu_ldl_data_ra)

> +DO_LD1_ZPZ_S(sve_ldbss_zss, int32_t, int8_t,   cpu_ldub_data_ra)

> +DO_LD1_ZPZ_S(sve_ldhss_zss, int32_t, int16_t,  cpu_lduw_data_ra)

> +

> +DO_LD1_ZPZ_D(sve_ldbdu_zsu, uint32_t, uint8_t,  cpu_ldub_data_ra)

> +DO_LD1_ZPZ_D(sve_ldhdu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)

> +DO_LD1_ZPZ_D(sve_ldsdu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)

> +DO_LD1_ZPZ_D(sve_ldddu_zsu, uint32_t, uint64_t, cpu_ldq_data_ra)

> +DO_LD1_ZPZ_D(sve_ldbds_zsu, uint32_t, int8_t,   cpu_ldub_data_ra)

> +DO_LD1_ZPZ_D(sve_ldhds_zsu, uint32_t, int16_t,  cpu_lduw_data_ra)

> +DO_LD1_ZPZ_D(sve_ldsds_zsu, uint32_t, int32_t,  cpu_ldl_data_ra)

> +

> +DO_LD1_ZPZ_D(sve_ldbdu_zss, int32_t, uint8_t,  cpu_ldub_data_ra)

> +DO_LD1_ZPZ_D(sve_ldhdu_zss, int32_t, uint16_t, cpu_lduw_data_ra)

> +DO_LD1_ZPZ_D(sve_ldsdu_zss, int32_t, uint32_t, cpu_ldl_data_ra)

> +DO_LD1_ZPZ_D(sve_ldddu_zss, int32_t, uint64_t, cpu_ldq_data_ra)

> +DO_LD1_ZPZ_D(sve_ldbds_zss, int32_t, int8_t,   cpu_ldub_data_ra)

> +DO_LD1_ZPZ_D(sve_ldhds_zss, int32_t, int16_t,  cpu_lduw_data_ra)

> +DO_LD1_ZPZ_D(sve_ldsds_zss, int32_t, int32_t,  cpu_ldl_data_ra)

> +

> +DO_LD1_ZPZ_D(sve_ldbdu_zd, uint64_t, uint8_t,  cpu_ldub_data_ra)

> +DO_LD1_ZPZ_D(sve_ldhdu_zd, uint64_t, uint16_t, cpu_lduw_data_ra)

> +DO_LD1_ZPZ_D(sve_ldsdu_zd, uint64_t, uint32_t, cpu_ldl_data_ra)

> +DO_LD1_ZPZ_D(sve_ldddu_zd, uint64_t, uint64_t, cpu_ldq_data_ra)

> +DO_LD1_ZPZ_D(sve_ldbds_zd, uint64_t, int8_t,   cpu_ldub_data_ra)

> +DO_LD1_ZPZ_D(sve_ldhds_zd, uint64_t, int16_t,  cpu_lduw_data_ra)

> +DO_LD1_ZPZ_D(sve_ldsds_zd, uint64_t, int32_t,  cpu_ldl_data_ra)

> +

>  /* Stores with a vector index.  */

>

>  #define DO_ST1_ZPZ_S(NAME, TYPEI, FN)                                   \

> diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c

> index 27854e0042..33ffb217d0 100644

> --- a/target/arm/translate-sve.c

> +++ b/target/arm/translate-sve.c

> @@ -4255,6 +4255,106 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, int scale,

>      tcg_temp_free_i32(desc);

>  }

>

> +/* Indexed by [ff][xs][u][msz].  */

> +static gen_helper_gvec_mem_scatter * const gather_load_fn32[2][2][2][3] = {

> +    { { { gen_helper_sve_ldbss_zsu,

> +          gen_helper_sve_ldhss_zsu,

> +          NULL, },

> +        { gen_helper_sve_ldbsu_zsu,

> +          gen_helper_sve_ldhsu_zsu,

> +          gen_helper_sve_ldssu_zsu, } },

> +      { { gen_helper_sve_ldbss_zss,

> +          gen_helper_sve_ldhss_zss,

> +          NULL, },

> +        { gen_helper_sve_ldbsu_zss,

> +          gen_helper_sve_ldhsu_zss,

> +          gen_helper_sve_ldssu_zss, } } },

> +    /* TODO fill in first-fault handlers */

> +};

> +

> +/* Note that we overload xs=2 to indicate 64-bit offset.  */

> +static gen_helper_gvec_mem_scatter * const gather_load_fn64[2][3][2][4] = {

> +    { { { gen_helper_sve_ldbds_zsu,

> +          gen_helper_sve_ldhds_zsu,

> +          gen_helper_sve_ldsds_zsu,

> +          NULL, },

> +        { gen_helper_sve_ldbdu_zsu,

> +          gen_helper_sve_ldhdu_zsu,

> +          gen_helper_sve_ldsdu_zsu,

> +          gen_helper_sve_ldddu_zsu, } },

> +      { { gen_helper_sve_ldbds_zss,

> +          gen_helper_sve_ldhds_zss,

> +          gen_helper_sve_ldsds_zss,

> +          NULL, },

> +        { gen_helper_sve_ldbdu_zss,

> +          gen_helper_sve_ldhdu_zss,

> +          gen_helper_sve_ldsdu_zss,

> +          gen_helper_sve_ldddu_zss, } },

> +      { { gen_helper_sve_ldbds_zd,

> +          gen_helper_sve_ldhds_zd,

> +          gen_helper_sve_ldsds_zd,

> +          NULL, },

> +        { gen_helper_sve_ldbdu_zd,

> +          gen_helper_sve_ldhdu_zd,

> +          gen_helper_sve_ldsdu_zd,

> +          gen_helper_sve_ldddu_zd, } } },

> +    /* TODO fill in first-fault handlers */

> +};

> +

> +static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a, uint32_t insn)

> +{

> +    gen_helper_gvec_mem_scatter *fn = NULL;

> +

> +    if (!sve_access_check(s)) {

> +        return true;

> +    }

> +

> +    switch (a->esz) {

> +    case MO_32:

> +        fn = gather_load_fn32[a->ff][a->xs][a->u][a->msz];

> +        break;

> +    case MO_64:

> +        fn = gather_load_fn64[a->ff][a->xs][a->u][a->msz];

> +        break;

> +    }

> +    assert(fn != NULL);

> +

> +    do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,

> +               cpu_reg_sp(s, a->rn), fn);

> +    return true;

> +}

> +

> +static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a, uint32_t insn)

> +{

> +    gen_helper_gvec_mem_scatter *fn = NULL;

> +    TCGv_i64 imm;

> +

> +    if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {

> +        return false;

> +    }

> +    if (!sve_access_check(s)) {

> +        return true;

> +    }

> +

> +    switch (a->esz) {

> +    case MO_32:

> +        fn = gather_load_fn32[a->ff][0][a->u][a->msz];

> +        break;

> +    case MO_64:

> +        fn = gather_load_fn64[a->ff][2][a->u][a->msz];

> +        break;

> +    }

> +    assert(fn != NULL);

> +

> +    /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])

> +     * by loading the immediate into the scalar parameter.

> +     */

> +    imm = tcg_const_i64(a->imm << a->msz);

> +    do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, fn);

> +    tcg_temp_free_i64(imm);

> +    return true;

> +}

> +

>  static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a, uint32_t insn)

>  {

>      /* Indexed by [xs][msz].  */

> diff --git a/target/arm/sve.decode b/target/arm/sve.decode

> index 80b955ff84..45016c6042 100644

> --- a/target/arm/sve.decode

> +++ b/target/arm/sve.decode

> @@ -80,6 +80,8 @@

>  &rpri_load      rd pg rn imm dtype nreg

>  &rprr_store     rd pg rn rm msz esz nreg

>  &rpri_store     rd pg rn imm msz esz nreg

> +&rprr_gather_load       rd pg rn rm esz msz u ff xs scale

> +&rpri_gather_load       rd pg rn imm esz msz u ff

>  &rprr_scatter_store     rd pg rn rm esz msz xs scale

>

>  ###########################################################################

> @@ -194,6 +196,22 @@

>  @rpri_load_msz  ....... .... . imm:s4 ... pg:3 rn:5 rd:5 \

>                  &rpri_load dtype=%msz_dtype

>

> +# Gather Loads.

> +@rprr_g_load_u        ....... .. .    . rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \

> +                      &rprr_gather_load xs=2

> +@rprr_g_load_xs_u     ....... .. xs:1 . rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \

> +                      &rprr_gather_load

> +@rprr_g_load_xs_u_sc  ....... .. xs:1 scale:1 rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \

> +                      &rprr_gather_load

> +@rprr_g_load_xs_sc    ....... .. xs:1 scale:1 rm:5 . . ff:1 pg:3 rn:5 rd:5 \

> +                      &rprr_gather_load

> +@rprr_g_load_u_sc     ....... .. .    scale:1 rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \

> +                      &rprr_gather_load xs=2

> +@rprr_g_load_sc       ....... .. .    scale:1 rm:5 . . ff:1 pg:3 rn:5 rd:5 \

> +                      &rprr_gather_load xs=2

> +@rpri_g_load          ....... msz:2 .. imm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \

> +                      &rpri_gather_load

> +

>  # Stores; user must fill in ESZ, MSZ, NREG as needed.

>  @rprr_store         ....... ..    ..     rm:5 ... pg:3 rn:5 rd:5    &rprr_store

>  @rpri_store_msz     ....... msz:2 .. . imm:s4 ... pg:3 rn:5 rd:5    &rpri_store

> @@ -759,6 +777,19 @@ LDR_zri         10000101 10 ...... 010 ... ..... .....          @rd_rn_i9

>  LD1R_zpri       1000010 .. 1 imm:6 1.. pg:3 rn:5 rd:5 \

>                  &rpri_load dtype=%dtype_23_13 nreg=0

>

> +# SVE 32-bit gather load (scalar plus 32-bit unscaled offsets)

> +# SVE 32-bit gather load (scalar plus 32-bit scaled offsets)

> +LD1_zprz        1000010 00 .0 ..... 0.. ... ..... ..... \

> +                @rprr_g_load_xs_u esz=2 msz=0 scale=0

> +LD1_zprz        1000010 01 .. ..... 0.. ... ..... ..... \

> +                @rprr_g_load_xs_u_sc esz=2 msz=1

> +LD1_zprz        1000010 10 .. ..... 01. ... ..... ..... \

> +                @rprr_g_load_xs_sc esz=2 msz=2 u=1

> +

> +# SVE 32-bit gather load (vector plus immediate)

> +LD1_zpiz        1000010 .. 01 ..... 1.. ... ..... ..... \

> +                @rpri_g_load esz=2

> +

>  ### SVE Memory Contiguous Load Group

>

>  # SVE contiguous load (scalar plus scalar)

> @@ -808,6 +839,32 @@ PRF_rr          1000010 -- 00 rm:5 110 --- ----- 0 ----

>

>  ### SVE Memory 64-bit Gather Group

>

> +# SVE 64-bit gather load (scalar plus 32-bit unpacked unscaled offsets)

> +# SVE 64-bit gather load (scalar plus 32-bit unpacked scaled offsets)

> +LD1_zprz        1100010 00 .0 ..... 0.. ... ..... ..... \

> +                @rprr_g_load_xs_u esz=3 msz=0 scale=0

> +LD1_zprz        1100010 01 .. ..... 0.. ... ..... ..... \

> +                @rprr_g_load_xs_u_sc esz=3 msz=1

> +LD1_zprz        1100010 10 .. ..... 0.. ... ..... ..... \

> +                @rprr_g_load_xs_u_sc esz=3 msz=2

> +LD1_zprz        1100010 11 .. ..... 01. ... ..... ..... \

> +                @rprr_g_load_xs_sc esz=3 msz=3 u=1

> +

> +# SVE 64-bit gather load (scalar plus 64-bit unscaled offsets)

> +# SVE 64-bit gather load (scalar plus 64-bit scaled offsets)

> +LD1_zprz        1100010 00 10 ..... 1.. ... ..... ..... \

> +                @rprr_g_load_u esz=3 msz=0 scale=0

> +LD1_zprz        1100010 01 1. ..... 1.. ... ..... ..... \

> +                @rprr_g_load_u_sc esz=3 msz=1

> +LD1_zprz        1100010 10 1. ..... 1.. ... ..... ..... \

> +                @rprr_g_load_u_sc esz=3 msz=2

> +LD1_zprz        1100010 11 1. ..... 11. ... ..... ..... \

> +                @rprr_g_load_sc esz=3 msz=3 u=1

> +

> +# SVE 64-bit gather load (vector plus immediate)

> +LD1_zpiz        1100010 .. 01 ..... 1.. ... ..... ..... \

> +                @rpri_g_load esz=3

> +

>  # SVE 64-bit gather prefetch (scalar plus 64-bit scaled offsets)

>  PRF             1100010 00 11 ----- 1-- --- ----- 0 ----



--
Alex Bennée
diff mbox series

Patch

diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h
index 8880128f9c..aeb62afc34 100644
--- a/target/arm/helper-sve.h
+++ b/target/arm/helper-sve.h
@@ -959,6 +959,73 @@  DEF_HELPER_FLAGS_4(sve_st1hd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 
 DEF_HELPER_FLAGS_4(sve_st1sd_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 
+DEF_HELPER_FLAGS_6(sve_ldbsu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhsu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldssu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbss_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhss_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbsu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhsu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldssu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbss_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhss_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbdu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhdu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsdu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldddu_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbds_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhds_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsds_zsu, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbdu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhdu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsdu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldddu_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbds_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhds_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsds_zss, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_6(sve_ldbdu_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhdu_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsdu_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldddu_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldbds_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldhds_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_6(sve_ldsds_zd, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, ptr, tl, i32)
+
 DEF_HELPER_FLAGS_6(sve_stbs_zsu, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 DEF_HELPER_FLAGS_6(sve_sths_zsu, TCG_CALL_NO_WG,
diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c
index 7622bb2af0..24f75a32d3 100644
--- a/target/arm/sve_helper.c
+++ b/target/arm/sve_helper.c
@@ -3714,6 +3714,83 @@  void HELPER(sve_st4dd_r)(CPUARMState *env, void *vg,
     }
 }
 
+/* Loads with a vector index.  */
+
+#define DO_LD1_ZPZ_S(NAME, TYPEI, TYPEM, FN)                            \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm,       \
+                  target_ulong base, uint32_t desc)                     \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    unsigned scale = simd_data(desc);                                   \
+    uintptr_t ra = GETPC();                                             \
+    for (i = 0; i < oprsz; i++) {                                       \
+        uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));                 \
+        do {                                                            \
+            TYPEM m = 0;                                                \
+            if (pg & 1) {                                               \
+                target_ulong off = *(TYPEI *)(vm + H1_4(i));            \
+                m = FN(env, base + (off << scale), ra);                 \
+            }                                                           \
+            *(uint32_t *)(vd + H1_4(i)) = m;                            \
+            i += 4, pg >>= 4;                                           \
+        } while (i & 15);                                               \
+    }                                                                   \
+}
+
+#define DO_LD1_ZPZ_D(NAME, TYPEI, TYPEM, FN)                            \
+void HELPER(NAME)(CPUARMState *env, void *vd, void *vg, void *vm,       \
+                  target_ulong base, uint32_t desc)                     \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;                           \
+    unsigned scale = simd_data(desc);                                   \
+    uintptr_t ra = GETPC();                                             \
+    uint64_t *d = vd, *m = vm; uint8_t *pg = vg;                        \
+    for (i = 0; i < oprsz; i++) {                                       \
+        TYPEM mm = 0;                                                   \
+        if (pg[H1(i)] & 1) {                                            \
+            target_ulong off = (TYPEI)m[i];                             \
+            mm = FN(env, base + (off << scale), ra);                    \
+        }                                                               \
+        d[i] = mm;                                                      \
+    }                                                                   \
+}
+
+DO_LD1_ZPZ_S(sve_ldbsu_zsu, uint32_t, uint8_t,  cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhsu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_S(sve_ldssu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_S(sve_ldbss_zsu, uint32_t, int8_t,   cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhss_zsu, uint32_t, int16_t,  cpu_lduw_data_ra)
+
+DO_LD1_ZPZ_S(sve_ldbsu_zss, int32_t, uint8_t,  cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhsu_zss, int32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_S(sve_ldssu_zss, int32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_S(sve_ldbss_zss, int32_t, int8_t,   cpu_ldub_data_ra)
+DO_LD1_ZPZ_S(sve_ldhss_zss, int32_t, int16_t,  cpu_lduw_data_ra)
+
+DO_LD1_ZPZ_D(sve_ldbdu_zsu, uint32_t, uint8_t,  cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhdu_zsu, uint32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsdu_zsu, uint32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_D(sve_ldddu_zsu, uint32_t, uint64_t, cpu_ldq_data_ra)
+DO_LD1_ZPZ_D(sve_ldbds_zsu, uint32_t, int8_t,   cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhds_zsu, uint32_t, int16_t,  cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsds_zsu, uint32_t, int32_t,  cpu_ldl_data_ra)
+
+DO_LD1_ZPZ_D(sve_ldbdu_zss, int32_t, uint8_t,  cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhdu_zss, int32_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsdu_zss, int32_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_D(sve_ldddu_zss, int32_t, uint64_t, cpu_ldq_data_ra)
+DO_LD1_ZPZ_D(sve_ldbds_zss, int32_t, int8_t,   cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhds_zss, int32_t, int16_t,  cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsds_zss, int32_t, int32_t,  cpu_ldl_data_ra)
+
+DO_LD1_ZPZ_D(sve_ldbdu_zd, uint64_t, uint8_t,  cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhdu_zd, uint64_t, uint16_t, cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsdu_zd, uint64_t, uint32_t, cpu_ldl_data_ra)
+DO_LD1_ZPZ_D(sve_ldddu_zd, uint64_t, uint64_t, cpu_ldq_data_ra)
+DO_LD1_ZPZ_D(sve_ldbds_zd, uint64_t, int8_t,   cpu_ldub_data_ra)
+DO_LD1_ZPZ_D(sve_ldhds_zd, uint64_t, int16_t,  cpu_lduw_data_ra)
+DO_LD1_ZPZ_D(sve_ldsds_zd, uint64_t, int32_t,  cpu_ldl_data_ra)
+
 /* Stores with a vector index.  */
 
 #define DO_ST1_ZPZ_S(NAME, TYPEI, FN)                                   \
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 27854e0042..33ffb217d0 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -4255,6 +4255,106 @@  static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, int scale,
     tcg_temp_free_i32(desc);
 }
 
+/* Indexed by [ff][xs][u][msz].  */
+static gen_helper_gvec_mem_scatter * const gather_load_fn32[2][2][2][3] = {
+    { { { gen_helper_sve_ldbss_zsu,
+          gen_helper_sve_ldhss_zsu,
+          NULL, },
+        { gen_helper_sve_ldbsu_zsu,
+          gen_helper_sve_ldhsu_zsu,
+          gen_helper_sve_ldssu_zsu, } },
+      { { gen_helper_sve_ldbss_zss,
+          gen_helper_sve_ldhss_zss,
+          NULL, },
+        { gen_helper_sve_ldbsu_zss,
+          gen_helper_sve_ldhsu_zss,
+          gen_helper_sve_ldssu_zss, } } },
+    /* TODO fill in first-fault handlers */
+};
+
+/* Note that we overload xs=2 to indicate 64-bit offset.  */
+static gen_helper_gvec_mem_scatter * const gather_load_fn64[2][3][2][4] = {
+    { { { gen_helper_sve_ldbds_zsu,
+          gen_helper_sve_ldhds_zsu,
+          gen_helper_sve_ldsds_zsu,
+          NULL, },
+        { gen_helper_sve_ldbdu_zsu,
+          gen_helper_sve_ldhdu_zsu,
+          gen_helper_sve_ldsdu_zsu,
+          gen_helper_sve_ldddu_zsu, } },
+      { { gen_helper_sve_ldbds_zss,
+          gen_helper_sve_ldhds_zss,
+          gen_helper_sve_ldsds_zss,
+          NULL, },
+        { gen_helper_sve_ldbdu_zss,
+          gen_helper_sve_ldhdu_zss,
+          gen_helper_sve_ldsdu_zss,
+          gen_helper_sve_ldddu_zss, } },
+      { { gen_helper_sve_ldbds_zd,
+          gen_helper_sve_ldhds_zd,
+          gen_helper_sve_ldsds_zd,
+          NULL, },
+        { gen_helper_sve_ldbdu_zd,
+          gen_helper_sve_ldhdu_zd,
+          gen_helper_sve_ldsdu_zd,
+          gen_helper_sve_ldddu_zd, } } },
+    /* TODO fill in first-fault handlers */
+};
+
+static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a, uint32_t insn)
+{
+    gen_helper_gvec_mem_scatter *fn = NULL;
+
+    if (!sve_access_check(s)) {
+        return true;
+    }
+
+    switch (a->esz) {
+    case MO_32:
+        fn = gather_load_fn32[a->ff][a->xs][a->u][a->msz];
+        break;
+    case MO_64:
+        fn = gather_load_fn64[a->ff][a->xs][a->u][a->msz];
+        break;
+    }
+    assert(fn != NULL);
+
+    do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
+               cpu_reg_sp(s, a->rn), fn);
+    return true;
+}
+
+static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a, uint32_t insn)
+{
+    gen_helper_gvec_mem_scatter *fn = NULL;
+    TCGv_i64 imm;
+
+    if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
+        return false;
+    }
+    if (!sve_access_check(s)) {
+        return true;
+    }
+
+    switch (a->esz) {
+    case MO_32:
+        fn = gather_load_fn32[a->ff][0][a->u][a->msz];
+        break;
+    case MO_64:
+        fn = gather_load_fn64[a->ff][2][a->u][a->msz];
+        break;
+    }
+    assert(fn != NULL);
+
+    /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
+     * by loading the immediate into the scalar parameter.
+     */
+    imm = tcg_const_i64(a->imm << a->msz);
+    do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, fn);
+    tcg_temp_free_i64(imm);
+    return true;
+}
+
 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a, uint32_t insn)
 {
     /* Indexed by [xs][msz].  */
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 80b955ff84..45016c6042 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -80,6 +80,8 @@ 
 &rpri_load      rd pg rn imm dtype nreg
 &rprr_store     rd pg rn rm msz esz nreg
 &rpri_store     rd pg rn imm msz esz nreg
+&rprr_gather_load       rd pg rn rm esz msz u ff xs scale
+&rpri_gather_load       rd pg rn imm esz msz u ff
 &rprr_scatter_store     rd pg rn rm esz msz xs scale
 
 ###########################################################################
@@ -194,6 +196,22 @@ 
 @rpri_load_msz  ....... .... . imm:s4 ... pg:3 rn:5 rd:5 \
                 &rpri_load dtype=%msz_dtype
 
+# Gather Loads.
+@rprr_g_load_u        ....... .. .    . rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+                      &rprr_gather_load xs=2
+@rprr_g_load_xs_u     ....... .. xs:1 . rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+                      &rprr_gather_load
+@rprr_g_load_xs_u_sc  ....... .. xs:1 scale:1 rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+                      &rprr_gather_load
+@rprr_g_load_xs_sc    ....... .. xs:1 scale:1 rm:5 . . ff:1 pg:3 rn:5 rd:5 \
+                      &rprr_gather_load
+@rprr_g_load_u_sc     ....... .. .    scale:1 rm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+                      &rprr_gather_load xs=2
+@rprr_g_load_sc       ....... .. .    scale:1 rm:5 . . ff:1 pg:3 rn:5 rd:5 \
+                      &rprr_gather_load xs=2
+@rpri_g_load          ....... msz:2 .. imm:5 . u:1 ff:1 pg:3 rn:5 rd:5 \
+                      &rpri_gather_load
+
 # Stores; user must fill in ESZ, MSZ, NREG as needed.
 @rprr_store         ....... ..    ..     rm:5 ... pg:3 rn:5 rd:5    &rprr_store
 @rpri_store_msz     ....... msz:2 .. . imm:s4 ... pg:3 rn:5 rd:5    &rpri_store
@@ -759,6 +777,19 @@  LDR_zri         10000101 10 ...... 010 ... ..... .....          @rd_rn_i9
 LD1R_zpri       1000010 .. 1 imm:6 1.. pg:3 rn:5 rd:5 \
                 &rpri_load dtype=%dtype_23_13 nreg=0
 
+# SVE 32-bit gather load (scalar plus 32-bit unscaled offsets)
+# SVE 32-bit gather load (scalar plus 32-bit scaled offsets)
+LD1_zprz        1000010 00 .0 ..... 0.. ... ..... ..... \
+                @rprr_g_load_xs_u esz=2 msz=0 scale=0
+LD1_zprz        1000010 01 .. ..... 0.. ... ..... ..... \
+                @rprr_g_load_xs_u_sc esz=2 msz=1
+LD1_zprz        1000010 10 .. ..... 01. ... ..... ..... \
+                @rprr_g_load_xs_sc esz=2 msz=2 u=1
+
+# SVE 32-bit gather load (vector plus immediate)
+LD1_zpiz        1000010 .. 01 ..... 1.. ... ..... ..... \
+                @rpri_g_load esz=2
+
 ### SVE Memory Contiguous Load Group
 
 # SVE contiguous load (scalar plus scalar)
@@ -808,6 +839,32 @@  PRF_rr          1000010 -- 00 rm:5 110 --- ----- 0 ----
 
 ### SVE Memory 64-bit Gather Group
 
+# SVE 64-bit gather load (scalar plus 32-bit unpacked unscaled offsets)
+# SVE 64-bit gather load (scalar plus 32-bit unpacked scaled offsets)
+LD1_zprz        1100010 00 .0 ..... 0.. ... ..... ..... \
+                @rprr_g_load_xs_u esz=3 msz=0 scale=0
+LD1_zprz        1100010 01 .. ..... 0.. ... ..... ..... \
+                @rprr_g_load_xs_u_sc esz=3 msz=1
+LD1_zprz        1100010 10 .. ..... 0.. ... ..... ..... \
+                @rprr_g_load_xs_u_sc esz=3 msz=2
+LD1_zprz        1100010 11 .. ..... 01. ... ..... ..... \
+                @rprr_g_load_xs_sc esz=3 msz=3 u=1
+
+# SVE 64-bit gather load (scalar plus 64-bit unscaled offsets)
+# SVE 64-bit gather load (scalar plus 64-bit scaled offsets)
+LD1_zprz        1100010 00 10 ..... 1.. ... ..... ..... \
+                @rprr_g_load_u esz=3 msz=0 scale=0
+LD1_zprz        1100010 01 1. ..... 1.. ... ..... ..... \
+                @rprr_g_load_u_sc esz=3 msz=1
+LD1_zprz        1100010 10 1. ..... 1.. ... ..... ..... \
+                @rprr_g_load_u_sc esz=3 msz=2
+LD1_zprz        1100010 11 1. ..... 11. ... ..... ..... \
+                @rprr_g_load_sc esz=3 msz=3 u=1
+
+# SVE 64-bit gather load (vector plus immediate)
+LD1_zpiz        1100010 .. 01 ..... 1.. ... ..... ..... \
+                @rpri_g_load esz=3
+
 # SVE 64-bit gather prefetch (scalar plus 64-bit scaled offsets)
 PRF             1100010 00 11 ----- 1-- --- ----- 0 ----