diff mbox series

[v2,28/30] tcg/i386: Support 128-bit load/store with have_atomic16

Message ID 20230216025739.1211680-29-richard.henderson@linaro.org
State New
Headers show
Series tcg: Improve atomicity support | expand

Commit Message

Richard Henderson Feb. 16, 2023, 2:57 a.m. UTC
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
 tcg/i386/tcg-target.h     |   3 +-
 tcg/i386/tcg-target.c.inc | 325 +++++++++++++++++++++++++++++++++++---
 2 files changed, 304 insertions(+), 24 deletions(-)
diff mbox series

Patch

diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index 6d8a536a32..37d8e70fdc 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -194,7 +194,8 @@  extern bool have_atomic16;
 #define TCG_TARGET_HAS_qemu_st8_i32     1
 #endif
 
-#define TCG_TARGET_HAS_qemu_ldst_i128   0
+#define TCG_TARGET_HAS_qemu_ldst_i128 \
+    (TCG_TARGET_REG_BITS == 64 && have_atomic16)
 
 /* We do not support older SSE systems, only beginning with AVX1.  */
 #define TCG_TARGET_HAS_v64              have_avx1
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index 6ee7bc5a9a..6fdf79020f 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -91,6 +91,8 @@  static const int tcg_target_reg_alloc_order[] = {
 #endif
 };
 
+#define TCG_TMP_VEC  TCG_REG_XMM5
+
 static const int tcg_target_call_iarg_regs[] = {
 #if TCG_TARGET_REG_BITS == 64
 #if defined(_WIN64)
@@ -347,6 +349,8 @@  static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
 #define OPC_PCMPGTW     (0x65 | P_EXT | P_DATA16)
 #define OPC_PCMPGTD     (0x66 | P_EXT | P_DATA16)
 #define OPC_PCMPGTQ     (0x37 | P_EXT38 | P_DATA16)
+#define OPC_PEXTRD      (0x16 | P_EXT3A | P_DATA16)
+#define OPC_PINSRD      (0x22 | P_EXT3A | P_DATA16)
 #define OPC_PMAXSB      (0x3c | P_EXT38 | P_DATA16)
 #define OPC_PMAXSW      (0xee | P_EXT | P_DATA16)
 #define OPC_PMAXSD      (0x3d | P_EXT38 | P_DATA16)
@@ -1730,8 +1734,7 @@  static void tcg_out_nopn(TCGContext *s, int n)
 }
 
 /* Test register R vs immediate bits I, setting Z flag for EQ/NE. */
-static void __attribute__((unused))
-tcg_out_testi(TCGContext *s, TCGReg r, uint32_t i)
+static void tcg_out_testi(TCGContext *s, TCGReg r, uint32_t i)
 {
     /*
      * This is used for testing alignment, so we can usually use testb.
@@ -1828,10 +1831,11 @@  static MemOp atom_and_align_for_opc(TCGContext *s, MemOp opc, MemOp *out_al)
  *                                  int mmu_idx, uintptr_t ra)
  */
 static void * const qemu_ld_helpers[MO_SIZE + 1] = {
-    [MO_UB] = helper_ldub_mmu,
-    [MO_UW] = helper_lduw_mmu,
-    [MO_UL] = helper_ldul_mmu,
-    [MO_UQ] = helper_ldq_mmu,
+    [MO_8] = helper_ldub_mmu,
+    [MO_16] = helper_lduw_mmu,
+    [MO_32] = helper_ldul_mmu,
+    [MO_64] = helper_ldq_mmu,
+    [MO_128] = helper_ld16_mmu,
 };
 
 /*
@@ -1839,10 +1843,11 @@  static void * const qemu_ld_helpers[MO_SIZE + 1] = {
  *                                  uintxx_t val, int mmu_idx, uintptr_t ra)
  */
 static void * const qemu_st_helpers[MO_SIZE + 1] = {
-    [MO_UB] = helper_stb_mmu,
-    [MO_UW] = helper_stw_mmu,
-    [MO_UL] = helper_stl_mmu,
-    [MO_UQ] = helper_stq_mmu,
+    [MO_8] = helper_stb_mmu,
+    [MO_16] = helper_stw_mmu,
+    [MO_32] = helper_stl_mmu,
+    [MO_64] = helper_stq_mmu,
+    [MO_128] = helper_st16_mmu,
 };
 
 /*
@@ -1870,6 +1875,13 @@  static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGType type,
     label->label_ptr[1] = label_ptr[1];
 }
 
+static void tcg_out_mov2_xchg(TCGContext *s, TCGType type1, TCGType type2,
+                              TCGReg dst1, TCGReg dst2)
+{
+    int w = (type1 == TCG_TYPE_I32 && type2 == TCG_TYPE_I32 ? 0 : P_REXW);
+    tcg_out_modrm(s, OPC_XCHG_EvGv + w, dst1, dst2);
+}
+
 /* Move src1 to dst1 and src2 to dst2, minding possible overlap. */
 static void tcg_out_mov2(TCGContext *s,
                          TCGType type1, TCGReg dst1, TCGReg src1,
@@ -1883,11 +1895,69 @@  static void tcg_out_mov2(TCGContext *s,
         tcg_out_mov(s, type1, dst1, src1);
     } else {
         /* dst1 == src2 && dst2 == src1 -> xchg. */
-        int w = (type1 == TCG_TYPE_I32 && type2 == TCG_TYPE_I32 ? 0 : P_REXW);
-        tcg_out_modrm(s, OPC_XCHG_EvGv + w, dst1, dst2);
+        tcg_out_mov2_xchg(s, type1, type2, dst1, dst2);
     }
 }
 
+/* Similarly for 3 pairs. */
+static void tcg_out_mov3(TCGContext *s,
+                         TCGType type1, TCGReg dst1, TCGReg src1,
+                         TCGType type2, TCGReg dst2, TCGReg src2,
+                         TCGType type3, TCGReg dst3, TCGReg src3)
+{
+    if (dst1 != src2 && dst1 != src3) {
+        tcg_out_mov(s, type1, dst1, src1);
+        tcg_out_mov2(s, type2, dst2, src2, type3, dst3, src3);
+        return;
+    }
+    if (dst2 != src2 && dst2 != src3) {
+        tcg_out_mov(s, type2, dst2, src2);
+        tcg_out_mov2(s, type1, dst1, src1, type3, dst3, src3);
+        return;
+    }
+    if (dst3 != src1 && dst3 != src2) {
+        tcg_out_mov(s, type3, dst3, src3);
+        tcg_out_mov2(s, type1, dst1, src1, type2, dst2, src2);
+        return;
+    }
+    /* Three-way overlap present, at least one xchg needed. */
+    if (dst1 == src2) {
+        tcg_out_mov2_xchg(s, type1, type2, src1, src2);
+        tcg_out_mov2(s, type2, dst2, src1, type3, dst3, src3);
+        return;
+    }
+    if (dst1 == src3) {
+        tcg_out_mov2_xchg(s, type1, type3, src1, src3);
+        tcg_out_mov2(s, type2, dst2, src2, type3, dst3, src1);
+        return;
+    }
+    g_assert_not_reached();
+}
+
+static void tcg_out_vec_to_pair(TCGContext *s, TCGType type,
+                                TCGReg l, TCGReg h, TCGReg v)
+{
+    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
+
+    /* vpmov{d,q} %v, %l */
+    tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, v, 0, l);
+    /* vpextr{d,q} $1, %v, %h */
+    tcg_out_vex_modrm(s, OPC_PEXTRD + rexw, v, 0, h);
+    tcg_out8(s, 1);
+}
+
+static void tcg_out_pair_to_vec(TCGContext *s, TCGType type,
+                                TCGReg v, TCGReg l, TCGReg h)
+{
+    int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
+
+    /* vmov{d,q} %l, %v */
+    tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, v, 0, l);
+    /* vpinsr{d,q} $1, %h, %v, %v */
+    tcg_out_vex_modrm(s, OPC_PINSRD + rexw, v, v, h);
+    tcg_out8(s, 1);
+}
+
 /*
  * Generate code for the slow path for a load at the end of block
  */
@@ -1897,7 +1967,7 @@  static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     MemOp opc = get_memop(oi);
     TCGReg data_reg;
     tcg_insn_unit **label_ptr = &l->label_ptr[0];
-    int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0);
+    int rexw = (l->type == TCG_TYPE_I32 ? 0 : P_REXW);
 
     /* resolve label address */
     tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
@@ -1961,6 +2031,22 @@  static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
                          TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
         }
         break;
+    case MO_128:
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+        switch (TCG_TARGET_CALL_RET_I128) {
+        case TCG_CALL_RET_NORMAL:
+            tcg_out_mov2(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX,
+                         TCG_TYPE_I64, l->datahi_reg, TCG_REG_RDX);
+            break;
+        case TCG_CALL_RET_BY_VEC:
+            tcg_out_vec_to_pair(s, TCG_TYPE_I64,
+                                data_reg, l->datahi_reg, TCG_REG_XMM0);
+            break;
+        default:
+            qemu_build_not_reached();
+        }
+        break;
+
     default:
         tcg_abort();
     }
@@ -1977,7 +2063,6 @@  static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
 {
     MemOpIdx oi = l->oi;
     MemOp opc = get_memop(oi);
-    MemOp s_bits = opc & MO_SIZE;
     tcg_insn_unit **label_ptr = &l->label_ptr[0];
     TCGReg retaddr;
 
@@ -2004,9 +2089,15 @@  static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
         ofs += 4;
 
-        if (s_bits == MO_64) {
+        switch (l->type) {
+        case TCG_TYPE_I32:
+            break;
+        case TCG_TYPE_I64:
             tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
             ofs += 4;
+            break;
+        default:
+            g_assert_not_reached();
         }
 
         tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
@@ -2016,15 +2107,54 @@  static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
         tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
         tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs);
     } else {
-        tcg_out_mov2(s, TCG_TYPE_TL,
-                     tcg_target_call_iarg_regs[1], l->addrlo_reg,
-                     s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32,
-                     tcg_target_call_iarg_regs[2], l->datalo_reg);
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi);
+        int slot;
 
-        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
-            retaddr = tcg_target_call_iarg_regs[4];
+        switch (l->type) {
+        case TCG_TYPE_I32:
+        case TCG_TYPE_I64:
+            tcg_out_mov2(s, TCG_TYPE_TL,
+                         tcg_target_call_iarg_regs[1], l->addrlo_reg,
+                         l->type, tcg_target_call_iarg_regs[2], l->datalo_reg);
+            slot = 3;
+            break;
+        case TCG_TYPE_I128:
+            switch (TCG_TARGET_CALL_ARG_I128) {
+            case TCG_CALL_ARG_NORMAL:
+                tcg_out_mov3(s, TCG_TYPE_TL,
+                             tcg_target_call_iarg_regs[1], l->addrlo_reg,
+                             TCG_TYPE_I64,
+                             tcg_target_call_iarg_regs[2], l->datalo_reg,
+                             TCG_TYPE_I64,
+                             tcg_target_call_iarg_regs[3], l->datahi_reg);
+                slot = 4;
+                break;
+            case TCG_CALL_ARG_BY_REF:
+                /* Leave room for retaddr below, take next 16 aligned bytes. */
+                tcg_out_st(s, TCG_TYPE_I64, l->datalo_reg,
+                           TCG_REG_ESP, TCG_TARGET_CALL_STACK_OFFSET + 16);
+                tcg_out_st(s, TCG_TYPE_I64, l->datahi_reg,
+                           TCG_REG_ESP, TCG_TARGET_CALL_STACK_OFFSET + 24);
+                tcg_out_mov(s, TCG_TYPE_TL,
+                            tcg_target_call_iarg_regs[1], l->addrlo_reg);
+                tcg_out_modrm_offset(s, OPC_LEA + P_REXW,
+                                     tcg_target_call_iarg_regs[2], TCG_REG_ESP,
+                                     TCG_TARGET_CALL_STACK_OFFSET + 16);
+                slot = 3;
+                break;
+            default:
+                qemu_build_not_reached();
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+
+        tcg_debug_assert(slot < (int)ARRAY_SIZE(tcg_target_call_iarg_regs) - 1);
+        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[slot++], oi);
+
+        if (slot < (int)ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+            retaddr = tcg_target_call_iarg_regs[slot];
             tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
         } else {
             retaddr = TCG_REG_RAX;
@@ -2288,6 +2418,71 @@  static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
             }
         }
         break;
+
+    case MO_128:
+        {
+            TCGLabel *l1 = NULL, *l2 = NULL;
+            bool use_pair = atom < MO_128;
+
+            tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+
+            if (use_movbe) {
+                TCGReg t = datalo;
+                datalo = datahi;
+                datahi = t;
+            }
+            if (!use_pair) {
+                /*
+                 * Atomicity requires that we use use VMOVDQA.
+                 * If we've already checked for 16-byte alignment, that's all
+                 * we need.  If we arrive here with lesser alignment, then we
+                 * have determined that less that 16-byte alignment can be
+                 * satisfied with two 8-byte loads.
+                 */
+                if (align < MO_128) {
+                    use_pair = true;
+                    l1 = gen_new_label();
+                    l2 = gen_new_label();
+
+                    tcg_out_testi(s, base, align == MO_64 ? 8 : 15);
+                    tcg_out_jxx(s, JCC_JNE, l2, true);
+                }
+
+                tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_VxWx + seg,
+                                             TCG_TMP_VEC, 0,
+                                             base, index, 0, ofs);
+                tcg_out_vec_to_pair(s, TCG_TYPE_I64,
+                                    datalo, datahi, TCG_TMP_VEC);
+
+                if (use_movbe) {
+                    tcg_out_bswap64(s, datalo);
+                    tcg_out_bswap64(s, datahi);
+                }
+
+                if (use_pair) {
+                    tcg_out_jxx(s, JCC_JMP, l1, true);
+                    tcg_out_label(s, l2);
+                }
+            }
+            if (use_pair) {
+                if (base != datalo) {
+                    tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
+                                             base, index, 0, ofs);
+                    tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datahi,
+                                             base, index, 0, ofs + 8);
+                } else {
+                    tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datahi,
+                                             base, index, 0, ofs + 8);
+                    tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
+                                             base, index, 0, ofs);
+                }
+            }
+            if (l1) {
+                tcg_out_label(s, l1);
+            }
+        }
+        break;
+
     default:
         g_assert_not_reached();
     }
@@ -2312,6 +2507,10 @@  static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, TCGType type)
     case TCG_TYPE_I64:
         datahi = (TCG_TARGET_REG_BITS == 32 ? *args++ : 0);
         break;
+    case TCG_TYPE_I128:
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+        datahi = *args++;
+        break;
     default:
         g_assert_not_reached();
     }
@@ -2394,6 +2593,68 @@  static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
                                      base, index, 0, ofs + 4);
         }
         break;
+
+    case MO_128:
+        {
+            TCGLabel *l1 = NULL, *l2 = NULL;
+            bool use_pair = atom < MO_128;
+
+            tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+
+            if (use_movbe) {
+                TCGReg t = datalo;
+                datalo = datahi;
+                datahi = t;
+            }
+            if (!use_pair) {
+                /*
+                 * Atomicity requires that we use use VMOVDQA.
+                 * If we've already checked for 16-byte alignment, that's all
+                 * we need.  If we arrive here with lesser alignment, then we
+                 * have determined that less that 16-byte alignment can be
+                 * satisfied with two 8-byte loads.
+                 */
+                if (align < MO_128) {
+                    use_pair = true;
+                    l1 = gen_new_label();
+                    l2 = gen_new_label();
+
+                    tcg_out_testi(s, base, align == MO_64 ? 8 : 15);
+                    tcg_out_jxx(s, JCC_JNE, l2, true);
+                }
+
+                if (use_movbe) {
+                    /* Byte swap while storing to the stack. */
+                    tcg_out_modrm_offset(s, movop + P_REXW + seg, datalo,
+                                         TCG_REG_ESP, 0);
+                    tcg_out_modrm_offset(s, movop + P_REXW + seg, datahi,
+                                         TCG_REG_ESP, 8);
+                    tcg_out_ld(s, TCG_TYPE_V128, TCG_TMP_VEC, TCG_REG_ESP, 0);
+                } else {
+                    tcg_out_pair_to_vec(s, TCG_TYPE_I64,
+                                        TCG_TMP_VEC, datalo, datahi);
+                }
+                tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_WxVx + seg,
+                                             TCG_TMP_VEC, 0,
+                                             base, index, 0, ofs);
+
+                if (use_pair) {
+                    tcg_out_jxx(s, JCC_JMP, l1, true);
+                    tcg_out_label(s, l2);
+                }
+            }
+            if (use_pair) {
+                tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
+                                         base, index, 0, ofs);
+                tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datahi,
+                                         base, index, 0, ofs + 8);
+            }
+            if (l1) {
+                tcg_out_label(s, l1);
+            }
+        }
+        break;
+
     default:
         g_assert_not_reached();
     }
@@ -2415,6 +2676,10 @@  static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, TCGType type)
     case TCG_TYPE_I64:
         datahi = (TCG_TARGET_REG_BITS == 32 ? *args++ : 0);
         break;
+    case TCG_TYPE_I128:
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+        datahi = *args++;
+        break;
     default:
         g_assert_not_reached();
     }
@@ -2752,6 +3017,9 @@  static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_qemu_ld_i64:
         tcg_out_qemu_ld(s, args, TCG_TYPE_I64);
         break;
+    case INDEX_op_qemu_ld_i128:
+        tcg_out_qemu_ld(s, args, TCG_TYPE_I128);
+        break;
     case INDEX_op_qemu_st_i32:
     case INDEX_op_qemu_st8_i32:
         tcg_out_qemu_st(s, args, TCG_TYPE_I32);
@@ -2759,6 +3027,9 @@  static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_qemu_st_i64:
         tcg_out_qemu_st(s, args, TCG_TYPE_I64);
         break;
+    case INDEX_op_qemu_st_i128:
+        tcg_out_qemu_st(s, args, TCG_TYPE_I128);
+        break;
 
     OP_32_64(mulu2):
         tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
@@ -3449,6 +3720,13 @@  static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
                 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(L, L, L)
                 : C_O0_I4(L, L, L, L));
 
+    case INDEX_op_qemu_ld_i128:
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+        return C_O2_I1(r, r, L);
+    case INDEX_op_qemu_st_i128:
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+        return C_O0_I3(L, L, L);
+
     case INDEX_op_brcond2_i32:
         return C_O0_I4(r, r, ri, ri);
 
@@ -4306,6 +4584,7 @@  static void tcg_target_init(TCGContext *s)
 
     s->reserved_regs = 0;
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
+    tcg_regset_set_reg(s->reserved_regs, TCG_TMP_VEC);
 #ifdef _WIN64
     /* These are call saved, and we don't save them, so don't use them. */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM6);