@@ -706,6 +706,10 @@ struct TCGContext {
#ifdef TCG_TARGET_NEED_LDST_LABELS
QSIMPLEQ_HEAD(ldst_labels, TCGLabelQemuLdst) ldst_labels;
#endif
+#ifdef TCG_TARGET_NEED_LDST_OOL_LABELS
+ QSIMPLEQ_HEAD(ldst_labels, TCGLabelQemuLdstOol) ldst_ool_labels;
+ GHashTable *ldst_ool_thunks;
+#endif
#ifdef TCG_TARGET_NEED_POOL_LABELS
struct TCGLabelPoolData *pool_labels;
#endif
new file mode 100644
@@ -0,0 +1,94 @@
+/*
+ * TCG Backend Data: load-store optimization only.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+typedef struct TCGLabelQemuLdstOol {
+ QSIMPLEQ_ENTRY(TCGLabelQemuLdstOol) next;
+ tcg_insn_unit *label; /* label pointer to be updated */
+ int reloc; /* relocation type from label_ptr */
+ intptr_t addend; /* relocation addend from label_ptr */
+ uint32_t key; /* oi : is_64 : is_ld */
+} TCGLabelQemuLdstOol;
+
+
+/*
+ * Generate TB finalization at the end of block
+ */
+
+static tcg_insn_unit *tcg_out_qemu_ldst_ool(TCGContext *s, bool is_ld,
+ bool is64, TCGMemOpIdx oi);
+
+static bool tcg_out_ldst_ool_finalize(TCGContext *s)
+{
+ TCGLabelQemuLdstOol *lb;
+
+ /* qemu_ld/st slow paths */
+ QSIMPLEQ_FOREACH(lb, &s->ldst_ool_labels, next) {
+ gpointer dest, key = (gpointer)(uintptr_t)lb->key;
+ TCGMemOpIdx oi;
+ bool is_ld, is_64, ok;
+
+ /* If we have generated the thunk, and it's still in range, all ok. */
+ dest = g_hash_table_lookup(s->ldst_ool_thunks, key);
+ if (dest &&
+ patch_reloc(lb->label, lb->reloc, (intptr_t)dest, lb->addend)) {
+ continue;
+ }
+
+ /* Generate a new thunk. */
+ is_ld = extract32(lb->key, 0, 1);
+ is_64 = extract32(lb->key, 1, 1);
+ oi = extract32(lb->key, 2, 30);
+ dest = tcg_out_qemu_ldst_ool(s, is_ld, is_64, oi);
+
+ /* Test for (pending) buffer overflow. The assumption is that any
+ one thunk beginning below the high water mark cannot overrun
+ the buffer completely. Thus we can test for overflow after
+ generating code without having to check during generation. */
+ if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+ return false;
+ }
+
+ /* Remember the thunk for next time. */
+ g_hash_table_replace(s->ldst_ool_thunks, key, dest);
+
+ /* The new thunk must be in range. */
+ ok = patch_reloc(lb->label, lb->reloc, (intptr_t)dest, lb->addend);
+ tcg_debug_assert(ok);
+ }
+ return true;
+}
+
+/*
+ * Allocate a new TCGLabelQemuLdstOol entry.
+ */
+
+static void add_ldst_ool_label(TCGContext *s, bool is_ld, bool is_64,
+ TCGMemOpIdx oi, int reloc, intptr_t addend)
+{
+ TCGLabelQemuLdstOol *lb = tcg_malloc(sizeof(*lb));
+
+ QSIMPLEQ_INSERT_TAIL(&s->ldst_ool_labels, lb, next);
+ lb->label = s->code_ptr;
+ lb->reloc = reloc;
+ lb->addend = addend;
+ lb->key = is_ld | (is_64 << 1) | (oi << 2);
+}
@@ -521,6 +521,13 @@ static void tcg_region_assign(TCGContext *s, size_t curr_region)
s->code_gen_ptr = start;
s->code_gen_buffer_size = end - start;
s->code_gen_highwater = end - TCG_HIGHWATER;
+
+#ifdef TCG_TARGET_NEED_LDST_OOL_LABELS
+ /* No thunks yet generated this region. Even if they were in range,
+ this is also the most convenient place to clear the table after a
+ full tb_flush. */
+ g_hash_table_remove_all(s->ldst_ool_thunks);
+#endif
}
static bool tcg_region_alloc__locked(TCGContext *s)
@@ -964,6 +971,11 @@ void tcg_context_init(TCGContext *s)
tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
cpu_env = temp_tcgv_ptr(ts);
+
+#ifdef TCG_TARGET_NEED_LDST_OOL_LABELS
+ /* Both key and value are raw pointers. */
+ s->ldst_ool_thunks = g_hash_table_new(NULL, NULL);
+#endif
}
/*
@@ -3540,6 +3552,9 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
#ifdef TCG_TARGET_NEED_LDST_LABELS
QSIMPLEQ_INIT(&s->ldst_labels);
#endif
+#ifdef TCG_TARGET_NEED_LDST_OOL_LABELS
+ QSIMPLEQ_INIT(&s->ldst_ool_labels);
+#endif
#ifdef TCG_TARGET_NEED_POOL_LABELS
s->pool_labels = NULL;
#endif
@@ -3620,6 +3635,11 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
return -1;
}
#endif
+#ifdef TCG_TARGET_NEED_LDST_OOL_LABELS
+ if (!tcg_out_ldst_ool_finalize(s)) {
+ return -1;
+ }
+#endif
#ifdef TCG_TARGET_NEED_POOL_LABELS
if (!tcg_out_pool_finalize(s)) {
return -1;
This variant of tcg-ldst.inc.c allows the entire thunk to be moved out-of-line, with caching across TBs within a region. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.h | 4 ++ tcg/tcg-ldst-ool.inc.c | 94 ++++++++++++++++++++++++++++++++++++++++++ tcg/tcg.c | 20 +++++++++ 3 files changed, 118 insertions(+) create mode 100644 tcg/tcg-ldst-ool.inc.c -- 2.17.2