@@ -1699,6 +1699,182 @@ bool tlb_plugin_lookup(CPUState *cpu, target_ulong addr, int mmu_idx,
#endif
+/*
+ * Probe for a load/store operation.
+ * Return the host address and into @flags.
+ */
+
+typedef struct MMULookupPageData {
+ CPUTLBEntryFull *full;
+ void *haddr;
+ target_ulong addr;
+ int flags;
+ int size;
+} MMULookupPageData;
+
+typedef struct MMULookupLocals {
+ MMULookupPageData page[2];
+ MemOp memop;
+ int mmu_idx;
+} MMULookupLocals;
+
+/**
+ * mmu_lookup1: translate one page
+ * @env: cpu context
+ * @data: lookup parameters
+ * @mmu_idx: virtual address context
+ * @access_type: load/store/code
+ * @ra: return address into tcg generated code, or 0
+ *
+ * Resolve the translation for the one page at @data.addr, filling in
+ * the rest of @data with the results. If the translation fails,
+ * tlb_fill will longjmp out. Return true if the softmmu tlb for
+ * @mmu_idx may have resized.
+ */
+static bool mmu_lookup1(CPUArchState *env, MMULookupPageData *data,
+ int mmu_idx, MMUAccessType access_type, uintptr_t ra)
+{
+ target_ulong addr = data->addr;
+ uintptr_t index = tlb_index(env, mmu_idx, addr);
+ CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+ target_ulong tlb_addr = tlb_read_idx(entry, access_type);
+ bool maybe_resized = false;
+
+ /* If the TLB entry is for a different page, reload and try again. */
+ if (!tlb_hit(tlb_addr, addr)) {
+ if (!victim_tlb_hit(env, mmu_idx, index, access_type,
+ addr & TARGET_PAGE_MASK)) {
+ tlb_fill(env_cpu(env), addr, data->size, access_type, mmu_idx, ra);
+ maybe_resized = true;
+ index = tlb_index(env, mmu_idx, addr);
+ entry = tlb_entry(env, mmu_idx, addr);
+ }
+ tlb_addr = tlb_read_idx(entry, access_type) & ~TLB_INVALID_MASK;
+ }
+
+ data->flags = tlb_addr & TLB_FLAGS_MASK;
+ data->full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
+ /* Compute haddr speculatively; depending on flags it might be invalid. */
+ data->haddr = (void *)((uintptr_t)addr + entry->addend);
+
+ return maybe_resized;
+}
+
+/**
+ * mmu_watch_or_dirty
+ * @env: cpu context
+ * @data: lookup parameters
+ * @access_type: load/store/code
+ * @ra: return address into tcg generated code, or 0
+ *
+ * Trigger watchpoints for @data.addr:@data.size;
+ * record writes to protected clean pages.
+ */
+static void mmu_watch_or_dirty(CPUArchState *env, MMULookupPageData *data,
+ MMUAccessType access_type, uintptr_t ra)
+{
+ CPUTLBEntryFull *full = data->full;
+ target_ulong addr = data->addr;
+ int flags = data->flags;
+ int size = data->size;
+
+ /* On watchpoint hit, this will longjmp out. */
+ if (flags & TLB_WATCHPOINT) {
+ int wp = access_type == MMU_DATA_STORE ? BP_MEM_WRITE : BP_MEM_READ;
+ cpu_check_watchpoint(env_cpu(env), addr, size, full->attrs, wp, ra);
+ flags &= ~TLB_WATCHPOINT;
+ }
+
+ if (flags & TLB_NOTDIRTY) {
+ notdirty_write(env_cpu(env), addr, size, full, ra);
+ flags &= ~TLB_NOTDIRTY;
+ }
+ data->flags = flags;
+}
+
+/**
+ * mmu_lookup: translate page(s)
+ * @env: cpu context
+ * @addr: virtual address
+ * @oi: combined mmu_idx and MemOp
+ * @ra: return address into tcg generated code, or 0
+ * @access_type: load/store/code
+ * @l: output result
+ *
+ * Resolve the translation for the page(s) beginning at @addr, for MemOp.size
+ * bytes. Return true if the lookup crosses a page boundary.
+ */
+static inline bool QEMU_ALWAYS_INLINE
+mmu_lookup(CPUArchState *env, target_ulong addr, MemOpIdx oi, uintptr_t ra,
+ MMUAccessType access_type, MMULookupLocals *l)
+{
+ unsigned a_bits;
+ bool crosspage;
+ int flags;
+
+ l->memop = get_memop(oi);
+ l->mmu_idx = get_mmuidx(oi);
+
+ tcg_debug_assert(l->mmu_idx < NB_MMU_MODES);
+
+ /* Handle CPU specific unaligned behaviour */
+ a_bits = get_alignment_bits(l->memop);
+ if (addr & ((1 << a_bits) - 1)) {
+ cpu_unaligned_access(env_cpu(env), addr, access_type, l->mmu_idx, ra);
+ }
+
+ l->page[0].addr = addr;
+ l->page[0].size = memop_size(l->memop);
+ l->page[1].addr = (addr + l->page[0].size - 1) & TARGET_PAGE_MASK;
+ l->page[1].size = 0;
+ crosspage = (addr ^ l->page[1].addr) & TARGET_PAGE_MASK;
+
+ if (likely(!crosspage)) {
+ mmu_lookup1(env, &l->page[0], l->mmu_idx, access_type, ra);
+
+ flags = l->page[0].flags;
+ if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
+ mmu_watch_or_dirty(env, &l->page[0], access_type, ra);
+ }
+ } else {
+ /* Finish compute of page crossing. */
+ int size1 = l->page[1].addr - addr;
+ l->page[1].size = l->page[0].size - size1;
+ l->page[0].size = size1;
+
+ /*
+ * Lookup both pages, recognizing exceptions from either. If the
+ * second lookup potentially resized, refresh first CPUTLBEntryFull.
+ */
+ mmu_lookup1(env, &l->page[0], l->mmu_idx, access_type, ra);
+ if (mmu_lookup1(env, &l->page[1], l->mmu_idx, access_type, ra)) {
+ uintptr_t index = tlb_index(env, l->mmu_idx, addr);
+ l->page[0].full = &env_tlb(env)->d[l->mmu_idx].fulltlb[index];
+ }
+
+ flags = l->page[0].flags | l->page[1].flags;
+ if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
+ mmu_watch_or_dirty(env, &l->page[0], access_type, ra);
+ mmu_watch_or_dirty(env, &l->page[1], access_type, ra);
+ }
+ }
+
+ return crosspage;
+}
+
+/*
+ * Since target/sparc is the only user of TLB_BSWAP, and all
+ * Sparc accesses are aligned, any treatment across two pages
+ * would be arbitrary. Refuse it until there's a use.
+ */
+#define assert_no_tlb_bswap_(F) \
+ tcg_debug_assert((F & TLB_BSWAP) == 0)
+#define assert_no_tlb_bswap \
+ do { \
+ assert_no_tlb_bswap_(l.page[0].flags); \
+ assert_no_tlb_bswap_(l.page[1].flags); \
+ } while (0)
+
/*
* Probe for an atomic operation. Do not allow unaligned operations,
* or io operations to proceed. Return the host address.
@@ -1855,113 +2031,6 @@ load_memop(const void *haddr, MemOp op)
}
}
-static inline uint64_t QEMU_ALWAYS_INLINE
-load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi,
- uintptr_t retaddr, MemOp op, MMUAccessType access_type,
- FullLoadHelper *full_load)
-{
- const unsigned a_bits = get_alignment_bits(get_memop(oi));
- const size_t size = memop_size(op);
- uintptr_t mmu_idx = get_mmuidx(oi);
- uintptr_t index;
- CPUTLBEntry *entry;
- target_ulong tlb_addr;
- void *haddr;
- uint64_t res;
-
- tcg_debug_assert(mmu_idx < NB_MMU_MODES);
-
- /* Handle CPU specific unaligned behaviour */
- if (addr & ((1 << a_bits) - 1)) {
- cpu_unaligned_access(env_cpu(env), addr, access_type,
- mmu_idx, retaddr);
- }
-
- index = tlb_index(env, mmu_idx, addr);
- entry = tlb_entry(env, mmu_idx, addr);
- tlb_addr = tlb_read_idx(entry, access_type);
-
- /* If the TLB entry is for a different page, reload and try again. */
- if (!tlb_hit(tlb_addr, addr)) {
- if (!victim_tlb_hit(env, mmu_idx, index, access_type,
- addr & TARGET_PAGE_MASK)) {
- tlb_fill(env_cpu(env), addr, size,
- access_type, mmu_idx, retaddr);
- index = tlb_index(env, mmu_idx, addr);
- entry = tlb_entry(env, mmu_idx, addr);
- }
- tlb_addr = tlb_read_idx(entry, access_type);
- tlb_addr &= ~TLB_INVALID_MASK;
- }
-
- /* Handle anything that isn't just a straight memory access. */
- if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
- CPUTLBEntryFull *full;
- bool need_swap;
-
- /* For anything that is unaligned, recurse through full_load. */
- if ((addr & (size - 1)) != 0) {
- goto do_unaligned_access;
- }
-
- full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
-
- /* Handle watchpoints. */
- if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
- /* On watchpoint hit, this will longjmp out. */
- cpu_check_watchpoint(env_cpu(env), addr, size,
- full->attrs, BP_MEM_READ, retaddr);
- }
-
- need_swap = size > 1 && (tlb_addr & TLB_BSWAP);
-
- /* Handle I/O access. */
- if (likely(tlb_addr & TLB_MMIO)) {
- return io_readx(env, full, mmu_idx, addr, retaddr,
- access_type, op ^ (need_swap * MO_BSWAP));
- }
-
- haddr = (void *)((uintptr_t)addr + entry->addend);
-
- /*
- * Keep these two load_memop separate to ensure that the compiler
- * is able to fold the entire function to a single instruction.
- * There is a build-time assert inside to remind you of this. ;-)
- */
- if (unlikely(need_swap)) {
- return load_memop(haddr, op ^ MO_BSWAP);
- }
- return load_memop(haddr, op);
- }
-
- /* Handle slow unaligned access (it spans two pages or IO). */
- if (size > 1
- && unlikely((addr & ~TARGET_PAGE_MASK) + size - 1
- >= TARGET_PAGE_SIZE)) {
- target_ulong addr1, addr2;
- uint64_t r1, r2;
- unsigned shift;
- do_unaligned_access:
- addr1 = addr & ~((target_ulong)size - 1);
- addr2 = addr1 + size;
- r1 = full_load(env, addr1, oi, retaddr);
- r2 = full_load(env, addr2, oi, retaddr);
- shift = (addr & (size - 1)) * 8;
-
- if (memop_big_endian(op)) {
- /* Big-endian combine. */
- res = (r1 << shift) | (r2 >> ((size * 8) - shift));
- } else {
- /* Little-endian combine. */
- res = (r1 >> shift) | (r2 << ((size * 8) - shift));
- }
- return res & MAKE_64BIT_MASK(0, size * 8);
- }
-
- haddr = (void *)((uintptr_t)addr + entry->addend);
- return load_memop(haddr, op);
-}
-
/*
* For the benefit of TCG generated code, we want to avoid the
* complication of ABI-specific return type promotion and always
@@ -1972,90 +2041,240 @@ load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi,
* We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
*/
-static uint64_t full_ldub_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
+/**
+ * do_ld_mmio_beN:
+ * @env: cpu context
+ * @p: translation parameters
+ * @ret_be: accumulated data
+ * @mmu_idx: virtual address context
+ * @ra: return address into tcg generated code, or 0
+ *
+ * Load @p->size bytes from @p->addr, which is memory-mapped i/o.
+ * The bytes are concatenated with in big-endian order with @ret_be.
+ */
+static uint64_t do_ld_mmio_beN(CPUArchState *env, MMULookupPageData *p,
+ uint64_t ret_be, int mmu_idx,
+ MMUAccessType type, uintptr_t ra)
{
- validate_memop(oi, MO_UB);
- return load_helper(env, addr, oi, retaddr, MO_UB, MMU_DATA_LOAD,
- full_ldub_mmu);
+ CPUTLBEntryFull *full = p->full;
+ target_ulong addr = p->addr;
+ int i, size = p->size;
+
+ QEMU_IOTHREAD_LOCK_GUARD();
+ for (i = 0; i < size; i++) {
+ uint8_t x = io_readx(env, full, mmu_idx, addr + i, ra, type, MO_UB);
+ ret_be = (ret_be << 8) | x;
+ }
+ return ret_be;
+}
+
+/**
+ * do_ld_bytes_beN
+ * @p: translation parameters
+ * @ret_be: accumulated data
+ *
+ * Load @p->size bytes from @p->haddr, which is RAM.
+ * The bytes to concatenated in big-endian order with @ret_be.
+ */
+static uint64_t do_ld_bytes_beN(MMULookupPageData *p, uint64_t ret_be)
+{
+ uint8_t *haddr = p->haddr;
+ int i, size = p->size;
+
+ for (i = 0; i < size; i++) {
+ ret_be = (ret_be << 8) | haddr[i];
+ }
+ return ret_be;
+}
+
+/*
+ * Wrapper for the above.
+ */
+static uint64_t do_ld_beN(CPUArchState *env, MMULookupPageData *p,
+ uint64_t ret_be, int mmu_idx,
+ MMUAccessType type, uintptr_t ra)
+{
+ if (unlikely(p->flags & TLB_MMIO)) {
+ return do_ld_mmio_beN(env, p, ret_be, mmu_idx, type, ra);
+ } else {
+ return do_ld_bytes_beN(p, ret_be);
+ }
+}
+
+static uint8_t do_ld_1(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
+ MMUAccessType type, uintptr_t ra)
+{
+ if (unlikely(p->flags & TLB_MMIO)) {
+ return io_readx(env, p->full, mmu_idx, p->addr, ra, type, MO_UB);
+ } else {
+ return *(uint8_t *)p->haddr;
+ }
+}
+
+static uint8_t do_ld1_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi,
+ uintptr_t ra, MMUAccessType access_type)
+{
+ MMULookupLocals l;
+ bool crosspage;
+
+ crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ tcg_debug_assert(!crosspage);
+
+ return do_ld_1(env, &l.page[0], l.mmu_idx, access_type, ra);
}
tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return full_ldub_mmu(env, addr, oi, retaddr);
+ validate_memop(oi, MO_UB);
+ return do_ld1_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
}
-static uint64_t full_le_lduw_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
+static uint16_t do_ld2_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi,
+ uintptr_t ra, MMUAccessType access_type)
{
- validate_memop(oi, MO_LEUW);
- return load_helper(env, addr, oi, retaddr, MO_LEUW, MMU_DATA_LOAD,
- full_le_lduw_mmu);
+ MMULookupLocals l;
+ bool crosspage;
+ uint16_t ret;
+
+ crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ if (likely(!crosspage)) {
+ if (l.page[0].flags & TLB_BSWAP) {
+ l.memop ^= MO_BSWAP;
+ }
+ if (unlikely(l.page[0].flags & TLB_MMIO)) {
+ ret = io_readx(env, l.page[0].full, l.mmu_idx, addr, ra,
+ access_type, l.memop);
+ } else {
+ /* Perform the load host endian, then swap if necessary. */
+ ret = load_memop(l.page[0].haddr, MO_UW);
+ if (l.memop & MO_BSWAP) {
+ ret = bswap16(ret);
+ }
+ }
+ } else {
+ uint8_t a, b;
+
+ assert_no_tlb_bswap;
+
+ a = do_ld_1(env, &l.page[0], l.mmu_idx, access_type, ra);
+ b = do_ld_1(env, &l.page[1], l.mmu_idx, access_type, ra);
+
+ if ((l.memop & MO_BSWAP) == MO_LE) {
+ ret = a | (b << 8);
+ } else {
+ ret = b | (a << 8);
+ }
+ }
+ return ret;
}
tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return full_le_lduw_mmu(env, addr, oi, retaddr);
-}
-
-static uint64_t full_be_lduw_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- validate_memop(oi, MO_BEUW);
- return load_helper(env, addr, oi, retaddr, MO_BEUW, MMU_DATA_LOAD,
- full_be_lduw_mmu);
+ validate_memop(oi, MO_LEUW);
+ return do_ld2_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
}
tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return full_be_lduw_mmu(env, addr, oi, retaddr);
+ validate_memop(oi, MO_BEUW);
+ return do_ld2_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
}
-static uint64_t full_le_ldul_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
+static uint32_t do_ld4_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi,
+ uintptr_t ra, MMUAccessType access_type)
{
- validate_memop(oi, MO_LEUL);
- return load_helper(env, addr, oi, retaddr, MO_LEUL, MMU_DATA_LOAD,
- full_le_ldul_mmu);
+ MMULookupLocals l;
+ bool crosspage;
+ uint32_t ret;
+
+ crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ if (likely(!crosspage)) {
+ if (l.page[0].flags & TLB_BSWAP) {
+ l.memop ^= MO_BSWAP;
+ }
+ if (unlikely(l.page[0].flags & TLB_MMIO)) {
+ ret = io_readx(env, l.page[0].full, l.mmu_idx, addr, ra,
+ access_type, l.memop);
+ } else {
+ /* Perform the load host endian. */
+ ret = load_memop(l.page[0].haddr, MO_UL);
+ if (l.memop & MO_BSWAP) {
+ ret = bswap32(ret);
+ }
+ }
+ } else {
+ assert_no_tlb_bswap;
+ ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, ra);
+ ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, ra);
+ if ((l.memop & MO_BSWAP) == MO_LE) {
+ ret = bswap32(ret);
+ }
+ }
+ return ret;
}
tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return full_le_ldul_mmu(env, addr, oi, retaddr);
-}
-
-static uint64_t full_be_ldul_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- validate_memop(oi, MO_BEUL);
- return load_helper(env, addr, oi, retaddr, MO_BEUL, MMU_DATA_LOAD,
- full_be_ldul_mmu);
+ validate_memop(oi, MO_LEUL);
+ return do_ld4_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
}
tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return full_be_ldul_mmu(env, addr, oi, retaddr);
+ validate_memop(oi, MO_BEUL);
+ return do_ld4_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
+}
+
+static uint64_t do_ld8_mmu(CPUArchState *env, target_ulong addr, MemOpIdx oi,
+ uintptr_t ra, MMUAccessType access_type)
+{
+ MMULookupLocals l;
+ bool crosspage;
+ uint64_t ret;
+
+ crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ if (likely(!crosspage)) {
+ if (l.page[0].flags & TLB_BSWAP) {
+ l.memop ^= MO_BSWAP;
+ }
+ if (unlikely(l.page[0].flags & TLB_MMIO)) {
+ ret = io_readx(env, l.page[0].full, l.mmu_idx, addr, ra,
+ access_type, l.memop);
+ } else {
+ /* Perform the load host endian. */
+ ret = load_memop(l.page[0].haddr, MO_UQ);
+ if (l.memop & MO_BSWAP) {
+ ret = bswap64(ret);
+ }
+ }
+ } else {
+ assert_no_tlb_bswap;
+ ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, ra);
+ ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, ra);
+ if ((l.memop & MO_BSWAP) == MO_LE) {
+ ret = bswap64(ret);
+ }
+ }
+ return ret;
}
uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
validate_memop(oi, MO_LEUQ);
- return load_helper(env, addr, oi, retaddr, MO_LEUQ, MMU_DATA_LOAD,
- helper_le_ldq_mmu);
+ return do_ld8_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
}
uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
validate_memop(oi, MO_BEUQ);
- return load_helper(env, addr, oi, retaddr, MO_BEUQ, MMU_DATA_LOAD,
- helper_be_ldq_mmu);
+ return do_ld8_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
}
/*
@@ -2098,56 +2317,85 @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
* Load helpers for cpu_ldst.h.
*/
-static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t retaddr,
- FullLoadHelper *full_load)
+static void plugin_load_cb(CPUArchState *env, abi_ptr addr, MemOpIdx oi)
{
- uint64_t ret;
-
- ret = full_load(env, addr, oi, retaddr);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
- return ret;
}
uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, full_ldub_mmu);
+ uint8_t ret;
+
+ validate_memop(oi, MO_UB);
+ ret = do_ld1_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, full_be_lduw_mmu);
+ uint16_t ret;
+
+ validate_memop(oi, MO_BEUW);
+ ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, full_be_ldul_mmu);
+ uint32_t ret;
+
+ validate_memop(oi, MO_BEUL);
+ ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, helper_be_ldq_mmu);
+ uint64_t ret;
+
+ validate_memop(oi, MO_BEUQ);
+ ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, full_le_lduw_mmu);
+ uint16_t ret;
+
+ validate_memop(oi, MO_LEUW);
+ ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, full_le_ldul_mmu);
+ uint32_t ret;
+
+ validate_memop(oi, MO_LEUL);
+ ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, oi, ra, helper_le_ldq_mmu);
+ uint64_t ret;
+
+ validate_memop(oi, MO_LEUQ);
+ ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
+ plugin_load_cb(env, addr, oi);
+ return ret;
}
Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr,
@@ -2649,54 +2897,26 @@ void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
/* Code access functions. */
-static uint64_t full_ldub_code(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return load_helper(env, addr, oi, retaddr, MO_8,
- MMU_INST_FETCH, full_ldub_code);
-}
-
uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(env, true));
- return full_ldub_code(env, addr, oi, 0);
-}
-
-static uint64_t full_lduw_code(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return load_helper(env, addr, oi, retaddr, MO_TEUW,
- MMU_INST_FETCH, full_lduw_code);
+ return do_ld1_mmu(env, addr, oi, 0, MMU_INST_FETCH);
}
uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(env, true));
- return full_lduw_code(env, addr, oi, 0);
-}
-
-static uint64_t full_ldl_code(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return load_helper(env, addr, oi, retaddr, MO_TEUL,
- MMU_INST_FETCH, full_ldl_code);
+ return do_ld2_mmu(env, addr, oi, 0, MMU_INST_FETCH);
}
uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(env, true));
- return full_ldl_code(env, addr, oi, 0);
-}
-
-static uint64_t full_ldq_code(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return load_helper(env, addr, oi, retaddr, MO_TEUQ,
- MMU_INST_FETCH, full_ldq_code);
+ return do_ld4_mmu(env, addr, oi, 0, MMU_INST_FETCH);
}
uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(env, true));
- return full_ldq_code(env, addr, oi, 0);
+ return do_ld8_mmu(env, addr, oi, 0, MMU_INST_FETCH);
}
Instead of trying to unify all operations on uint64_t, pull out mmu_lookup() to perform the basic tlb hit and resolution. Create individual functions to handle access by size. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/cputlb.c | 606 ++++++++++++++++++++++++++++++--------------- 1 file changed, 413 insertions(+), 193 deletions(-)