@@ -9,6 +9,7 @@
#define __ASM_IO_H
#include <linux/types.h>
+#include <linux/iorw.h>
#include <linux/pgtable.h>
#include <asm/byteorder.h>
@@ -24,24 +25,28 @@
#define __raw_writeb __raw_writeb
static inline void __raw_writeb(u8 val, volatile void __iomem *addr)
{
+ log_write_io(addr);
asm volatile("strb %w0, [%1]" : : "rZ" (val), "r" (addr));
}
#define __raw_writew __raw_writew
static inline void __raw_writew(u16 val, volatile void __iomem *addr)
{
+ log_write_io(addr);
asm volatile("strh %w0, [%1]" : : "rZ" (val), "r" (addr));
}
#define __raw_writel __raw_writel
static __always_inline void __raw_writel(u32 val, volatile void __iomem *addr)
{
+ log_write_io(addr);
asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr));
}
#define __raw_writeq __raw_writeq
static inline void __raw_writeq(u64 val, volatile void __iomem *addr)
{
+ log_write_io(addr);
asm volatile("str %x0, [%1]" : : "rZ" (val), "r" (addr));
}
@@ -49,6 +54,7 @@ static inline void __raw_writeq(u64 val, volatile void __iomem *addr)
static inline u8 __raw_readb(const volatile void __iomem *addr)
{
u8 val;
+ log_read_io(addr);
asm volatile(ALTERNATIVE("ldrb %w0, [%1]",
"ldarb %w0, [%1]",
ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -61,6 +67,7 @@ static inline u16 __raw_readw(const volatile void __iomem *addr)
{
u16 val;
+ log_read_io(addr);
asm volatile(ALTERNATIVE("ldrh %w0, [%1]",
"ldarh %w0, [%1]",
ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -72,6 +79,7 @@ static inline u16 __raw_readw(const volatile void __iomem *addr)
static __always_inline u32 __raw_readl(const volatile void __iomem *addr)
{
u32 val;
+ log_read_io(addr);
asm volatile(ALTERNATIVE("ldr %w0, [%1]",
"ldar %w0, [%1]",
ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -83,6 +91,79 @@ static __always_inline u32 __raw_readl(const volatile void __iomem *addr)
static inline u64 __raw_readq(const volatile void __iomem *addr)
{
u64 val;
+ log_read_io(addr);
+ asm volatile(ALTERNATIVE("ldr %0, [%1]",
+ "ldar %0, [%1]",
+ ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+ : "=r" (val) : "r" (addr));
+ return val;
+}
+
+#define __raw_writeb_no_log __raw_writeb_no_log
+static inline void __raw_writeb_no_log(u8 val, volatile void __iomem *addr)
+{
+ asm volatile("strb %w0, [%1]" : : "rZ" (val), "r" (addr));
+}
+
+#define __raw_writew_no_log __raw_writew_no_log
+static inline void __raw_writew_no_log(u16 val, volatile void __iomem *addr)
+{
+ asm volatile("strh %w0, [%1]" : : "rZ" (val), "r" (addr));
+}
+
+#define __raw_writel_no_log __raw_writel_no_log
+static inline void __raw_writel_no_log(u32 val, volatile void __iomem *addr)
+{
+ asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr));
+}
+
+#define __raw_writeq_no_log __raw_writeq_no_log
+static inline void __raw_writeq_no_log(u64 val, volatile void __iomem *addr)
+{
+ asm volatile("str %x0, [%1]" : : "rZ" (val), "r" (addr));
+}
+
+#define __raw_readb_no_log __raw_readb_no_log
+static inline u8 __raw_readb_no_log(const volatile void __iomem *addr)
+{
+ u8 val;
+
+ asm volatile(ALTERNATIVE("ldrb %w0, [%1]",
+ "ldarb %w0, [%1]",
+ ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+ : "=r" (val) : "r" (addr));
+ return val;
+}
+
+#define __raw_readw_no_log __raw_readw_no_log
+static inline u16 __raw_readw_no_log(const volatile void __iomem *addr)
+{
+ u16 val;
+
+ asm volatile(ALTERNATIVE("ldrh %w0, [%1]",
+ "ldarh %w0, [%1]",
+ ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+ : "=r" (val) : "r" (addr));
+ return val;
+}
+
+#define __raw_readl_no_log __raw_readl_no_log
+static inline u32 __raw_readl_no_log(const volatile void __iomem *addr)
+{
+ u32 val;
+
+ asm volatile(ALTERNATIVE("ldr %w0, [%1]",
+ "ldar %w0, [%1]",
+ ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
+ : "=r" (val) : "r" (addr));
+ return val;
+}
+
+#define __raw_readq_no_log __raw_readq_no_log
+static inline u64 __raw_readq_no_log(const volatile void __iomem *addr)
+{
+ u64 val;
+
asm volatile(ALTERNATIVE("ldr %0, [%1]",
"ldar %0, [%1]",
ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE)
@@ -121,10 +202,20 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
#define readl_relaxed(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl(c)); __r; })
#define readq_relaxed(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq(c)); __r; })
-#define writeb_relaxed(v,c) ((void)__raw_writeb((v),(c)))
-#define writew_relaxed(v,c) ((void)__raw_writew((__force u16)cpu_to_le16(v),(c)))
-#define writel_relaxed(v,c) ((void)__raw_writel((__force u32)cpu_to_le32(v),(c)))
-#define writeq_relaxed(v,c) ((void)__raw_writeq((__force u64)cpu_to_le64(v),(c)))
+#define writeb_relaxed(v, c) ((void)__raw_writeb((v), (c)))
+#define writew_relaxed(v, c) ((void)__raw_writew((__force u16)cpu_to_le16(v), (c)))
+#define writel_relaxed(v, c) ((void)__raw_writel((__force u32)cpu_to_le32(v), (c)))
+#define writeq_relaxed(v, c) ((void)__raw_writeq((__force u64)cpu_to_le64(v), (c)))
+
+#define readb_relaxed_no_log(c) ({ u8 __r = __raw_readb_no_log(c); __r; })
+#define readw_relaxed_no_log(c) ({ u16 __r = le16_to_cpu((__force __le16)__raw_readw_no_log(c)); __r; })
+#define readl_relaxed_no_log(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl_no_log(c)); __r; })
+#define readq_relaxed_no_log(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq_no_log(c)); __r; })
+
+#define writeb_relaxed_no_log(v, c) ((void)__raw_writeb_no_log((v), (c)))
+#define writew_relaxed_no_log(v, c) ((void)__raw_writew_no_log((__force u16)cpu_to_le16(v), (c)))
+#define writel_relaxed_no_log(v, c) ((void)__raw_writel_no_log((__force u32)cpu_to_le32(v), (c)))
+#define writeq_relaxed_no_log(v, c) ((void)__raw_writeq_no_log((__force u64)cpu_to_le64(v), (c)))
/*
* I/O memory access primitives. Reads are ordered relative to any
@@ -136,10 +227,20 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
#define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(__v); __v; })
#define readq(c) ({ u64 __v = readq_relaxed(c); __iormb(__v); __v; })
-#define writeb(v,c) ({ __iowmb(); writeb_relaxed((v),(c)); })
-#define writew(v,c) ({ __iowmb(); writew_relaxed((v),(c)); })
-#define writel(v,c) ({ __iowmb(); writel_relaxed((v),(c)); })
-#define writeq(v,c) ({ __iowmb(); writeq_relaxed((v),(c)); })
+#define writeb(v, c) ({ __iowmb(); writeb_relaxed((v), (c)); })
+#define writew(v, c) ({ __iowmb(); writew_relaxed((v), (c)); })
+#define writel(v, c) ({ __iowmb(); writel_relaxed((v), (c)); })
+#define writeq(v, c) ({ __iowmb(); writeq_relaxed((v), (c)); })
+
+#define readb_no_log(c) ({ u8 __v = readb_relaxed_no_log(c); __iormb(__v); __v; })
+#define readw_no_log(c) ({ u16 __v = readw_relaxed_no_log(c); __iormb(__v); __v; })
+#define readl_no_log(c) ({ u32 __v = readl_relaxed_no_log(c); __iormb(__v); __v; })
+#define readq_no_log(c) ({ u64 __v = readq_relaxed_no_log(c); __iormb(__v); __v; })
+
+#define writeb_no_log(v, c) ({ __iowmb(); writeb_relaxed_no_log((v), (c)); })
+#define writew_no_log(v, c) ({ __iowmb(); writew_relaxed_no_log((v), (c)); })
+#define writel_no_log(v, c) ({ __iowmb(); writel_relaxed_no_log((v), (c)); })
+#define writeq_no_log(v, c) ({ __iowmb(); writeq_relaxed_no_log((v), (c)); })
/*
* I/O port access primitives.
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *
+ */
+#ifndef __LOG_IORW_H__
+#define __LOG_IORW_H__
+
+#include <linux/types.h>
+
+#if IS_ENABLED(CONFIG_TRACE_RW)
+void log_write_io(volatile void __iomem *addr);
+void log_read_io(const volatile void __iomem *addr);
+#else
+static inline void log_write_io(volatile void __iomem *addr)
+{ }
+static inline void log_read_io(const volatile void __iomem *addr)
+{ }
+#endif /* CONFIG_TRACE_RW */
+
+#endif /* __LOG_IORW_H__ */
new file mode 100644
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rwio
+
+#if !defined(_TRACE_RWIO_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RWIO_H
+
+#include <linux/tracepoint.h>
+
+TRACE_EVENT(raw_write,
+
+ TP_PROTO(unsigned long fn, volatile void __iomem *addr),
+
+ TP_ARGS(fn, addr),
+
+ TP_STRUCT__entry(
+ __field(u64, fn)
+ __field(u64, addr)
+ ),
+
+ TP_fast_assign(
+ __entry->fn = fn;
+ __entry->addr = (u64)addr;
+ ),
+
+ TP_printk("%pS write addr=%p\n", __entry->fn, __entry->addr)
+);
+
+TRACE_EVENT(raw_read,
+
+ TP_PROTO(unsigned long fn, const volatile void __iomem *addr),
+
+ TP_ARGS(fn, addr),
+
+ TP_STRUCT__entry(
+ __field(u64, fn)
+ __field(u64, addr)
+ ),
+
+ TP_fast_assign(
+ __entry->fn = fn;
+ __entry->addr = (u64)addr;
+ ),
+
+ TP_printk("%pS read addr=%p\n", __entry->fn, __entry->addr)
+);
+
+#endif /* _TRACE_PREEMPTIRQ_H */
+
+#include <trace/define_trace.h>
@@ -81,6 +81,17 @@ config RING_BUFFER_ALLOW_SWAP
Allow the use of ring_buffer_swap_cpu.
Adds a very slight overhead to tracing when enabled.
+config TRACE_RW
+ bool "Register read/write tracing"
+ select TRACING
+ default n
+ help
+ Create tracepoints for IO read/write operations, so that
+ modules can register hooks to use them.
+
+ Disable this option, when there is support from corresponding
+ architecture.
+
config PREEMPTIRQ_TRACEPOINTS
bool
depends on TRACE_PREEMPT_TOGGLE || TRACE_IRQFLAGS
@@ -95,4 +95,5 @@ obj-$(CONFIG_BOOTTIME_TRACING) += trace_boot.o
obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
+obj-$(CONFIG_TRACE_RW) += trace_readwrite.o
libftrace-y := ftrace.o
new file mode 100644
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Register read and write tracepoints
+ *
+ * Copyright (c) 2020, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/kallsyms.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/ftrace.h>
+#include <linux/iorw.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/rwio.h>
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(raw_write);
+EXPORT_TRACEPOINT_SYMBOL_GPL(raw_read);
+
+void log_write_io(volatile void __iomem *addr)
+{
+ trace_raw_write(CALLER_ADDR0, addr);
+}
+EXPORT_SYMBOL_GPL(log_write_io);
+
+void log_read_io(const volatile void __iomem *addr)
+{
+ trace_raw_read(CALLER_ADDR0, addr);
+}
+EXPORT_SYMBOL_GPL(log_read_io);
Add register read/write operations tracing support. ftrace events helps trace register read and write location details of memory mapped IO registers. Also add _no_log variants the writel_relaxed/readl_relaed APIs to avoid excessive logging for certain register operations. Signed-off-by: Prasad Sodagudi <psodagud@codeaurora.org> --- arch/arm64/include/asm/io.h | 117 ++++++++++++++++++++++++++++++++++++++--- include/linux/iorw.h | 20 +++++++ include/trace/events/rwio.h | 51 ++++++++++++++++++ kernel/trace/Kconfig | 11 ++++ kernel/trace/Makefile | 1 + kernel/trace/trace_readwrite.c | 30 +++++++++++ 6 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 include/linux/iorw.h create mode 100644 include/trace/events/rwio.h create mode 100644 kernel/trace/trace_readwrite.c