@@ -804,4 +804,11 @@ config DMATEST
config DMA_ENGINE_RAID
bool
+config XILINX_XDMA
+ tristate "Xilinx XDMA Engine"
+ depends on PCI
+ select DMA_ENGINE
+ help
+ Enable support for Xilinx XDMA IP controller.
+
endif
@@ -2,3 +2,4 @@
obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o
obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o
obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o
+obj-$(CONFIG_XILINX_XDMA) += xilinx_xdma.o
new file mode 100644
@@ -0,0 +1,2042 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2016-2021, Xilinx, Inc.
+ * Copyright (c) 2022, Digiteq Automotive s.r.o.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/mm.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/vmalloc.h>
+#include <linux/pci.h>
+#include <linux/dma/xilinx_xdma.h>
+
+
+static unsigned int enable_credit_mp = 1;
+module_param(enable_credit_mp, uint, 0644);
+MODULE_PARM_DESC(enable_credit_mp,
+ "Set 0 to disable credit feature, default is 1 (enabled)");
+
+#define XDMA_BAR_SIZE 0x8000UL
+
+#define XDMA_CHANNEL_NUM_MAX 4
+#define XDMA_ENG_IRQ_NUM 1
+#define XDMA_MAX_ADJ_BLOCK_SIZE 0x40
+#define XDMA_PAGE_SIZE 0x1000
+#define RX_STATUS_EOP 1
+
+#define XDMA_OFS_INT_CTRL 0x2000UL
+#define XDMA_OFS_CONFIG 0x3000UL
+
+#define XDMA_TRANSFER_MAX_DESC 2048
+
+#define XDMA_DESC_BLEN_BITS 28
+#define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1)
+
+/* bits of the SG DMA control register */
+#define XDMA_CTRL_RUN_STOP (1UL << 0)
+#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1)
+#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2)
+#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3)
+#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4)
+#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6)
+#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9)
+#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
+#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
+#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
+#define XDMA_CTRL_STM_MODE_WB (1UL << 27)
+
+/* bits of the SG DMA status register */
+#define XDMA_STAT_BUSY (1UL << 0)
+#define XDMA_STAT_DESC_STOPPED (1UL << 1)
+#define XDMA_STAT_DESC_COMPLETED (1UL << 2)
+#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3)
+#define XDMA_STAT_MAGIC_STOPPED (1UL << 4)
+#define XDMA_STAT_INVALID_LEN (1UL << 5)
+#define XDMA_STAT_IDLE_STOPPED (1UL << 6)
+
+#define XDMA_STAT_COMMON_ERR_MASK \
+ (XDMA_STAT_ALIGN_MISMATCH | XDMA_STAT_MAGIC_STOPPED | \
+ XDMA_STAT_INVALID_LEN)
+
+/* desc_error, C2H & H2C */
+#define XDMA_STAT_DESC_UNSUPP_REQ (1UL << 19)
+#define XDMA_STAT_DESC_COMPL_ABORT (1UL << 20)
+#define XDMA_STAT_DESC_PARITY_ERR (1UL << 21)
+#define XDMA_STAT_DESC_HEADER_EP (1UL << 22)
+#define XDMA_STAT_DESC_UNEXP_COMPL (1UL << 23)
+
+#define XDMA_STAT_DESC_ERR_MASK \
+ (XDMA_STAT_DESC_UNSUPP_REQ | XDMA_STAT_DESC_COMPL_ABORT | \
+ XDMA_STAT_DESC_PARITY_ERR | XDMA_STAT_DESC_HEADER_EP | \
+ XDMA_STAT_DESC_UNEXP_COMPL)
+
+/* read error: H2C */
+#define XDMA_STAT_H2C_R_UNSUPP_REQ (1UL << 9)
+#define XDMA_STAT_H2C_R_COMPL_ABORT (1UL << 10)
+#define XDMA_STAT_H2C_R_PARITY_ERR (1UL << 11)
+#define XDMA_STAT_H2C_R_HEADER_EP (1UL << 12)
+#define XDMA_STAT_H2C_R_UNEXP_COMPL (1UL << 13)
+
+#define XDMA_STAT_H2C_R_ERR_MASK \
+ (XDMA_STAT_H2C_R_UNSUPP_REQ | XDMA_STAT_H2C_R_COMPL_ABORT | \
+ XDMA_STAT_H2C_R_PARITY_ERR | XDMA_STAT_H2C_R_HEADER_EP | \
+ XDMA_STAT_H2C_R_UNEXP_COMPL)
+
+/* write error, H2C only */
+#define XDMA_STAT_H2C_W_DECODE_ERR (1UL << 14)
+#define XDMA_STAT_H2C_W_SLAVE_ERR (1UL << 15)
+
+#define XDMA_STAT_H2C_W_ERR_MASK \
+ (XDMA_STAT_H2C_W_DECODE_ERR | XDMA_STAT_H2C_W_SLAVE_ERR)
+
+/* read error: C2H */
+#define XDMA_STAT_C2H_R_DECODE_ERR (1UL << 9)
+#define XDMA_STAT_C2H_R_SLAVE_ERR (1UL << 10)
+
+#define XDMA_STAT_C2H_R_ERR_MASK \
+ (XDMA_STAT_C2H_R_DECODE_ERR | XDMA_STAT_C2H_R_SLAVE_ERR)
+
+/* all combined */
+#define XDMA_STAT_H2C_ERR_MASK \
+ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \
+ XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK)
+
+#define XDMA_STAT_C2H_ERR_MASK \
+ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \
+ XDMA_STAT_C2H_R_ERR_MASK)
+
+/* bits of the SGDMA descriptor control field */
+#define XDMA_DESC_STOPPED (1UL << 0)
+#define XDMA_DESC_COMPLETED (1UL << 1)
+#define XDMA_DESC_EOP (1UL << 4)
+
+/* upper 16-bits of engine identifier register */
+#define XDMA_ID_H2C 0x1fc0U
+#define XDMA_ID_C2H 0x1fc1U
+
+#define LS_BYTE_MASK 0x000000FFUL
+
+#define BLOCK_ID_MASK 0xFFF00000
+#define BLOCK_ID_HEAD 0x1FC00000
+
+#define IRQ_BLOCK_ID 0x1fc20000UL
+#define CONFIG_BLOCK_ID 0x1fc30000UL
+
+#define WB_COUNT_MASK 0x00ffffffUL
+#define WB_ERR_MASK (1UL << 31)
+
+#define MAX_USER_IRQ 16
+
+#define DESC_MAGIC 0xAD4B0000UL
+
+#define C2H_WB 0x52B4UL
+
+#define H2C_CHANNEL_OFFSET 0x1000
+#define SGDMA_OFFSET_FROM_CHANNEL 0x4000
+#define CHANNEL_SPACING 0x100
+#define TARGET_SPACING 0x1000
+
+/* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */
+#define PCI_DMA_H(addr) ((addr >> 16) >> 16)
+/* obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */
+#define PCI_DMA_L(addr) (addr & 0xffffffffUL)
+
+
+enum transfer_state {
+ TRANSFER_STATE_NEW = 0,
+ TRANSFER_STATE_SUBMITTED,
+ TRANSFER_STATE_COMPLETED,
+ TRANSFER_STATE_FAILED,
+ TRANSFER_STATE_ABORTED
+};
+
+enum shutdown_state {
+ ENGINE_SHUTDOWN_NONE = 0, /* No shutdown in progress */
+ ENGINE_SHUTDOWN_REQUEST = 1, /* engine requested to shutdown */
+ ENGINE_SHUTDOWN_IDLE = 2 /* engine has shutdown and is idle */
+};
+
+struct config_regs {
+ u32 identifier;
+ u32 reserved_1[4];
+ u32 msi_enable;
+};
+
+struct engine_regs {
+ u32 identifier;
+ u32 control;
+ u32 control_w1s;
+ u32 control_w1c;
+ u32 reserved_1[12]; /* padding */
+
+ u32 status;
+ u32 status_rc;
+ u32 completed_desc_count;
+ u32 alignments;
+ u32 reserved_2[14]; /* padding */
+
+ u32 poll_mode_wb_lo;
+ u32 poll_mode_wb_hi;
+ u32 interrupt_enable_mask;
+ u32 interrupt_enable_mask_w1s;
+ u32 interrupt_enable_mask_w1c;
+ u32 reserved_3[9]; /* padding */
+
+ u32 perf_ctrl;
+ u32 perf_cyc_lo;
+ u32 perf_cyc_hi;
+ u32 perf_dat_lo;
+ u32 perf_dat_hi;
+ u32 perf_pnd_lo;
+ u32 perf_pnd_hi;
+} __packed;
+
+struct engine_sgdma_regs {
+ u32 identifier;
+ u32 reserved_1[31]; /* padding */
+
+ /* bus address to first descriptor in Root Complex Memory */
+ u32 first_desc_lo;
+ u32 first_desc_hi;
+ /* number of adjacent descriptors at first_desc */
+ u32 first_desc_adjacent;
+ u32 credits;
+} __packed;
+
+struct interrupt_regs {
+ u32 identifier;
+ u32 user_int_enable;
+ u32 user_int_enable_w1s;
+ u32 user_int_enable_w1c;
+ u32 channel_int_enable;
+ u32 channel_int_enable_w1s;
+ u32 channel_int_enable_w1c;
+ u32 reserved_1[9]; /* padding */
+
+ u32 user_int_request;
+ u32 channel_int_request;
+ u32 user_int_pending;
+ u32 channel_int_pending;
+ u32 reserved_2[12]; /* padding */
+
+ u32 user_msi_vector[8];
+ u32 channel_msi_vector[8];
+} __packed;
+
+struct sgdma_common_regs {
+ u32 padding[8];
+ u32 credit_mode_enable;
+ u32 credit_mode_enable_w1s;
+ u32 credit_mode_enable_w1c;
+} __packed;
+
+
+/*
+ * Descriptor for a single contiguous memory block transfer.
+ *
+ * Multiple descriptors are linked by means of the next pointer. An additional
+ * extra adjacent number gives the amount of extra contiguous descriptors.
+ *
+ * The descriptors are in root complex memory, and the bytes in the 32-bit
+ * words must be in little-endian byte ordering.
+ */
+struct xdma_desc {
+ u32 control;
+ u32 bytes; /* transfer length in bytes */
+ u32 src_addr_lo; /* source address (low 32-bit) */
+ u32 src_addr_hi; /* source address (high 32-bit) */
+ u32 dst_addr_lo; /* destination address (low 32-bit) */
+ u32 dst_addr_hi; /* destination address (high 32-bit) */
+ /*
+ * next descriptor in the single-linked list of descriptors;
+ * this is the PCIe (bus) address of the next descriptor in the
+ * root complex memory
+ */
+ u32 next_lo; /* next desc address (low 32-bit) */
+ u32 next_hi; /* next desc address (high 32-bit) */
+} __packed;
+
+/* 32 bytes (four 32-bit words) or 64 bytes (eight 32-bit words) */
+struct xdma_result {
+ u32 status;
+ u32 length;
+ u32 reserved_1[6]; /* padding */
+} __packed;
+
+struct sw_desc {
+ dma_addr_t addr;
+ unsigned int len;
+};
+
+/* Describes a (SG DMA) single transfer for the engine */
+#define XFER_FLAG_NEED_UNMAP 0x1
+#define XFER_FLAG_ST_C2H_EOP_RCVED 0x2 /* ST c2h only */
+struct xdma_transfer {
+ struct list_head entry; /* queue of non-completed transfers */
+ struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */
+ struct xdma_result *res_virt; /* virt addr of result, c2h streaming */
+ dma_addr_t res_bus; /* bus addr for result descriptors */
+ dma_addr_t desc_bus; /* bus addr of the first descriptor */
+ int desc_adjacent; /* adjacent descriptors at desc_bus */
+ int desc_num; /* number of descriptors in transfer */
+ int desc_index; /* index for 1st desc. in transfer */
+ int desc_cmpl; /* completed descriptors */
+ int desc_cmpl_th; /* completed descriptor threshold */
+ enum dma_data_direction dir;
+ struct swait_queue_head wq; /* wait queue for transfer completion */
+
+ enum transfer_state state; /* state of the transfer */
+ unsigned int flags;
+ int cyclic; /* flag if transfer is cyclic */
+ int last_in_request; /* flag if last within request */
+ unsigned int len;
+ struct sg_table *sgt;
+};
+
+struct xdma_request_cb {
+ struct sg_table *sgt;
+ unsigned int total_len;
+ u64 ep_addr;
+
+ struct xdma_transfer tfer;
+
+ unsigned int sw_desc_idx;
+ unsigned int sw_desc_cnt;
+ struct sw_desc sdesc[0];
+};
+
+struct xdma_engine {
+ struct xdma_dev *xdev; /* parent device */
+ char name[16]; /* name of this engine */
+
+ /* HW register address offsets */
+ struct engine_regs *regs; /* Control reg BAR offset */
+ struct engine_sgdma_regs *sgdma_regs; /* SGDAM reg BAR offset */
+
+ /* Engine state, configuration and flags */
+ enum shutdown_state shutdown; /* engine shutdown mode */
+ enum dma_data_direction dir;
+ u8 addr_align; /* source/dest alignment in bytes */
+ u8 len_granularity; /* transfer length multiple */
+ u8 addr_bits; /* HW datapath address width */
+ u8 channel:2; /* engine indices */
+ u8 streaming:1;
+ u8 device_open:1; /* flag if engine node open, ST mode only */
+ u8 running:1; /* flag if the driver started engine */
+ u8 non_incr_addr:1; /* flag if non-incremental addressing used */
+ u8 eop_flush:1; /* st c2h only, flush up the data with eop */
+ u8 filler:1;
+
+ int max_extra_adj; /* descriptor prefetch capability */
+ int desc_dequeued; /* num descriptors of completed transfers */
+ u32 status; /* last known status of device */
+ u32 interrupt_enable_mask_value; /* per-engine interrupt mask value */
+
+ /* Transfer list management */
+ struct list_head transfer_list; /* queue of transfers */
+
+ /* Members applicable to AXI-ST C2H (cyclic) transfers */
+ struct xdma_result *cyclic_result;
+ dma_addr_t cyclic_result_bus; /* bus addr for transfer */
+
+ /* Members associated with interrupt mode support */
+ struct swait_queue_head shutdown_wq;
+ spinlock_t lock; /* protects concurrent access */
+ int prev_cpu; /* remember CPU# of (last) locker */
+ int irq_line; /* IRQ vector for this engine */
+ u32 irq_bitmask; /* IRQ bit mask for this engine */
+ struct work_struct work; /* Work queue for interrupt handling */
+
+ struct mutex desc_lock; /* protects concurrent access */
+ dma_addr_t desc_bus;
+ struct xdma_desc *desc;
+ int desc_idx; /* current descriptor index */
+ int desc_used; /* total descriptors used */
+};
+
+struct xdma_dev {
+ struct pci_dev *pdev;
+ void __iomem *config_bar;
+ unsigned int mask_irq_user;
+ int engines_num;
+ struct xdma_engine engine_h2c[XDMA_CHANNEL_NUM_MAX];
+ struct xdma_engine engine_c2h[XDMA_CHANNEL_NUM_MAX];
+};
+
+
+static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->channel_int_enable_w1s);
+}
+
+static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->channel_int_enable_w1c);
+}
+
+static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->user_int_enable_w1s);
+}
+
+static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ iowrite32(mask, ®->user_int_enable_w1c);
+}
+
+static void read_interrupts(struct xdma_dev *xdev)
+{
+ struct interrupt_regs *reg =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+ u32 lo, hi;
+
+ hi = ioread32(®->user_int_request);
+ lo = ioread32(®->channel_int_request);
+}
+
+static void engine_reg_dump(struct xdma_engine *engine)
+{
+ u32 w;
+
+ w = ioread32(&engine->regs->identifier);
+ if ((w & BLOCK_ID_MASK) != BLOCK_ID_HEAD) {
+ pr_warn("XDMA: %s: 0x%08x: invalid engine id\n",
+ engine->name, w);
+ return;
+ }
+
+ pr_info("XDMA: %s: ENGINE REGISTER DUMP\n", engine->name);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n",
+ engine->name, &engine->regs->identifier, w);
+ w = ioread32(&engine->regs->status);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n",
+ engine->name, &engine->regs->status, w);
+ w = ioread32(&engine->regs->control);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n",
+ engine->name, &engine->regs->control, w);
+ w = ioread32(&engine->sgdma_regs->first_desc_lo);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n",
+ engine->name, &engine->sgdma_regs->first_desc_lo, w);
+ w = ioread32(&engine->sgdma_regs->first_desc_hi);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n",
+ engine->name, &engine->sgdma_regs->first_desc_hi, w);
+ w = ioread32(&engine->sgdma_regs->first_desc_adjacent);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n",
+ engine->name, &engine->sgdma_regs->first_desc_adjacent, w);
+ w = ioread32(&engine->regs->completed_desc_count);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (completed_desc_count).\n",
+ engine->name, &engine->regs->completed_desc_count, w);
+ w = ioread32(&engine->regs->interrupt_enable_mask);
+ pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n",
+ engine->name, &engine->regs->interrupt_enable_mask, w);
+}
+
+static void engine_status_dump(struct xdma_engine *engine)
+{
+ u32 v = engine->status;
+ char buffer[256];
+ char *buf = buffer;
+ int len = 0;
+
+ len = sprintf(buf, "XDMA: %s: status: 0x%08x: ", engine->name, v);
+
+ if ((v & XDMA_STAT_BUSY))
+ len += sprintf(buf + len, "BUSY,");
+ if ((v & XDMA_STAT_DESC_STOPPED))
+ len += sprintf(buf + len, "DESC_STOPPED,");
+ if ((v & XDMA_STAT_DESC_COMPLETED))
+ len += sprintf(buf + len, "DESC_COMPL,");
+
+ /* common H2C & C2H */
+ if ((v & XDMA_STAT_COMMON_ERR_MASK)) {
+ if ((v & XDMA_STAT_ALIGN_MISMATCH))
+ len += sprintf(buf + len, "ALIGN_MISMATCH ");
+ if ((v & XDMA_STAT_MAGIC_STOPPED))
+ len += sprintf(buf + len, "MAGIC_STOPPED ");
+ if ((v & XDMA_STAT_INVALID_LEN))
+ len += sprintf(buf + len, "INVLIAD_LEN ");
+ if ((v & XDMA_STAT_IDLE_STOPPED))
+ len += sprintf(buf + len, "IDLE_STOPPED ");
+ buf[len - 1] = ',';
+ }
+
+ if (engine->dir == DMA_TO_DEVICE) {
+ /* H2C only */
+ if ((v & XDMA_STAT_H2C_R_ERR_MASK)) {
+ len += sprintf(buf + len, "R:");
+ if ((v & XDMA_STAT_H2C_R_UNSUPP_REQ))
+ len += sprintf(buf + len, "UNSUPP_REQ ");
+ if ((v & XDMA_STAT_H2C_R_COMPL_ABORT))
+ len += sprintf(buf + len, "COMPL_ABORT ");
+ if ((v & XDMA_STAT_H2C_R_PARITY_ERR))
+ len += sprintf(buf + len, "PARITY ");
+ if ((v & XDMA_STAT_H2C_R_HEADER_EP))
+ len += sprintf(buf + len, "HEADER_EP ");
+ if ((v & XDMA_STAT_H2C_R_UNEXP_COMPL))
+ len += sprintf(buf + len, "UNEXP_COMPL ");
+ buf[len - 1] = ',';
+ }
+
+ if ((v & XDMA_STAT_H2C_W_ERR_MASK)) {
+ len += sprintf(buf + len, "W:");
+ if ((v & XDMA_STAT_H2C_W_DECODE_ERR))
+ len += sprintf(buf + len, "DECODE_ERR ");
+ if ((v & XDMA_STAT_H2C_W_SLAVE_ERR))
+ len += sprintf(buf + len, "SLAVE_ERR ");
+ buf[len - 1] = ',';
+ }
+
+ } else {
+ /* C2H only */
+ if ((v & XDMA_STAT_C2H_R_ERR_MASK)) {
+ len += sprintf(buf + len, "R:");
+ if ((v & XDMA_STAT_C2H_R_DECODE_ERR))
+ len += sprintf(buf + len, "DECODE_ERR ");
+ if ((v & XDMA_STAT_C2H_R_SLAVE_ERR))
+ len += sprintf(buf + len, "SLAVE_ERR ");
+ buf[len - 1] = ',';
+ }
+ }
+
+ /* common H2C & C2H */
+ if ((v & XDMA_STAT_DESC_ERR_MASK)) {
+ len += sprintf(buf + len, "DESC_ERR:");
+ if ((v & XDMA_STAT_DESC_UNSUPP_REQ))
+ len += sprintf(buf + len, "UNSUPP_REQ ");
+ if ((v & XDMA_STAT_DESC_COMPL_ABORT))
+ len += sprintf(buf + len, "COMPL_ABORT ");
+ if ((v & XDMA_STAT_DESC_PARITY_ERR))
+ len += sprintf(buf + len, "PARITY ");
+ if ((v & XDMA_STAT_DESC_HEADER_EP))
+ len += sprintf(buf + len, "HEADER_EP ");
+ if ((v & XDMA_STAT_DESC_UNEXP_COMPL))
+ len += sprintf(buf + len, "UNEXP_COMPL ");
+ buf[len - 1] = ',';
+ }
+
+ buf[len - 1] = '\0';
+ pr_info("%s\n", buffer);
+}
+
+static void engine_status_read(struct xdma_engine *engine, bool clear, bool dump)
+{
+ if (dump)
+ engine_reg_dump(engine);
+
+ if (clear)
+ engine->status = ioread32(&engine->regs->status_rc);
+ else
+ engine->status = ioread32(&engine->regs->status);
+
+ if (dump)
+ engine_status_dump(engine);
+}
+
+static void engine_stop(struct xdma_engine *engine)
+{
+ u32 w;
+
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE)
+ iowrite32(0, &engine->sgdma_regs->credits);
+
+ w = 0;
+ w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
+ w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED;
+ w |= (u32)XDMA_CTRL_IE_READ_ERROR;
+ w |= (u32)XDMA_CTRL_IE_DESC_ERROR;
+
+ w |= (u32)XDMA_CTRL_IE_DESC_STOPPED;
+ w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED;
+
+ iowrite32(w, &engine->regs->control);
+
+ engine->running = 0;
+}
+
+static int engine_start_mode_config(struct xdma_engine *engine)
+{
+ u32 w;
+
+ /* write control register of SG DMA engine */
+ w = (u32)XDMA_CTRL_RUN_STOP;
+ w |= (u32)XDMA_CTRL_IE_READ_ERROR;
+ w |= (u32)XDMA_CTRL_IE_DESC_ERROR;
+ w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
+ w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED;
+
+ w |= (u32)XDMA_CTRL_IE_DESC_STOPPED;
+ w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED;
+
+ /* set non-incremental addressing mode */
+ if (engine->non_incr_addr)
+ w |= (u32)XDMA_CTRL_NON_INCR_ADDR;
+
+ /* start the engine */
+ iowrite32(w, &engine->regs->control);
+ /* dummy read of status register to flush all previous writes */
+ w = ioread32(&engine->regs->status);
+
+ return 0;
+}
+
+/*
+ * Get the number for adjacent descriptors to set in a descriptor, based on the
+ * remaining number of descriptors and the lower bits of the address of the
+ * next descriptor.
+ * Since the number of descriptors in a page (XDMA_PAGE_SIZE) is 128 and the
+ * maximum size of a block of adjacent descriptors is 64 (63 max adjacent
+ * descriptors for any descriptor), align the blocks of adjacent descriptors
+ * to the block size.
+ */
+static u32 xdma_get_next_adj(unsigned int remaining, u32 next_lo)
+{
+ unsigned int next_index;
+
+ if (remaining <= 1)
+ return 0;
+
+ /* shift right 5 times corresponds to a division by
+ * sizeof(xdma_desc) = 32
+ */
+ next_index = ((next_lo & (XDMA_PAGE_SIZE - 1)) >> 5) %
+ XDMA_MAX_ADJ_BLOCK_SIZE;
+ return min(XDMA_MAX_ADJ_BLOCK_SIZE - next_index - 1, remaining - 1);
+}
+
+/*
+ * start an idle engine with its first transfer on queue
+ *
+ * The engine will run and process all transfers that are queued using
+ * transfer_queue() and thus have their descriptor lists chained.
+ *
+ * During the run, new transfers will be processed if transfer_queue() has
+ * chained the descriptors before the hardware fetches the last descriptor.
+ * A transfer that was chained too late will invoke a new run of the engine
+ * initiated from the engine_service() routine.
+ *
+ * The engine must be idle and at least one transfer must be queued.
+ */
+static int engine_start(struct xdma_engine *engine)
+{
+ struct xdma_transfer *transfer;
+ u32 w, next_adj;
+ int rv;
+
+ /* engine transfer queue must not be empty */
+ if (list_empty(&engine->transfer_list)) {
+ pr_warn("XDMA: %s: transfer queue must not be empty\n",
+ engine->name);
+ return -EIO;
+ }
+ /* inspect first transfer queued on the engine */
+ transfer = list_entry(engine->transfer_list.next, struct xdma_transfer,
+ entry);
+ if (!transfer) {
+ pr_warn("XDMA: %s: queued transfer must not be empty\n",
+ engine->name);
+ return -EIO;
+ }
+
+ /* engine is no longer shutdown */
+ engine->shutdown = ENGINE_SHUTDOWN_NONE;
+
+ /* Add credits for Streaming mode C2H */
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE)
+ iowrite32(engine->desc_used, &engine->sgdma_regs->credits);
+
+ /* initialize number of descriptors of dequeued transfers */
+ engine->desc_dequeued = 0;
+
+ /* write lower 32-bit of bus address of transfer first descriptor */
+ w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus));
+ iowrite32(w, &engine->sgdma_regs->first_desc_lo);
+ /* write upper 32-bit of bus address of transfer first descriptor */
+ w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus));
+ iowrite32(w, &engine->sgdma_regs->first_desc_hi);
+
+ next_adj = xdma_get_next_adj(transfer->desc_adjacent,
+ cpu_to_le32(PCI_DMA_L(transfer->desc_bus)));
+ iowrite32(next_adj, &engine->sgdma_regs->first_desc_adjacent);
+
+ rv = engine_start_mode_config(engine);
+ if (rv < 0)
+ return rv;
+ engine_status_read(engine, 0, 0);
+
+ engine->running = 1;
+
+ return 0;
+}
+
+static void engine_service_shutdown(struct xdma_engine *engine)
+{
+ engine_stop(engine);
+ /* awake task on engine's shutdown wait queue */
+ swake_up_one(&engine->shutdown_wq);
+}
+
+static struct xdma_transfer *engine_transfer_completion(
+ struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ if (unlikely(!transfer)) {
+ pr_warn("XDMA: %s empty xfer\n", engine->name);
+ return NULL;
+ }
+
+ /* synchronous I/O? */
+ /* awake task on transfer's wait queue */
+ swake_up_one(&transfer->wq);
+
+ return transfer;
+}
+
+static struct xdma_transfer *engine_service_transfer_list(
+ struct xdma_engine *engine,
+ struct xdma_transfer *transfer,
+ u32 *pdesc_completed)
+{
+ if (unlikely(!transfer)) {
+ pr_warn("XDMA: %s empty xfer\n", engine->name);
+ return NULL;
+ }
+
+ /*
+ * iterate over all the transfers completed by the engine,
+ * except for the last
+ */
+ while (transfer && (!transfer->cyclic) &&
+ (*pdesc_completed > transfer->desc_num)) {
+ /* remove this transfer from pdesc_completed */
+ *pdesc_completed -= transfer->desc_num;
+
+ /* remove completed transfer from list */
+ list_del(engine->transfer_list.next);
+ /* add to dequeued number of descriptors during this run */
+ engine->desc_dequeued += transfer->desc_num;
+ /* mark transfer as successfully completed */
+ transfer->state = TRANSFER_STATE_COMPLETED;
+
+ /*
+ * Complete transfer - sets transfer to NULL if an async
+ * transfer has completed
+ */
+ transfer = engine_transfer_completion(engine, transfer);
+
+ /* if exists, get the next transfer on the list */
+ if (!list_empty(&engine->transfer_list)) {
+ transfer = list_entry(engine->transfer_list.next,
+ struct xdma_transfer, entry);
+ } else {
+ /* no further transfers? */
+ transfer = NULL;
+ }
+ }
+
+ return transfer;
+}
+
+static void engine_err_handle(struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ u32 value;
+
+ /*
+ * The BUSY bit is expected to be clear now but older HW has a race
+ * condition which could cause it to be still set. If it's set, re-read
+ * and check again. If it's still set, log the issue.
+ */
+ if (engine->status & XDMA_STAT_BUSY) {
+ value = ioread32(&engine->regs->status);
+ if ((value & XDMA_STAT_BUSY))
+ pr_warn("XDMA: %s has errors but is still BUSY\n",
+ engine->name);
+ }
+
+ /* mark transfer as failed */
+ transfer->state = TRANSFER_STATE_FAILED;
+ engine_stop(engine);
+}
+
+static struct xdma_transfer *
+engine_service_final_transfer(struct xdma_engine *engine,
+ struct xdma_transfer *transfer,
+ u32 *pdesc_completed)
+{
+ /* inspect the current transfer */
+ if (unlikely(!transfer)) {
+ pr_warn("XDMA: %s: empty xfer\n", engine->name);
+ return NULL;
+ }
+
+ if (((engine->dir == DMA_FROM_DEVICE) &&
+ (engine->status & XDMA_STAT_C2H_ERR_MASK)) ||
+ ((engine->dir == DMA_TO_DEVICE) &&
+ (engine->status & XDMA_STAT_H2C_ERR_MASK))) {
+ pr_warn("XDMA: %s: status error 0x%x.\n", engine->name,
+ engine->status);
+ engine_status_dump(engine);
+ engine_err_handle(engine, transfer);
+ goto transfer_del;
+ }
+
+ if (engine->status & XDMA_STAT_BUSY)
+ pr_info("XDMA: %s: engine unexpectedly busy, ignoring\n",
+ engine->name);
+
+ /* the engine stopped on current transfer? */
+ if (*pdesc_completed < transfer->desc_num) {
+ if (engine->eop_flush) {
+ /* check if eop received */
+ struct xdma_result *result = transfer->res_virt;
+ int i;
+ int max = *pdesc_completed;
+
+ for (i = 0; i < max; i++) {
+ if ((result[i].status & RX_STATUS_EOP) != 0) {
+ transfer->flags |=
+ XFER_FLAG_ST_C2H_EOP_RCVED;
+ break;
+ }
+ }
+
+ transfer->desc_cmpl += *pdesc_completed;
+ if (!(transfer->flags & XFER_FLAG_ST_C2H_EOP_RCVED))
+ return NULL;
+
+ /* mark transfer as successfully completed */
+ engine_service_shutdown(engine);
+ transfer->state = TRANSFER_STATE_COMPLETED;
+ engine->desc_dequeued += transfer->desc_cmpl;
+ } else {
+ transfer->state = TRANSFER_STATE_FAILED;
+ pr_warn("XDMA: %s: xfer stopped half-way\n",
+ engine->name);
+
+ /* add dequeued number of descriptors during this run */
+ engine->desc_dequeued += transfer->desc_num;
+ transfer->desc_cmpl = *pdesc_completed;
+ }
+ } else {
+ if (!transfer->cyclic) {
+ /*
+ * if the engine stopped on this transfer,
+ * it should be the last
+ */
+ WARN_ON(*pdesc_completed > transfer->desc_num);
+ }
+ /* mark transfer as successfully completed */
+ transfer->state = TRANSFER_STATE_COMPLETED;
+ transfer->desc_cmpl = transfer->desc_num;
+ /* add dequeued number of descriptors during this run */
+ engine->desc_dequeued += transfer->desc_num;
+ }
+
+transfer_del:
+ /* remove completed transfer from list */
+ list_del(engine->transfer_list.next);
+
+ /*
+ * Complete transfer - sets transfer to NULL if an asynchronous
+ * transfer has completed
+ */
+ transfer = engine_transfer_completion(engine, transfer);
+
+ return transfer;
+}
+
+static int engine_service_resume(struct xdma_engine *engine)
+{
+ int rv;
+
+ if (!engine->running) {
+ /* in the case of shutdown, let it finish what's in the Q */
+ if (!list_empty(&engine->transfer_list)) {
+ /* (re)start engine */
+ rv = engine_start(engine);
+ if (rv)
+ return rv;
+ /* engine was requested to be shutdown? */
+ } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) {
+ engine->shutdown |= ENGINE_SHUTDOWN_IDLE;
+ /* awake task on engine's shutdown wait queue */
+ swake_up_one(&engine->shutdown_wq);
+ }
+ } else if (list_empty(&engine->transfer_list)) {
+ engine_service_shutdown(engine);
+ }
+
+ return 0;
+}
+
+static int engine_service(struct xdma_engine *engine, int desc_writeback)
+{
+ struct xdma_transfer *transfer = NULL;
+ u32 desc_count = desc_writeback & WB_COUNT_MASK;
+ u32 err_flag = desc_writeback & WB_ERR_MASK;
+ int rv;
+
+ if (!engine->running) {
+ engine_status_read(engine, 1, 0);
+ return 0;
+ }
+
+ /*
+ * If called by the ISR detected an error, read and clear
+ * engine status.
+ */
+ if ((desc_count == 0) || (err_flag != 0))
+ engine_status_read(engine, 1, 0);
+
+ /*
+ * engine was running but is no longer busy, or writeback occurred,
+ * shut down
+ */
+ if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) ||
+ (!engine->eop_flush && desc_count != 0))
+ engine_service_shutdown(engine);
+
+ /*
+ * If called from the ISR, or if an error occurred, the descriptor
+ * count will be zero. In this scenario, read the descriptor count
+ * from HW.
+ */
+ if (!desc_count)
+ desc_count = ioread32(&engine->regs->completed_desc_count);
+ if (!desc_count)
+ goto done;
+
+ /* transfers on queue? */
+ if (!list_empty(&engine->transfer_list)) {
+ /* pick first transfer on queue (was submitted to the engine) */
+ transfer = list_entry(engine->transfer_list.next,
+ struct xdma_transfer, entry);
+ }
+
+ /* account for already dequeued transfers during this engine run */
+ desc_count -= engine->desc_dequeued;
+
+ /* Process all but the last transfer */
+ transfer = engine_service_transfer_list(engine, transfer, &desc_count);
+
+ /*
+ * Process final transfer - includes checks of number of descriptors to
+ * detect faulty completion
+ */
+ transfer = engine_service_final_transfer(engine, transfer, &desc_count);
+
+ /* Restart the engine following the servicing */
+ if (!engine->eop_flush) {
+ rv = engine_service_resume(engine);
+ if (rv)
+ return rv;
+ }
+
+done:
+ return err_flag ? -1 : 0;
+}
+
+static void engine_service_work(struct work_struct *work)
+{
+ struct xdma_engine *engine;
+ unsigned long flags;
+ int rv;
+
+ engine = container_of(work, struct xdma_engine, work);
+
+ spin_lock_irqsave(&engine->lock, flags);
+
+ rv = engine_service(engine, 0);
+ if (rv < 0)
+ goto unlock;
+
+ /* re-enable interrupts for this engine */
+ iowrite32(engine->interrupt_enable_mask_value,
+ &engine->regs->interrupt_enable_mask_w1s);
+
+unlock:
+ spin_unlock_irqrestore(&engine->lock, flags);
+}
+
+static irqreturn_t xdma_isr(int irq, void *dev_id)
+{
+ struct xdma_dev *xdev;
+ struct xdma_engine *engine;
+ struct interrupt_regs *irq_regs;
+
+ engine = (struct xdma_engine *)dev_id;
+ xdev = engine->xdev;
+
+ irq_regs = (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+
+ /* Disable the interrupt for this engine */
+ iowrite32(engine->interrupt_enable_mask_value,
+ &engine->regs->interrupt_enable_mask_w1c);
+ /* Dummy read to flush the above write */
+ ioread32(&irq_regs->channel_int_pending);
+ schedule_work(&engine->work);
+
+ return IRQ_HANDLED;
+}
+
+static int is_config_bar(void *bar)
+{
+ u32 irq_id = 0;
+ u32 cfg_id = 0;
+ u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */
+ struct interrupt_regs *irq_regs =
+ (struct interrupt_regs *)(bar + XDMA_OFS_INT_CTRL);
+ struct config_regs *cfg_regs =
+ (struct config_regs *)(bar + XDMA_OFS_CONFIG);
+
+ irq_id = ioread32(&irq_regs->identifier);
+ cfg_id = ioread32(&cfg_regs->identifier);
+
+ if (((irq_id & mask) == IRQ_BLOCK_ID)
+ && ((cfg_id & mask) == CONFIG_BLOCK_ID))
+ return 1;
+
+ return 0;
+}
+
+static void unmap_config_bar(struct xdma_dev *xdev, int config_bar_id)
+{
+ pci_iounmap(xdev->pdev, xdev->config_bar);
+ pci_release_selected_regions(xdev->pdev, 1U<<config_bar_id);
+}
+
+static int map_config_bar(struct xdma_dev *xdev, int config_bar_id)
+{
+ int rv, bar_len;
+
+ bar_len = pci_resource_len(xdev->pdev, config_bar_id);
+ if (bar_len < XDMA_BAR_SIZE) {
+ pr_err("XDMA: %d: Not a config BAR\n", config_bar_id);
+ return -EINVAL;
+ }
+ rv = pci_request_selected_regions(xdev->pdev, 1U<<config_bar_id,
+ "xdma");
+ if (rv) {
+ pr_err("XDMA: Failed to request config BAR memory\n");
+ return rv;
+ }
+ xdev->config_bar = pci_iomap(xdev->pdev, config_bar_id, bar_len);
+ if (!xdev->config_bar) {
+ pr_err("XDMA: Failed to map config BAR memory\n");
+ rv = -ENOMEM;
+ goto err_map;
+ }
+ if (!is_config_bar(xdev->config_bar)) {
+ pr_err("XDMA: %d: Not a config BAR\n", config_bar_id);
+ rv = -EINVAL;
+ goto err_bar;
+ }
+
+ pr_debug("XDMA: Config BAR %d mapped at %p\n",
+ config_bar_id, xdev->config_bar);
+
+ return 0;
+
+err_bar:
+ pci_iounmap(xdev->pdev, xdev->config_bar);
+err_map:
+ pci_release_selected_regions(xdev->pdev, 1U<<config_bar_id);
+
+ return rv;
+}
+
+static void prog_irq_user(struct xdma_dev *xdev, int num_channel, int num_irq,
+ bool clear)
+{
+ struct interrupt_regs *int_regs =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+ int i = num_channel;
+ int max = i + num_irq;
+ int j;
+
+ for (j = 0; i < max; j++) {
+ u32 val = 0;
+ int k, shift = 0;
+
+ if (clear)
+ i += 4;
+ else
+ for (k = 0; k < 4 && i < max; i++, k++, shift += 8)
+ val |= (i & 0x1f) << shift;
+
+ iowrite32(val, &int_regs->user_msi_vector[j]);
+ }
+}
+
+static void prog_irq_channel(struct xdma_dev *xdev, int num_channel, bool clear)
+{
+ struct interrupt_regs *int_regs =
+ (struct interrupt_regs *)(xdev->config_bar + XDMA_OFS_INT_CTRL);
+ int i, j;
+
+ for (i = 0, j = 0; i < num_channel; j++) {
+ u32 val = 0;
+ int k, shift = 0;
+
+ if (clear)
+ i += 4;
+ else
+ for (k = 0; k < 4 && i < num_channel; i++, k++, shift += 8)
+ val |= (i & 0x1f) << shift;
+
+ iowrite32(val, &int_regs->channel_msi_vector[j]);
+ }
+}
+
+static void irq_channel_teardown(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ struct xdma_engine *engine;
+ int i = 0, j = 0;
+
+ engine = xdev->engine_h2c;
+ for (i = 0; i < h2c_channel_max; i++, j++, engine++) {
+ if (!engine->irq_line)
+ break;
+ free_irq(engine->irq_line, engine);
+ }
+
+ engine = xdev->engine_c2h;
+ for (i = 0; i < c2h_channel_max; i++, j++, engine++) {
+ if (!engine->irq_line)
+ break;
+ free_irq(engine->irq_line, engine);
+ }
+}
+
+static int irq_channel_setup(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ int i, j, rv;
+ u32 vector;
+ struct xdma_engine *engine;
+
+ j = h2c_channel_max;
+ engine = xdev->engine_h2c;
+ for (i = 0; i < h2c_channel_max; i++, engine++) {
+ vector = pci_irq_vector(xdev->pdev, i);
+ rv = request_irq(vector, xdma_isr, 0, engine->name, engine);
+ if (rv) {
+ pr_err("XDMA: %s: error requesting irq#%d\n",
+ engine->name, vector);
+ return rv;
+ }
+ pr_info("XDMA: %s: irq#%d\n", engine->name, vector);
+ engine->irq_line = vector;
+ }
+
+ engine = xdev->engine_c2h;
+ for (i = 0; i < c2h_channel_max; i++, j++, engine++) {
+ vector = pci_irq_vector(xdev->pdev, j);
+ rv = request_irq(vector, xdma_isr, 0, engine->name, engine);
+ if (rv) {
+ pr_err("XDMA: %s: error requesting irq#%d\n",
+ engine->name, vector);
+ return rv;
+ }
+ pr_info("XDMA: %s: irq#%d\n", engine->name, vector);
+ engine->irq_line = vector;
+ }
+
+ return 0;
+}
+
+static void irq_teardown(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max, int user_irq_max)
+{
+ int num_channel = h2c_channel_max + c2h_channel_max;
+
+ prog_irq_user(xdev, num_channel, user_irq_max, 1);
+ prog_irq_channel(xdev, num_channel, 1);
+
+ irq_channel_teardown(xdev, h2c_channel_max, c2h_channel_max);
+}
+
+static int irq_setup(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max, int user_irq_max)
+{
+ int rv;
+ int num_channel = h2c_channel_max + c2h_channel_max;
+
+ rv = irq_channel_setup(xdev, h2c_channel_max, c2h_channel_max);
+ if (rv)
+ return rv;
+
+ prog_irq_channel(xdev, num_channel, 0);
+ prog_irq_user(xdev, num_channel, user_irq_max, 0);
+
+ return 0;
+}
+
+/* Chains the descriptors as a singly-linked list
+ *
+ * Each descriptor's next pointer specifies the bus address of the next
+ * descriptor.
+ * Terminates the last descriptor to form a singly-linked list.
+ */
+static void transfer_desc_init(struct xdma_transfer *transfer, int count)
+{
+ struct xdma_desc *desc_virt = transfer->desc_virt;
+ dma_addr_t desc_bus = transfer->desc_bus;
+ int i;
+
+ BUG_ON(count > XDMA_TRANSFER_MAX_DESC);
+
+ /* create singly-linked list for SG DMA controller */
+ for (i = 0; i < count - 1; i++) {
+ /* increment bus address to next in array */
+ desc_bus += sizeof(struct xdma_desc);
+
+ /* singly-linked list uses bus addresses */
+ desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus));
+ desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus));
+ desc_virt[i].bytes = cpu_to_le32(0);
+
+ desc_virt[i].control = cpu_to_le32(DESC_MAGIC);
+ }
+
+ /* zero the last descriptor next pointer */
+ desc_virt[i].next_lo = cpu_to_le32(0);
+ desc_virt[i].next_hi = cpu_to_le32(0);
+ desc_virt[i].bytes = cpu_to_le32(0);
+ desc_virt[i].control = cpu_to_le32(DESC_MAGIC);
+}
+
+/* Set how many descriptors are adjacent to this one */
+static void xdma_desc_adjacent(struct xdma_desc *desc, u32 next_adjacent)
+{
+ /* remember reserved and control bits */
+ u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL;
+ /* merge adjacent and control field */
+ control |= 0xAD4B0000UL | (next_adjacent << 8);
+ /* write control and next_adjacent */
+ desc->control = cpu_to_le32(control);
+}
+
+/* Set complete control field of a descriptor */
+static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field)
+{
+ /* remember magic and adjacent number */
+ u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK);
+
+ /* merge adjacent and control field */
+ control |= control_field;
+ /* write control and next_adjacent */
+ first->control = cpu_to_le32(control);
+}
+
+static inline void xdma_desc_done(struct xdma_desc *desc_virt, int count)
+{
+ memset(desc_virt, 0, count * sizeof(struct xdma_desc));
+}
+
+/* Fill a descriptor with the transfer details */
+static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr,
+ u64 ep_addr, int len, int dir)
+{
+ /* transfer length */
+ desc->bytes = cpu_to_le32(len);
+ if (dir == DMA_TO_DEVICE) {
+ /* read from root complex memory (source address) */
+ desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr));
+ desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr));
+ /* write to end point address (destination address) */
+ desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr));
+ desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr));
+ } else {
+ /* read from end point address (source address) */
+ desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr));
+ desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr));
+ /* write to root complex memory (destination address) */
+ desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr));
+ desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr));
+ }
+}
+
+static void transfer_abort(struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ struct xdma_transfer *head;
+
+ head = list_entry(engine->transfer_list.next, struct xdma_transfer,
+ entry);
+ if (head == transfer)
+ list_del(engine->transfer_list.next);
+ else
+ pr_warn("XDMA: %s: transfer for abort NOT found\n",
+ engine->name);
+
+ if (transfer->state == TRANSFER_STATE_SUBMITTED)
+ transfer->state = TRANSFER_STATE_ABORTED;
+}
+
+static int transfer_queue(struct xdma_engine *engine,
+ struct xdma_transfer *transfer)
+{
+ int rv = 0;
+ unsigned long flags;
+
+ /* lock the engine state */
+ spin_lock_irqsave(&engine->lock, flags);
+
+ engine->prev_cpu = get_cpu();
+ put_cpu();
+
+ /* engine is being shutdown; do not accept new transfers */
+ if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) {
+ pr_info("XDMA: %s: engine offline, transfer not queued\n",
+ engine->name);
+ rv = -EBUSY;
+ goto shutdown;
+ }
+
+ /* mark the transfer as submitted */
+ transfer->state = TRANSFER_STATE_SUBMITTED;
+ /* add transfer to the tail of the engine transfer queue */
+ list_add_tail(&transfer->entry, &engine->transfer_list);
+
+ if (!engine->running)
+ rv = engine_start(engine);
+
+shutdown:
+ spin_unlock_irqrestore(&engine->lock, flags);
+
+ return rv;
+}
+
+static void engine_alignments(struct xdma_engine *engine)
+{
+ u32 w = ioread32(&engine->regs->alignments);
+
+ if (w) {
+ engine->addr_align = (w & 0x00ff0000U) >> 16;
+ engine->len_granularity = (w & 0x0000ff00U) >> 8;
+ engine->addr_bits = (w & 0x000000ffU);
+ } else {
+ /* Some default values if alignments are unspecified */
+ engine->addr_align = 1;
+ engine->len_granularity = 1;
+ engine->addr_bits = 64;
+ }
+}
+
+static void engine_free_resource(struct xdma_engine *engine)
+{
+ struct xdma_dev *xdev = engine->xdev;
+
+ if (engine->desc) {
+ dma_free_coherent(&xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC *
+ sizeof(struct xdma_desc),
+ engine->desc, engine->desc_bus);
+ engine->desc = NULL;
+ }
+
+ if (engine->cyclic_result) {
+ dma_free_coherent(
+ &xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result),
+ engine->cyclic_result, engine->cyclic_result_bus);
+ engine->cyclic_result = NULL;
+ }
+}
+
+static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine)
+{
+ /* Disable interrupts to stop processing new events during shutdown */
+ iowrite32(0x0, &engine->regs->interrupt_enable_mask);
+
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE) {
+ u32 reg_value = (0x1 << engine->channel) << 16;
+ struct sgdma_common_regs *reg =
+ (struct sgdma_common_regs *)
+ (xdev->config_bar + (0x6 * TARGET_SPACING));
+ iowrite32(reg_value, ®->credit_mode_enable_w1c);
+ }
+
+ /* Release memory use for descriptor writebacks */
+ engine_free_resource(engine);
+
+ memset(engine, 0, sizeof(struct xdma_engine));
+ /* Decrement the number of engines available */
+ xdev->engines_num--;
+}
+
+static void engine_init_regs(struct xdma_engine *engine)
+{
+ u32 reg_value;
+
+ iowrite32(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c);
+
+ engine_alignments(engine);
+
+ /* Configure error interrupts by default */
+ reg_value = XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
+ reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED;
+ reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED;
+ reg_value |= XDMA_CTRL_IE_READ_ERROR;
+ reg_value |= XDMA_CTRL_IE_DESC_ERROR;
+
+ /* enable the relevant completion interrupts */
+ reg_value |= XDMA_CTRL_IE_DESC_STOPPED;
+ reg_value |= XDMA_CTRL_IE_DESC_COMPLETED;
+
+ /* Apply engine configurations */
+ iowrite32(reg_value, &engine->regs->interrupt_enable_mask);
+
+ engine->interrupt_enable_mask_value = reg_value;
+
+ /* only enable credit mode for AXI-ST C2H */
+ if (enable_credit_mp && engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE) {
+ struct xdma_dev *xdev = engine->xdev;
+ u32 reg_value = (0x1 << engine->channel) << 16;
+ struct sgdma_common_regs *reg =
+ (struct sgdma_common_regs *)
+ (xdev->config_bar + (0x6 * TARGET_SPACING));
+
+ iowrite32(reg_value, ®->credit_mode_enable_w1s);
+ }
+}
+
+static int engine_alloc_resource(struct xdma_engine *engine)
+{
+ struct xdma_dev *xdev = engine->xdev;
+
+ engine->desc = dma_alloc_coherent(&xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC *
+ sizeof(struct xdma_desc),
+ &engine->desc_bus, GFP_KERNEL);
+ if (!engine->desc)
+ goto err_out;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ engine->cyclic_result = dma_alloc_coherent(
+ &xdev->pdev->dev,
+ XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result),
+ &engine->cyclic_result_bus, GFP_KERNEL);
+
+ if (!engine->cyclic_result)
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ engine_free_resource(engine);
+ return -ENOMEM;
+}
+
+static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev,
+ int offset, enum dma_data_direction dir, int channel)
+{
+ int rv;
+ u32 val;
+
+ engine->channel = channel;
+ engine->xdev = xdev;
+
+ /* engine interrupt request bit */
+ engine->irq_bitmask = (1 << XDMA_ENG_IRQ_NUM) - 1;
+ engine->irq_bitmask <<= (xdev->engines_num * XDMA_ENG_IRQ_NUM);
+
+ /* register address */
+ engine->regs = xdev->config_bar + offset;
+ engine->sgdma_regs = xdev->config_bar + offset +
+ SGDMA_OFFSET_FROM_CHANNEL;
+ val = ioread32(&engine->regs->identifier);
+ if (val & 0x8000U)
+ engine->streaming = 1;
+
+ /* remember SG DMA direction */
+ engine->dir = dir;
+ sprintf(engine->name, "xdma-%s%d%s", (dir == DMA_TO_DEVICE) ? "H2C" : "C2H",
+ channel, engine->streaming ? "ST" : "MM");
+
+ /* initialize the deferred work for transfer completion */
+ INIT_WORK(&engine->work, engine_service_work);
+
+ xdev->engines_num++;
+
+ rv = engine_alloc_resource(engine);
+ if (rv)
+ return rv;
+ engine_init_regs(engine);
+
+ return 0;
+}
+
+static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer)
+{
+ xdma_desc_done(xfer->desc_virt, xfer->desc_num);
+
+ if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) {
+ struct sg_table *sgt = xfer->sgt;
+
+ if (sgt->nents) {
+ dma_unmap_sg(&xdev->pdev->dev, sgt->sgl, sgt->nents,
+ xfer->dir);
+ sgt->nents = 0;
+ }
+ }
+}
+
+static void transfer_build(struct xdma_engine *engine,
+ struct xdma_request_cb *req, struct xdma_transfer *xfer,
+ unsigned int desc_max)
+{
+ struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]);
+ int i, j;
+ dma_addr_t bus = xfer->res_bus;
+
+ for (i = 0, j = 0; i < desc_max; i++, j++, sdesc++) {
+ /* fill in descriptor entry j with transfer details */
+ xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr,
+ sdesc->len, xfer->dir);
+ xfer->len += sdesc->len;
+
+ /* for non-inc-add mode don't increment ep_addr */
+ if (!engine->non_incr_addr)
+ req->ep_addr += sdesc->len;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ memset(xfer->res_virt + j, 0,
+ sizeof(struct xdma_result));
+ xfer->desc_virt[j].src_addr_lo =
+ cpu_to_le32(PCI_DMA_L(bus));
+ xfer->desc_virt[j].src_addr_hi =
+ cpu_to_le32(PCI_DMA_H(bus));
+ bus += sizeof(struct xdma_result);
+ }
+
+ }
+
+ req->sw_desc_idx += desc_max;
+}
+
+static void transfer_init(struct xdma_engine *engine,
+ struct xdma_request_cb *req, struct xdma_transfer *xfer)
+{
+ unsigned int desc_max = min_t(unsigned int,
+ req->sw_desc_cnt - req->sw_desc_idx,
+ XDMA_TRANSFER_MAX_DESC);
+ int i, last;
+ u32 control;
+ unsigned long flags;
+
+ memset(xfer, 0, sizeof(*xfer));
+
+ spin_lock_irqsave(&engine->lock, flags);
+ init_swait_queue_head(&xfer->wq);
+
+ /* remember direction of transfer */
+ xfer->dir = engine->dir;
+ xfer->desc_virt = engine->desc + engine->desc_idx;
+ xfer->res_virt = engine->cyclic_result + engine->desc_idx;
+ xfer->desc_bus = engine->desc_bus +
+ (sizeof(struct xdma_desc) * engine->desc_idx);
+ xfer->res_bus = engine->cyclic_result_bus +
+ (sizeof(struct xdma_result) * engine->desc_idx);
+ xfer->desc_index = engine->desc_idx;
+
+ if ((engine->desc_idx + desc_max) >= XDMA_TRANSFER_MAX_DESC)
+ desc_max = XDMA_TRANSFER_MAX_DESC - engine->desc_idx;
+
+ transfer_desc_init(xfer, desc_max);
+ transfer_build(engine, req, xfer, desc_max);
+
+ xfer->desc_adjacent = desc_max;
+
+ /* terminate last descriptor */
+ last = desc_max - 1;
+ /* stop engine, EOP for AXI ST, req IRQ on last descriptor */
+ control = XDMA_DESC_STOPPED;
+ control |= XDMA_DESC_EOP;
+ control |= XDMA_DESC_COMPLETED;
+ xdma_desc_control_set(xfer->desc_virt + last, control);
+
+ if (engine->eop_flush) {
+ for (i = 0; i < last; i++)
+ xdma_desc_control_set(xfer->desc_virt + i,
+ XDMA_DESC_COMPLETED);
+ xfer->desc_cmpl_th = 1;
+ } else
+ xfer->desc_cmpl_th = desc_max;
+
+ xfer->desc_num = desc_max;
+ engine->desc_idx = (engine->desc_idx + desc_max) % XDMA_TRANSFER_MAX_DESC;
+ engine->desc_used += desc_max;
+
+ /* fill in adjacent numbers */
+ for (i = 0; i < xfer->desc_num; i++) {
+ u32 next_adj = xdma_get_next_adj(xfer->desc_num - i - 1,
+ (xfer->desc_virt + i)->next_lo);
+ xdma_desc_adjacent(xfer->desc_virt + i, next_adj);
+ }
+
+ spin_unlock_irqrestore(&engine->lock, flags);
+}
+
+static void xdma_request_free(struct xdma_request_cb *req)
+{
+ kvfree(req);
+}
+
+static struct xdma_request_cb *xdma_request_alloc(struct xdma_dev *xdev,
+ unsigned int sdesc_nr)
+{
+ unsigned int size = sizeof(struct xdma_request_cb) +
+ sdesc_nr * sizeof(struct sw_desc);
+
+ return kvzalloc(size, GFP_KERNEL);
+}
+
+static struct xdma_request_cb *xdma_init_request(struct xdma_dev *xdev,
+ struct sg_table *sgt,
+ u64 ep_addr)
+{
+ struct xdma_request_cb *req;
+ struct scatterlist *sg = sgt->sgl;
+ int max = sgt->nents;
+ int extra = 0;
+ int i, j = 0;
+
+ for (i = 0; i < max; i++, sg = sg_next(sg)) {
+ unsigned int len = sg_dma_len(sg);
+
+ if (unlikely(len > XDMA_DESC_BLEN_MAX))
+ extra += (len + XDMA_DESC_BLEN_MAX - 1) / XDMA_DESC_BLEN_MAX;
+ }
+
+ max += extra;
+ req = xdma_request_alloc(xdev, max);
+ if (!req)
+ return NULL;
+
+ req->sgt = sgt;
+ req->ep_addr = ep_addr;
+
+ for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) {
+ unsigned int tlen = sg_dma_len(sg);
+ dma_addr_t addr = sg_dma_address(sg);
+
+ req->total_len += tlen;
+ while (tlen) {
+ req->sdesc[j].addr = addr;
+ if (tlen > XDMA_DESC_BLEN_MAX) {
+ req->sdesc[j].len = XDMA_DESC_BLEN_MAX;
+ addr += XDMA_DESC_BLEN_MAX;
+ tlen -= XDMA_DESC_BLEN_MAX;
+ } else {
+ req->sdesc[j].len = tlen;
+ tlen = 0;
+ }
+ j++;
+ }
+ }
+
+ if (j > max) {
+ pr_err("XDMA: Max. transfer length (%d) exceeded",
+ XDMA_DESC_BLEN_MAX);
+ xdma_request_free(req);
+ return NULL;
+ }
+ req->sw_desc_cnt = j;
+
+ return req;
+}
+
+static struct xdma_engine *channel_engine(struct xdma_core *xdma, int channel,
+ bool write)
+{
+ if (write) {
+ if (channel >= xdma->h2c_channel_max) {
+ pr_err("XDMA: %d: invalid H2C channel\n", channel);
+ return NULL;
+ } else
+ return &xdma->xdev->engine_h2c[channel];
+ } else {
+ if (channel >= xdma->c2h_channel_max) {
+ pr_err("XDMA: %d: invalid C2H channel\n", channel);
+ return NULL;
+ } else
+ return &xdma->xdev->engine_c2h[channel];
+ }
+}
+
+static struct xdma_dev *alloc_dev(struct pci_dev *pdev)
+{
+ int i;
+ struct xdma_dev *xdev;
+ struct xdma_engine *engine;
+
+ xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL);
+ if (!xdev)
+ return NULL;
+
+ xdev->pdev = pdev;
+
+ engine = xdev->engine_h2c;
+ for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) {
+ spin_lock_init(&engine->lock);
+ mutex_init(&engine->desc_lock);
+ INIT_LIST_HEAD(&engine->transfer_list);
+ init_swait_queue_head(&engine->shutdown_wq);
+ }
+
+ engine = xdev->engine_c2h;
+ for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) {
+ spin_lock_init(&engine->lock);
+ mutex_init(&engine->desc_lock);
+ INIT_LIST_HEAD(&engine->transfer_list);
+ init_swait_queue_head(&engine->shutdown_wq);
+ }
+
+ return xdev;
+}
+
+static int set_dma_mask(struct xdma_dev *xdev)
+{
+ if (!dma_set_mask(&xdev->pdev->dev, DMA_BIT_MASK(64))) {
+ pr_devel("XDMA: Using a 64-bit DMA mask\n");
+ /* use 32-bit DMA for descriptors */
+ dma_set_coherent_mask(&xdev->pdev->dev, DMA_BIT_MASK(32));
+ } else if (!dma_set_mask(&xdev->pdev->dev, DMA_BIT_MASK(32))) {
+ pr_devel("XDMA: Using a 32-bit DMA mask\n");
+ dma_set_coherent_mask(&xdev->pdev->dev, DMA_BIT_MASK(32));
+ } else {
+ pr_err("XDMA: No suitable DMA possible.\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int get_engine_channel_id(struct engine_regs *regs)
+{
+ int value = ioread32(®s->identifier);
+
+ return (value & 0x00000f00U) >> 8;
+}
+
+static int get_engine_id(struct engine_regs *regs)
+{
+ int value = ioread32(®s->identifier);
+
+ return (value & 0xffff0000U) >> 16;
+}
+
+static void remove_engines(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ int i;
+
+ for (i = 0; i < h2c_channel_max; i++)
+ engine_destroy(xdev, &xdev->engine_h2c[i]);
+
+ for (i = 0; i < c2h_channel_max; i++)
+ engine_destroy(xdev, &xdev->engine_c2h[i]);
+}
+
+static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir,
+ int channel)
+{
+ struct engine_regs *regs;
+ int offset = channel * CHANNEL_SPACING;
+ u32 engine_id;
+ u32 engine_id_expected;
+ u32 channel_id;
+ struct xdma_engine *engine;
+
+ if (dir == DMA_TO_DEVICE) {
+ engine_id_expected = XDMA_ID_H2C;
+ engine = &xdev->engine_h2c[channel];
+ } else {
+ offset += H2C_CHANNEL_OFFSET;
+ engine_id_expected = XDMA_ID_C2H;
+ engine = &xdev->engine_c2h[channel];
+ }
+
+ regs = xdev->config_bar + offset;
+ engine_id = get_engine_id(regs);
+ channel_id = get_engine_channel_id(regs);
+
+ if ((engine_id != engine_id_expected) || (channel_id != channel)) {
+ pr_err("XDMA: %s engine #%d not found\n",
+ dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel);
+ return -EINVAL;
+ }
+
+ engine_init(engine, xdev, offset, dir, channel);
+
+ return 0;
+}
+
+static int probe_engines(struct xdma_dev *xdev, int h2c_channel_max,
+ int c2h_channel_max)
+{
+ int i, rv;
+
+ for (i = 0; i < h2c_channel_max; i++) {
+ rv = probe_for_engine(xdev, DMA_TO_DEVICE, i);
+ if (rv)
+ return rv;
+ }
+
+ for (i = 0; i < c2h_channel_max; i++) {
+ rv = probe_for_engine(xdev, DMA_FROM_DEVICE, i);
+ if (rv)
+ return rv;
+ }
+
+ return 0;
+}
+
+
+int xdma_probe(struct xdma_core *xdma)
+{
+ int rv;
+
+ if (xdma->user_irq_max > MAX_USER_IRQ) {
+ pr_err("XDMA: %d: Invalid number of user IRQs\n",
+ xdma->user_irq_max);
+ return -EINVAL;
+ }
+ if (xdma->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) {
+ pr_err("XDMA: %d: Invalid number of H2C channels\n",
+ xdma->h2c_channel_max);
+ return -EINVAL;
+ }
+ if (xdma->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) {
+ pr_err("XDMA: %d: Invalid number of C2H channels\n",
+ xdma->c2h_channel_max);
+ return -EINVAL;
+ }
+
+ xdma->xdev = alloc_dev(xdma->pdev);
+ if (!xdma->xdev)
+ return -ENOMEM;
+
+ rv = map_config_bar(xdma->xdev, xdma->config_bar_id);
+ if (rv)
+ goto err_map;
+
+ rv = set_dma_mask(xdma->xdev);
+ if (rv)
+ goto err_mask;
+
+ channel_interrupts_disable(xdma->xdev, ~0);
+ user_interrupts_disable(xdma->xdev, ~0);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+
+ rv = probe_engines(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max);
+ if (rv)
+ goto err_engines;
+
+ rv = irq_setup(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max,
+ xdma->user_irq_max);
+ if (rv < 0)
+ goto err_interrupts;
+ channel_interrupts_enable(xdma->xdev, ~0);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+
+ return 0;
+
+err_interrupts:
+ irq_teardown(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max,
+ xdma->user_irq_max);
+err_engines:
+ remove_engines(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max);
+err_mask:
+ unmap_config_bar(xdma->xdev, xdma->config_bar_id);
+err_map:
+ kfree(xdma->xdev);
+
+ return rv;
+}
+EXPORT_SYMBOL_GPL(xdma_probe);
+
+void xdma_remove(struct xdma_core *xdma)
+{
+ channel_interrupts_disable(xdma->xdev, ~0);
+ user_interrupts_disable(xdma->xdev, ~0);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+
+ irq_teardown(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max,
+ xdma->user_irq_max);
+
+ remove_engines(xdma->xdev, xdma->h2c_channel_max, xdma->c2h_channel_max);
+ unmap_config_bar(xdma->xdev, xdma->config_bar_id);
+
+ kfree(xdma->xdev);
+}
+EXPORT_SYMBOL_GPL(xdma_remove);
+
+/**
+ * xdma_irq_enable - enable XDMA user interrupt(s)
+ * @xdma: XDMA device handle
+ * @mask: bitmask of user interrupts (0 ~ 15) to be registered
+ */
+void xdma_irq_enable(struct xdma_core *xdma, unsigned int mask)
+{
+ xdma->xdev->mask_irq_user |= mask;
+ user_interrupts_enable(xdma->xdev, mask);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+}
+EXPORT_SYMBOL_GPL(xdma_irq_enable);
+
+/**
+ * xdma_irq_disable - disable XDMA user interrupt(s)
+ * @xdma: XDMA device handle
+ * @mask: bitmask of user interrupts (0 ~ 15) to be unregistered
+ */
+void xdma_irq_disable(struct xdma_core *xdma, unsigned int mask)
+{
+ xdma->xdev->mask_irq_user &= ~mask;
+ user_interrupts_disable(xdma->xdev, mask);
+ /* Flush writes */
+ read_interrupts(xdma->xdev);
+}
+EXPORT_SYMBOL_GPL(xdma_irq_disable);
+
+/**
+ * xdma_transfer - do a DMA transfer
+ * @xdma: XDMA device handle
+ * @channel: channel number
+ * @write: slecets read/write operation
+ * @ep_addr: offset into the DDR/BRAM (card) memory to read from or write to
+ * @sg_table: the scatter-gather list of data buffers
+ * @timeout_ms: timeout in mili-seconds
+ *
+ * Returns # of bytes transferred on success, negative on failure
+ */
+ssize_t xdma_transfer(struct xdma_core *xdma, int channel, bool write,
+ u64 ep_addr, struct sg_table *sgt, int timeout_ms)
+{
+ struct xdma_engine *engine;
+ int rv = 0, i, nents;
+ ssize_t done = 0;
+ struct xdma_request_cb *req = NULL;
+
+
+ engine = channel_engine(xdma, channel, write);
+ if (!engine)
+ return -EINVAL;
+
+ req = xdma_init_request(xdma->xdev, sgt, ep_addr);
+ if (!req)
+ return -ENOMEM;
+
+ nents = req->sw_desc_cnt;
+ mutex_lock(&engine->desc_lock);
+
+ while (nents) {
+ unsigned long flags;
+ struct xdma_transfer *xfer;
+
+ /* build transfer */
+ transfer_init(engine, req, &req->tfer);
+ xfer = &req->tfer;
+
+ /* last transfer for the given request? */
+ nents -= xfer->desc_num;
+ if (!nents) {
+ xfer->last_in_request = 1;
+ xfer->sgt = sgt;
+ }
+
+ rv = transfer_queue(engine, xfer);
+ if (rv < 0)
+ break;
+
+ if (timeout_ms > 0)
+ swait_event_interruptible_timeout_exclusive(xfer->wq,
+ (xfer->state != TRANSFER_STATE_SUBMITTED),
+ msecs_to_jiffies(timeout_ms));
+ else
+ swait_event_interruptible_exclusive(xfer->wq,
+ (xfer->state != TRANSFER_STATE_SUBMITTED));
+
+ spin_lock_irqsave(&engine->lock, flags);
+
+ switch (xfer->state) {
+ case TRANSFER_STATE_COMPLETED:
+ spin_unlock_irqrestore(&engine->lock, flags);
+ /* For C2H streaming use writeback results */
+ if (engine->streaming &&
+ engine->dir == DMA_FROM_DEVICE) {
+ struct xdma_result *result = xfer->res_virt;
+
+ for (i = 0; i < xfer->desc_cmpl; i++)
+ done += result[i].length;
+
+ /* finish the whole request */
+ if (engine->eop_flush)
+ nents = 0;
+ } else
+ done += xfer->len;
+ rv = 0;
+ break;
+ case TRANSFER_STATE_FAILED:
+ pr_warn("XDMA: transfer failed\n");
+ spin_unlock_irqrestore(&engine->lock, flags);
+ rv = -EIO;
+ break;
+ default:
+ /* transfer can still be in-flight */
+ pr_warn("XDMA: transfer timed out\n");
+ engine_status_read(engine, 0, 1);
+ transfer_abort(engine, xfer);
+ engine_stop(engine);
+ spin_unlock_irqrestore(&engine->lock, flags);
+ rv = -ERESTARTSYS;
+ break;
+ }
+
+ engine->desc_used -= xfer->desc_num;
+ transfer_destroy(xdma->xdev, xfer);
+
+ if (rv < 0)
+ break;
+ }
+
+ mutex_unlock(&engine->desc_lock);
+ xdma_request_free(req);
+
+ return rv ? rv : done;
+}
+EXPORT_SYMBOL_GPL(xdma_transfer);
+
+MODULE_AUTHOR("Digiteq Automotive s.r.o.");
+MODULE_DESCRIPTION("Xilinx XDMA Driver");
+MODULE_LICENSE("GPL");
new file mode 100644
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2016-2021, Xilinx, Inc.
+ * Copyright (c) 2020-2022, Digiteq Automotive s.r.o.
+ */
+
+#ifndef XILINX_XDMA_H
+#define XILINX_XDMA_H
+
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+
+struct xdma_dev;
+
+/**
+ * struct xdma_core - representation of XDMA hardware
+ * @pdev: The parent PCIe device which contains the XDMA core
+ * @config_bar_id: PCI BAR id where XDMA config regs are located
+ * @user_irq_max: number of user IRQs
+ * @c2h_channel_max: number of C2H DMA channels
+ * @h2c_channel_max: number of H2C DMA channels
+ * @xdev: struct xdma_dev that is filed by ->probe()
+ */
+struct xdma_core {
+ struct pci_dev *pdev;
+ int config_bar_id;
+ unsigned int user_irq_max;
+ unsigned int c2h_channel_max;
+ unsigned int h2c_channel_max;
+ struct xdma_dev *xdev;
+};
+
+int xdma_probe(struct xdma_core *xdma);
+void xdma_remove(struct xdma_core *xdma);
+
+void xdma_irq_enable(struct xdma_core *xdma, unsigned int mask);
+void xdma_irq_disable(struct xdma_core *xdma, unsigned int mask);
+
+ssize_t xdma_transfer(struct xdma_core *xdma, int channel, bool write,
+ u64 ep_addr, struct sg_table *sgt, int timeout_ms);
+
+#endif /* XILINX_XDMA_H */