@@ -31,6 +31,7 @@
#include <asm/processor-flags.h>
#include <asm/msr.h>
#include <asm/cmdline.h>
+#include <asm/mshyperv.h>
#include "mm_internal.h"
@@ -4,6 +4,7 @@
#include <linux/dma-map-ops.h>
#include <linux/pci.h>
+#include <linux/hyperv.h>
#include <xen/swiotlb-xen.h>
#include <asm/xen/hypervisor.h>
@@ -91,6 +92,6 @@ int pci_xen_swiotlb_init_late(void)
EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late);
IOMMU_INIT_FINISH(pci_xen_swiotlb_detect,
- NULL,
+ hyperv_swiotlb_detect,
pci_xen_swiotlb_init,
NULL);
@@ -9,6 +9,7 @@ config HYPERV
select PARAVIRT
select X86_HV_CALLBACK_VECTOR if X86
select VMAP_PFN
+ select DMA_OPS_BYPASS
help
Select this option to run Linux as a Hyper-V client operating
system.
@@ -33,6 +33,7 @@
#include <linux/random.h>
#include <linux/kernel.h>
#include <linux/syscore_ops.h>
+#include <linux/dma-map-ops.h>
#include <clocksource/hyperv_timer.h>
#include "hyperv_vmbus.h"
@@ -2078,6 +2079,7 @@ struct hv_device *vmbus_device_create(const guid_t *type,
return child_device_obj;
}
+static u64 vmbus_dma_mask = DMA_BIT_MASK(64);
/*
* vmbus_device_register - Register the child device
*/
@@ -2118,6 +2120,10 @@ int vmbus_device_register(struct hv_device *child_device_obj)
}
hv_debug_add_dev_dir(child_device_obj);
+ child_device_obj->device.dma_ops_bypass = true;
+ child_device_obj->device.dma_ops = &hyperv_iommu_dma_ops;
+ child_device_obj->device.dma_mask = &vmbus_dma_mask;
+ child_device_obj->device.dma_parms = &child_device_obj->dma_parms;
return 0;
err_kset_unregister:
@@ -13,14 +13,21 @@
#include <linux/irq.h>
#include <linux/iommu.h>
#include <linux/module.h>
+#include <linux/hyperv.h>
+#include <linux/io.h>
#include <asm/apic.h>
#include <asm/cpu.h>
#include <asm/hw_irq.h>
#include <asm/io_apic.h>
+#include <asm/iommu.h>
+#include <asm/iommu_table.h>
#include <asm/irq_remapping.h>
#include <asm/hypervisor.h>
#include <asm/mshyperv.h>
+#include <asm/swiotlb.h>
+#include <linux/dma-map-ops.h>
+#include <linux/dma-direct.h>
#include "irq_remapping.h"
@@ -337,4 +344,161 @@ static const struct irq_domain_ops hyperv_root_ir_domain_ops = {
.free = hyperv_root_irq_remapping_free,
};
+static void __init hyperv_iommu_swiotlb_init(void)
+{
+ unsigned long hyperv_io_tlb_size;
+ void *hyperv_io_tlb_start;
+
+ /*
+ * Allocate Hyper-V swiotlb bounce buffer at early place
+ * to reserve large contiguous memory.
+ */
+ hyperv_io_tlb_size = swiotlb_size_or_default();
+ hyperv_io_tlb_start = memblock_alloc(hyperv_io_tlb_size, PAGE_SIZE);
+
+ if (!hyperv_io_tlb_start)
+ pr_warn("Fail to allocate Hyper-V swiotlb buffer.\n");
+
+ swiotlb_init_with_tbl(hyperv_io_tlb_start,
+ hyperv_io_tlb_size >> IO_TLB_SHIFT, true);
+}
+
+int __init hyperv_swiotlb_detect(void)
+{
+ if (!hypervisor_is_type(X86_HYPER_MS_HYPERV))
+ return 0;
+
+ if (!hv_is_isolation_supported())
+ return 0;
+
+ /*
+ * Enable swiotlb force mode in Isolation VM to
+ * use swiotlb bounce buffer for dma transaction.
+ */
+ if (hv_isolation_type_snp())
+ swiotlb_unencrypted_base = ms_hyperv.shared_gpa_boundary;
+ swiotlb_force = SWIOTLB_FORCE;
+ return 1;
+}
+
+static void __init hyperv_iommu_swiotlb_later_init(void)
+{
+ /*
+ * Swiotlb bounce buffer needs to be mapped in extra address
+ * space. Map function doesn't work in the early place and so
+ * call swiotlb_update_mem_attributes() here.
+ */
+ swiotlb_update_mem_attributes();
+}
+
+IOMMU_INIT_FINISH(hyperv_swiotlb_detect,
+ NULL, hyperv_iommu_swiotlb_init,
+ hyperv_iommu_swiotlb_later_init);
+
+static struct sg_table *hyperv_dma_alloc_noncontiguous(struct device *dev,
+ size_t size, enum dma_data_direction dir, gfp_t gfp,
+ unsigned long attrs)
+{
+ struct dma_sgt_handle *sh;
+ struct page **pages;
+ int num_pages = size >> PAGE_SHIFT;
+ void *vaddr, *ptr;
+ int rc, i;
+
+ if (!hv_isolation_type_snp())
+ return NULL;
+
+ sh = kmalloc(sizeof(*sh), gfp);
+ if (!sh)
+ return NULL;
+
+ vaddr = vmalloc(size);
+ if (!vaddr)
+ goto free_sgt;
+
+ pages = kvmalloc_array(num_pages, sizeof(struct page *),
+ GFP_KERNEL | __GFP_ZERO);
+ if (!pages)
+ goto free_mem;
+
+ for (i = 0, ptr = vaddr; i < num_pages; ++i, ptr += PAGE_SIZE)
+ pages[i] = vmalloc_to_page(ptr);
+
+ rc = sg_alloc_table_from_pages(&sh->sgt, pages, num_pages, 0, size, GFP_KERNEL);
+ if (rc)
+ goto free_pages;
+
+ sh->sgt.sgl->dma_address = (dma_addr_t)vaddr;
+ sh->sgt.sgl->dma_length = size;
+ sh->pages = pages;
+
+ return &sh->sgt;
+
+free_pages:
+ kvfree(pages);
+free_mem:
+ vfree(vaddr);
+free_sgt:
+ kfree(sh);
+ return NULL;
+}
+
+static void hyperv_dma_free_noncontiguous(struct device *dev, size_t size,
+ struct sg_table *sgt, enum dma_data_direction dir)
+{
+ struct dma_sgt_handle *sh = sgt_handle(sgt);
+
+ if (!hv_isolation_type_snp())
+ return;
+
+ vfree((void *)sh->sgt.sgl->dma_address);
+ sg_free_table(&sh->sgt);
+ kvfree(sh->pages);
+ kfree(sh);
+}
+
+static void *hyperv_dma_vmap_noncontiguous(struct device *dev, size_t size,
+ struct sg_table *sgt)
+{
+ int pg_count = size >> PAGE_SHIFT;
+ unsigned long *pfns;
+ struct page **pages = sgt_handle(sgt)->pages;
+ void *vaddr = NULL;
+ int i;
+
+ if (!hv_isolation_type_snp())
+ return NULL;
+
+ if (!pages)
+ return NULL;
+
+ pfns = kcalloc(pg_count, sizeof(*pfns), GFP_KERNEL);
+ if (!pfns)
+ return NULL;
+
+ for (i = 0; i < pg_count; i++)
+ pfns[i] = page_to_pfn(pages[i]) +
+ (ms_hyperv.shared_gpa_boundary >> PAGE_SHIFT);
+
+ vaddr = vmap_pfn(pfns, pg_count, PAGE_KERNEL);
+ kfree(pfns);
+ return vaddr;
+
+}
+
+static void hyperv_dma_vunmap_noncontiguous(struct device *dev, void *addr)
+{
+ if (!hv_isolation_type_snp())
+ return;
+ vunmap(addr);
+}
+
+const struct dma_map_ops hyperv_iommu_dma_ops = {
+ .alloc_noncontiguous = hyperv_dma_alloc_noncontiguous,
+ .free_noncontiguous = hyperv_dma_free_noncontiguous,
+ .vmap_noncontiguous = hyperv_dma_vmap_noncontiguous,
+ .vunmap_noncontiguous = hyperv_dma_vunmap_noncontiguous,
+};
+EXPORT_SYMBOL_GPL(hyperv_iommu_dma_ops);
+
#endif
@@ -1726,6 +1726,16 @@ int hyperv_write_cfg_blk(struct pci_dev *dev, void *buf, unsigned int len,
int hyperv_reg_block_invalidate(struct pci_dev *dev, void *context,
void (*block_invalidate)(void *context,
u64 block_mask));
+#ifdef CONFIG_HYPERV
+int __init hyperv_swiotlb_detect(void);
+#else
+static inline int __init hyperv_swiotlb_detect(void)
+{
+ return 0;
+}
+#endif
+
+extern const struct dma_map_ops hyperv_iommu_dma_ops;
struct hyperv_pci_block_ops {
int (*read_block)(struct pci_dev *dev, void *buf, unsigned int buf_len,