diff --git a/docs/man/xl.cfg.5.pod.in b/docs/man/xl.cfg.5.pod.in index 8e1422104e50..10ed4590d25d 100644 --- a/docs/man/xl.cfg.5.pod.in +++ b/docs/man/xl.cfg.5.pod.in @@ -3094,6 +3094,19 @@ assigned to the domain. =back +=over 4 + +=item B + +To enable viommu, user must specify the following option in the VM +config file: + +viommu = "smmuv3" + +Currently, only the "smmuv3" type is supported for ARM. + +=back + =head3 x86 =over 4 diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc index dba5d7f31ed6..e4e6dd073017 100644 --- a/docs/misc/xen-command-line.pandoc +++ b/docs/misc/xen-command-line.pandoc @@ -2058,6 +2058,13 @@ This option can be specified more than once (up to 8 times at present). Flag to enable or disable support for PCI passthrough +### viommu (arm) +> `= ` + +> Default: `false` + +Flag to enable or disable support for Virtual IOMMU for guests. + ### pcid (x86) > `= | xpti=` diff --git a/tools/golang/xenlight/helpers.gen.go b/tools/golang/xenlight/helpers.gen.go index 90846ea8e882..ba66bff4aee0 100644 --- a/tools/golang/xenlight/helpers.gen.go +++ b/tools/golang/xenlight/helpers.gen.go @@ -1163,6 +1163,7 @@ x.ArchArm.GicVersion = GicVersion(xc.arch_arm.gic_version) x.ArchArm.Vuart = VuartType(xc.arch_arm.vuart) x.ArchArm.SveVl = SveType(xc.arch_arm.sve_vl) x.ArchArm.NrSpis = uint32(xc.arch_arm.nr_spis) +x.ArchArm.Viommu = ViommuType(xc.arch_arm.viommu) if err := x.ArchX86.MsrRelaxed.fromC(&xc.arch_x86.msr_relaxed);err != nil { return fmt.Errorf("converting field ArchX86.MsrRelaxed: %v", err) } @@ -1688,6 +1689,7 @@ xc.arch_arm.gic_version = C.libxl_gic_version(x.ArchArm.GicVersion) xc.arch_arm.vuart = C.libxl_vuart_type(x.ArchArm.Vuart) xc.arch_arm.sve_vl = C.libxl_sve_type(x.ArchArm.SveVl) xc.arch_arm.nr_spis = C.uint32_t(x.ArchArm.NrSpis) +xc.arch_arm.viommu = C.libxl_viommu_type(x.ArchArm.Viommu) if err := x.ArchX86.MsrRelaxed.toC(&xc.arch_x86.msr_relaxed); err != nil { return fmt.Errorf("converting field ArchX86.MsrRelaxed: %v", err) } diff --git a/tools/golang/xenlight/types.gen.go b/tools/golang/xenlight/types.gen.go index e7667f1ce3a3..ca0b974d8109 100644 --- a/tools/golang/xenlight/types.gen.go +++ b/tools/golang/xenlight/types.gen.go @@ -599,6 +599,7 @@ GicVersion GicVersion Vuart VuartType SveVl SveType NrSpis uint32 +Viommu ViommuType } ArchX86 struct { MsrRelaxed Defbool diff --git a/tools/include/libxl.h b/tools/include/libxl.h index f8fe4afd7dca..93e4b43eba01 100644 --- a/tools/include/libxl.h +++ b/tools/include/libxl.h @@ -313,6 +313,11 @@ */ #define LIBXL_HAVE_BUILDINFO_ARCH_NR_SPIS 1 +/* + * libxl_domain_build_info has the arch_arm.viommu_type field. + */ +#define LIBXL_HAVE_BUILDINFO_ARM_VIOMMU 1 + /* * LIBXL_HAVE_SOFT_RESET indicates that libxl supports performing * 'soft reset' for domains and there is 'soft_reset' shutdown reason diff --git a/tools/libs/light/libxl_arm.c b/tools/libs/light/libxl_arm.c index 5a9db5e85f6f..272c77bc3637 100644 --- a/tools/libs/light/libxl_arm.c +++ b/tools/libs/light/libxl_arm.c @@ -86,8 +86,8 @@ int libxl__arch_domain_prepare_config(libxl__gc *gc, { uint32_t nr_spis = 0; unsigned int i; - uint32_t vuart_irq, virtio_irq = 0; - bool vuart_enabled = false, virtio_enabled = false; + uint32_t vuart_irq, virtio_irq = 0, vsmmu_irq = 0; + bool vuart_enabled = false, virtio_enabled = false, vsmmu_enabled = false; uint64_t virtio_mmio_base = GUEST_VIRTIO_MMIO_BASE; uint32_t virtio_mmio_irq = GUEST_VIRTIO_MMIO_SPI_FIRST; int rc; @@ -102,6 +102,12 @@ int libxl__arch_domain_prepare_config(libxl__gc *gc, vuart_enabled = true; } + if (d_config->num_pcidevs || d_config->b_info.device_tree) { + nr_spis += (GUEST_VSMMU_SPI - 32) + 1; + vsmmu_irq = GUEST_VSMMU_SPI; + vsmmu_enabled = true; + } + for (i = 0; i < d_config->num_disks; i++) { libxl_device_disk *disk = &d_config->disks[i]; @@ -170,6 +176,11 @@ int libxl__arch_domain_prepare_config(libxl__gc *gc, return ERROR_FAIL; } + if (vsmmu_enabled && irq == vsmmu_irq) { + LOG(ERROR, "Physical IRQ %u conflicting with vSMMUv3 SPI\n", irq); + return ERROR_FAIL; + } + if (irq < 32) continue; @@ -222,6 +233,19 @@ int libxl__arch_domain_prepare_config(libxl__gc *gc, config->arch.sve_vl = d_config->b_info.arch_arm.sve_vl / 128U; } + switch (d_config->b_info.arch_arm.viommu_type) { + case LIBXL_VIOMMU_TYPE_NONE: + config->arch.viommu_type = XEN_DOMCTL_CONFIG_VIOMMU_NONE; + break; + case LIBXL_VIOMMU_TYPE_SMMUV3: + config->arch.viommu_type = XEN_DOMCTL_CONFIG_VIOMMU_SMMUV3; + break; + default: + LOG(ERROR, "Unknown vIOMMU type %d", + d_config->b_info.arch_arm.viommu_type); + return ERROR_FAIL; + } + return 0; } @@ -861,6 +885,45 @@ static int make_vpl011_uart_node(libxl__gc *gc, void *fdt, return 0; } +static int make_vsmmuv3_node(libxl__gc *gc, void *fdt, + const struct arch_info *ainfo, + struct xc_dom_image *dom) +{ + int res; + const char *name = GCSPRINTF("iommu@%llx", GUEST_VSMMUV3_BASE); + gic_interrupt intr; + + res = fdt_begin_node(fdt, name); + if (res) return res; + + res = fdt_property_compat(gc, fdt, 1, "arm,smmu-v3"); + if (res) return res; + + res = fdt_property_regs(gc, fdt, GUEST_ROOT_ADDRESS_CELLS, + GUEST_ROOT_SIZE_CELLS, 1, GUEST_VSMMUV3_BASE, + GUEST_VSMMUV3_SIZE); + if (res) return res; + + res = fdt_property_cell(fdt, "phandle", GUEST_PHANDLE_VSMMUV3); + if (res) return res; + + res = fdt_property_cell(fdt, "#iommu-cells", 1); + if (res) return res; + + res = fdt_property_string(fdt, "interrupt-names", "combined"); + if (res) return res; + + set_interrupt(intr, GUEST_VSMMU_SPI, 0xf, DT_IRQ_TYPE_LEVEL_HIGH); + + res = fdt_property_interrupts(gc, fdt, &intr, 1); + if (res) return res; + + res = fdt_end_node(fdt); + if (res) return res; + + return 0; +} + static int make_vpci_node(libxl__gc *gc, void *fdt, const struct arch_info *ainfo, struct xc_dom_image *dom) @@ -902,6 +965,10 @@ static int make_vpci_node(libxl__gc *gc, void *fdt, GUEST_VPCI_PREFETCH_MEM_SIZE); if (res) return res; + res = fdt_property_values(gc, fdt, "iommu-map", 4, 0, + GUEST_PHANDLE_VSMMUV3, 0, 0x10000); + if (res) return res; + res = fdt_end_node(fdt); if (res) return res; @@ -1228,6 +1295,41 @@ static int copy_partial_fdt(libxl__gc *gc, void *fdt, void *pfdt) return 0; } +static int modify_partial_fdt(libxl__gc *gc, void *pfdt) +{ + int nodeoff, proplen, i, r; + const fdt32_t *prop; + fdt32_t *prop_c; + + nodeoff = fdt_path_offset(pfdt, "/passthrough"); + if (nodeoff < 0) + return nodeoff; + + for (nodeoff = fdt_first_subnode(pfdt, nodeoff); + nodeoff >= 0; + nodeoff = fdt_next_subnode(pfdt, nodeoff)) { + + prop = fdt_getprop(pfdt, nodeoff, "iommus", &proplen); + if (!prop) + continue; + + prop_c = libxl__zalloc(gc, proplen); + + for (i = 0; i < proplen / 8; ++i) { + prop_c[i * 2] = cpu_to_fdt32(GUEST_PHANDLE_VSMMUV3); + prop_c[i * 2 + 1] = prop[i * 2 + 1]; + } + + r = fdt_setprop(pfdt, nodeoff, "iommus", prop_c, proplen); + if (r) { + LOG(ERROR, "Can't set the iommus property in partial FDT"); + return r; + } + } + + return 0; +} + #else static int check_partial_fdt(libxl__gc *gc, void *fdt, size_t size) @@ -1246,6 +1348,13 @@ static int copy_partial_fdt(libxl__gc *gc, void *fdt, void *pfdt) return -FDT_ERR_INTERNAL; } +static int modify_partial_fdt(libxl__gc *gc, void *pfdt) +{ + LOG(ERROR, "partial device tree not supported"); + + return ERROR_FAIL; +} + #endif /* ENABLE_PARTIAL_DEVICE_TREE */ #define FDT_MAX_SIZE (1<<20) @@ -1368,6 +1477,12 @@ static int libxl__prepare_dtb(libxl__gc *gc, libxl_domain_config *d_config, if (d_config->num_pcidevs) FDT( make_vpci_node(gc, fdt, ainfo, dom) ); + if (info->arch_arm.viommu_type == LIBXL_VIOMMU_TYPE_SMMUV3) { + FDT( make_vsmmuv3_node(gc, fdt, ainfo, dom) ); + if (pfdt) + FDT( modify_partial_fdt(gc, pfdt) ); + } + for (i = 0; i < d_config->num_disks; i++) { libxl_device_disk *disk = &d_config->disks[i]; diff --git a/tools/libs/light/libxl_types.idl b/tools/libs/light/libxl_types.idl index bd4b8721ff19..58a74b476970 100644 --- a/tools/libs/light/libxl_types.idl +++ b/tools/libs/light/libxl_types.idl @@ -551,6 +551,11 @@ libxl_sve_type = Enumeration("sve_type", [ (2048, "2048") ], init_val = "LIBXL_SVE_TYPE_DISABLED") +libxl_viommu_type = Enumeration("viommu_type", [ + (0, "none"), + (1, "smmuv3") + ], init_val = "LIBXL_VIOMMU_TYPE_NONE") + libxl_rdm_reserve = Struct("rdm_reserve", [ ("strategy", libxl_rdm_reserve_strategy), ("policy", libxl_rdm_reserve_policy), @@ -723,7 +728,8 @@ libxl_domain_build_info = Struct("domain_build_info",[ ("arch_arm", Struct(None, [("gic_version", libxl_gic_version), ("vuart", libxl_vuart_type), ("sve_vl", libxl_sve_type), - ("nr_spis", uint32), + ("nr_spis", uint32, {'init_val': 'LIBXL_NR_SPIS_DEFAULT'}), + ("viommu_type", libxl_viommu_type), ])), ("arch_x86", Struct(None, [("msr_relaxed", libxl_defbool), ])), diff --git a/tools/xl/xl_parse.c b/tools/xl/xl_parse.c index 089a88935aff..6cfbb66e6850 100644 --- a/tools/xl/xl_parse.c +++ b/tools/xl/xl_parse.c @@ -2975,6 +2975,19 @@ void parse_config_data(const char *config_source, if (!xlu_cfg_get_long (config, "nr_spis", &l, 0)) b_info->arch_arm.nr_spis = l; + xlu_cfg_get_defbool(config, "trap_unmapped_accesses", + &b_info->trap_unmapped_accesses, 0); + + + if (!xlu_cfg_get_string (config, "viommu", &buf, 1)) { + e = libxl_viommu_type_from_string(buf, &b_info->arch_arm.viommu_type); + if (e) { + fprintf(stderr, + "Unknown vIOMMU type \"%s\" specified\n", buf); + exit(-ERROR_FAIL); + } + } + parse_vkb_list(config, d_config); d_config->virtios = NULL; diff --git a/xen/arch/arm/dom0less-build.c b/xen/arch/arm/dom0less-build.c index 49d1f14d659b..da1c09eea0e0 100644 --- a/xen/arch/arm/dom0less-build.c +++ b/xen/arch/arm/dom0less-build.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include bool __init is_dom0less_mode(void) { @@ -240,6 +242,65 @@ static int __init make_vpl011_uart_node(struct kernel_info *kinfo) } #endif +#ifdef CONFIG_VIRTUAL_ARM_SMMU_V3 +static int __init make_vsmmuv3_node(const struct kernel_info *kinfo) +{ + int res; + char buf[24]; + __be32 reg[GUEST_ROOT_ADDRESS_CELLS + GUEST_ROOT_SIZE_CELLS]; + __be32 *cells; + gic_interrupt_t intr; + void *fdt = kinfo->fdt; + + snprintf(buf, sizeof(buf), "iommu@%llx", GUEST_VSMMUV3_BASE); + + res = fdt_begin_node(fdt, buf); + if ( res ) + return res; + + res = fdt_property_string(fdt, "compatible", "arm,smmu-v3"); + if ( res ) + return res; + + /* Create reg property */ + cells = ®[0]; + dt_child_set_range(&cells, GUEST_ROOT_ADDRESS_CELLS, GUEST_ROOT_SIZE_CELLS, + GUEST_VSMMUV3_BASE, GUEST_VSMMUV3_SIZE); + res = fdt_property(fdt, "reg", reg, + (GUEST_ROOT_ADDRESS_CELLS + + GUEST_ROOT_SIZE_CELLS) * sizeof(*reg)); + if ( res ) + return res; + + res = fdt_property_cell(fdt, "phandle", GUEST_PHANDLE_VSMMUV3); + if ( res ) + return res; + + res = fdt_property_cell(fdt, "#iommu-cells", 1); + if ( res ) + return res; + + res = fdt_property_string(fdt, "interrupt-names", "combined"); + if ( res ) + return res; + + set_interrupt(intr, GUEST_VSMMU_SPI, 0xf, DT_IRQ_TYPE_LEVEL_HIGH); + + res = fdt_property(kinfo->fdt, "interrupts", + intr, sizeof(intr)); + if ( res ) + return res; + + res = fdt_property_cell(kinfo->fdt, "interrupt-parent", + kinfo->phandle_intc); + if ( res ) + return res; + + res = fdt_end_node(fdt); + + return res; +} +#endif /* * Scan device tree properties for passthrough specific information. * Returns < 0 on error @@ -415,7 +476,35 @@ static int __init handle_prop_pfdt(struct kernel_info *kinfo, return ( propoff != -FDT_ERR_NOTFOUND ) ? propoff : 0; } -static int __init scan_pfdt_node(struct kernel_info *kinfo, const void *pfdt, +static void modify_pfdt_node(void *pfdt, int nodeoff) +{ + int proplen, i, rc; + const fdt32_t *prop; + fdt32_t *prop_c; + + prop = fdt_getprop(pfdt, nodeoff, "iommus", &proplen); + if ( !prop ) + return; + + prop_c = xzalloc_bytes(proplen); + + for ( i = 0; i < proplen / 8; ++i ) + { + prop_c[i * 2] = cpu_to_fdt32(GUEST_PHANDLE_VSMMUV3); + prop_c[i * 2 + 1] = prop[i * 2 + 1]; + } + + rc = fdt_setprop(pfdt, nodeoff, "iommus", prop_c, proplen); + if ( rc ) + { + dprintk(XENLOG_ERR, "Can't set the iommus property in partial FDT"); + return; + } + + return; +} + +static int __init scan_pfdt_node(struct kernel_info *kinfo, void *pfdt, int nodeoff, uint32_t address_cells, uint32_t size_cells, bool scan_passthrough_prop) @@ -441,6 +530,7 @@ static int __init scan_pfdt_node(struct kernel_info *kinfo, const void *pfdt, node_next = fdt_first_subnode(pfdt, nodeoff); while ( node_next > 0 ) { + modify_pfdt_node(pfdt, node_next); rc = scan_pfdt_node(kinfo, pfdt, node_next, address_cells, size_cells, scan_passthrough_prop); if ( rc ) @@ -653,6 +743,16 @@ static int __init prepare_dtb_domU(struct domain *d, struct kernel_info *kinfo) goto err; } + +#ifdef CONFIG_VIRTUAL_ARM_SMMU_V3 + if ( is_viommu_enabled() ) + { + ret = make_vsmmuv3_node(kinfo); + if ( ret ) + goto err; + } +#endif + ret = fdt_end_node(kinfo->fdt); if ( ret < 0 ) goto err; @@ -825,6 +925,7 @@ void __init create_domUs(void) struct domain *d; struct xen_domctl_createdomain d_cfg = { .arch.gic_version = XEN_DOMCTL_CONFIG_GIC_NATIVE, + .arch.viommu_type = viommu_get_type(), .flags = XEN_DOMCTL_CDF_hvm | XEN_DOMCTL_CDF_hap, /* * The default of 1023 should be sufficient for guests because diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c index 3ba959f86633..c69215d72a8a 100644 --- a/xen/arch/arm/domain.c +++ b/xen/arch/arm/domain.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "vpci.h" @@ -694,6 +695,14 @@ int arch_sanitise_domain_config(struct xen_domctl_createdomain *config) return -EINVAL; } + if ( config->arch.viommu_type != XEN_DOMCTL_CONFIG_VIOMMU_NONE && + config->arch.viommu_type != viommu_get_type() ) + { + dprintk(XENLOG_INFO, + "vIOMMU type requested not supported by the platform or Xen\n"); + return -EINVAL; + } + return 0; } @@ -786,6 +795,9 @@ int arch_domain_create(struct domain *d, d->arch.sve_vl = config->arch.sve_vl; #endif + if ( (rc = domain_viommu_init(d, config->arch.viommu_type)) != 0 ) + goto fail; + return 0; fail: @@ -1033,6 +1045,7 @@ static int relinquish_memory(struct domain *d, struct page_list_head *list) enum { PROG_pci = 1, PROG_tee, + PROG_viommu, PROG_xen, PROG_page, PROG_mapping, @@ -1084,6 +1097,11 @@ int domain_relinquish_resources(struct domain *d) if (ret ) return ret; + PROGRESS(viommu): + ret = viommu_relinquish_resources(d); + if (ret ) + return ret; + PROGRESS(xen): ret = relinquish_memory(d, &d->xenpage_list); if ( ret ) diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c index 7b47abade196..395301fb6a16 100644 --- a/xen/arch/arm/domain_build.c +++ b/xen/arch/arm/domain_build.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -643,9 +644,12 @@ static int __init write_properties(struct domain *d, struct kernel_info *kinfo, continue; } - if ( iommu_node ) + /* + * Expose IOMMU specific properties to hwdom when vIOMMU is + * enabled. + */ + if ( iommu_node && !is_viommu_enabled() ) { - /* Don't expose IOMMU specific properties to hwdom */ if ( dt_property_name_is_equal(prop, "iommus") ) continue; @@ -1803,6 +1807,95 @@ int __init make_chosen_node(const struct kernel_info *kinfo) return res; } +#ifdef CONFIG_VIRTUAL_IOMMU +static int make_hwdom_viommu_node(const struct kernel_info *kinfo) +{ + uint32_t len; + int res; + char buf[24]; + void *fdt = kinfo->fdt; + const void *prop = NULL; + const struct dt_device_node *iommu = NULL; + struct host_iommu *iommu_data; + gic_interrupt_t intr; + + if ( list_empty(&host_iommu_list) ) + return 0; + + list_for_each_entry( iommu_data, &host_iommu_list, entry ) + { + if ( iommu_data->hwdom_node_created ) + return 0; + + iommu = iommu_data->dt_node; + + snprintf(buf, sizeof(buf), "iommu@%"PRIx64, iommu_data->addr); + + res = fdt_begin_node(fdt, buf); + if ( res ) + return res; + + prop = dt_get_property(iommu, "compatible", &len); + if ( !prop ) + { + res = -FDT_ERR_XEN(ENOENT); + return res; + } + + res = fdt_property(fdt, "compatible", prop, len); + if ( res ) + return res; + + if ( iommu->phandle ) + { + res = fdt_property_cell(fdt, "phandle", iommu->phandle); + if ( res ) + return res; + } + + /* Use the same reg regions as the IOMMU node in host DTB. */ + prop = dt_get_property(iommu, "reg", &len); + if ( !prop ) + { + printk(XENLOG_ERR "vIOMMU: Can't find IOMMU reg property.\n"); + res = -FDT_ERR_XEN(ENOENT); + return res; + } + + res = fdt_property(fdt, "reg", prop, len); + if ( res ) + return res; + + prop = dt_get_property(iommu, "#iommu-cells", &len); + if ( !prop ) + { + res = -FDT_ERR_XEN(ENOENT); + return res; + } + + res = fdt_property(fdt, "#iommu-cells", prop, len); + if ( res ) + return res; + + res = fdt_property_string(fdt, "interrupt-names", "combined"); + if ( res ) + return res; + + set_interrupt(intr, iommu_data->irq, 0xf, DT_IRQ_TYPE_LEVEL_HIGH); + + res = fdt_property_interrupts(kinfo, &intr, 1); + if ( res ) + return res; + + iommu_data->hwdom_node_created = true; + + fdt_end_node(fdt); + } + + return res; +} +#endif + static int __init handle_node(struct domain *d, struct kernel_info *kinfo, struct dt_device_node *node, p2m_type_t p2mt) @@ -1871,6 +1964,11 @@ static int __init handle_node(struct domain *d, struct kernel_info *kinfo, if ( dt_match_node(timer_matches, node) ) return make_timer_node(kinfo); +#ifdef CONFIG_VIRTUAL_IOMMU + if ( device_get_class(node) == DEVICE_IOMMU && is_viommu_enabled() ) + return make_hwdom_viommu_node(kinfo); +#endif + /* Skip nodes used by Xen */ if ( dt_device_used_by(node) == DOMID_XEN ) { @@ -2365,6 +2463,7 @@ void __init create_dom0(void) printk(XENLOG_WARNING "Maximum number of vGIC IRQs exceeded.\n"); dom0_cfg.arch.tee_type = tee_get_type(); dom0_cfg.max_vcpus = dom0_max_vcpus(); + dom0_cfg.arch.viommu_type = viommu_get_type(); if ( iommu_enabled ) dom0_cfg.flags |= XEN_DOMCTL_CDF_iommu; diff --git a/xen/arch/arm/include/asm/domain.h b/xen/arch/arm/include/asm/domain.h index f1d72c6e48df..5897d1f64e5a 100644 --- a/xen/arch/arm/include/asm/domain.h +++ b/xen/arch/arm/include/asm/domain.h @@ -119,6 +119,10 @@ struct arch_domain void *tee; #endif +#ifdef CONFIG_VIRTUAL_IOMMU + struct list_head viommu_list; /* List of virtual IOMMUs */ +#endif + } __cacheline_aligned; struct arch_vcpu diff --git a/xen/arch/arm/include/asm/viommu.h b/xen/arch/arm/include/asm/viommu.h new file mode 100644 index 000000000000..e6018f435b00 --- /dev/null +++ b/xen/arch/arm/include/asm/viommu.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef __ARCH_ARM_VIOMMU_H__ +#define __ARCH_ARM_VIOMMU_H__ + +#ifdef CONFIG_VIRTUAL_IOMMU + +#include +#include +#include +#include + +extern struct list_head host_iommu_list; +extern bool viommu_enabled; + +/* data structure for each hardware IOMMU */ +struct host_iommu { + struct list_head entry; + const struct dt_device_node *dt_node; + paddr_t addr; + paddr_t size; + uint32_t irq; + bool hwdom_node_created; +}; + +struct viommu_ops { + /* + * Called during domain construction if toolstack requests to enable + * vIOMMU support. + */ + int (*domain_init)(struct domain *d); + + /* + * Called during domain destruction to free resources used by vIOMMU. + */ + int (*relinquish_resources)(struct domain *d); +}; + +struct viommu_desc { + /* vIOMMU domains init/free operations described above. */ + const struct viommu_ops *ops; + + /* + * ID of vIOMMU. Corresponds to xen_arch_domainconfig.viommu_type. + * Should be one of XEN_DOMCTL_CONFIG_VIOMMU_xxx + */ + uint16_t viommu_type; +}; + +int domain_viommu_init(struct domain *d, uint16_t viommu_type); +int viommu_relinquish_resources(struct domain *d); +uint16_t viommu_get_type(void); +void add_to_host_iommu_list(paddr_t addr, paddr_t size, + const struct dt_device_node *node); + +static always_inline bool is_viommu_enabled(void) +{ + return viommu_enabled; +} + +#else + +static inline uint8_t viommu_get_type(void) +{ + return XEN_DOMCTL_CONFIG_VIOMMU_NONE; +} + +static inline int domain_viommu_init(struct domain *d, uint16_t viommu_type) +{ + if ( likely(viommu_type == XEN_DOMCTL_CONFIG_VIOMMU_NONE) ) + return 0; + + return -ENODEV; +} + +static inline int viommu_relinquish_resources(struct domain *d) +{ + return 0; +} + +static inline void add_to_host_iommu_list(paddr_t addr, paddr_t size, + const struct dt_device_node *node) +{ + return; +} + +static always_inline bool is_viommu_enabled(void) +{ + return false; +} + +#endif /* CONFIG_VIRTUAL_IOMMU */ + +#endif /* __ARCH_ARM_VIOMMU_H__ */ + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/drivers/passthrough/Kconfig b/xen/drivers/passthrough/Kconfig index 78edd805365e..13c46998ae70 100644 --- a/xen/drivers/passthrough/Kconfig +++ b/xen/drivers/passthrough/Kconfig @@ -35,6 +35,20 @@ config IPMMU_VMSA (H3 ES3.0, M3-W+, etc) or Gen4 SoCs which IPMMU hardware supports stage 2 translation table format and is able to use CPU's P2M table as is. +config VIRTUAL_IOMMU + bool "Virtual IOMMU Support (UNSUPPORTED)" if UNSUPPORTED + default n + help + Support virtual IOMMU infrastructure to implement vIOMMU. + +config VIRTUAL_ARM_SMMU_V3 + bool "ARM Ltd. Virtual SMMUv3 Support (UNSUPPORTED)" if UNSUPPORTED + depends on ARM_SMMU_V3 && VIRTUAL_IOMMU + help + Support for implementations of the virtual ARM System MMU architecture + version 3. Virtual SMMUv3 is unsupported feature and should not be used + in production. + endif config AMD_IOMMU diff --git a/xen/drivers/passthrough/arm/Makefile b/xen/drivers/passthrough/arm/Makefile index c5fb3b58a510..e758a9d6aa25 100644 --- a/xen/drivers/passthrough/arm/Makefile +++ b/xen/drivers/passthrough/arm/Makefile @@ -2,3 +2,5 @@ obj-y += iommu.o iommu_helpers.o iommu_fwspec.o obj-$(CONFIG_ARM_SMMU) += smmu.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_ARM_SMMU_V3) += smmu-v3.o +obj-$(CONFIG_VIRTUAL_IOMMU) += viommu.o +obj-$(CONFIG_VIRTUAL_ARM_SMMU_V3) += vsmmu-v3.o diff --git a/xen/drivers/passthrough/arm/smmu-v3.c b/xen/drivers/passthrough/arm/smmu-v3.c index cee572402203..1c34823e1a83 100644 --- a/xen/drivers/passthrough/arm/smmu-v3.c +++ b/xen/drivers/passthrough/arm/smmu-v3.c @@ -93,6 +93,7 @@ #include #include "smmu-v3.h" +#include "vsmmu-v3.h" #define ARM_SMMU_VTCR_SH_IS 3 #define ARM_SMMU_VTCR_RGN_WBWA 1 @@ -664,8 +665,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, * 3. Update Config, sync */ u64 val = le64_to_cpu(dst[0]); - bool ste_live = false; + bool s1_live = false, s2_live = false, ste_live = false; + bool abort, translate = false; struct arm_smmu_device *smmu = NULL; + struct arm_smmu_s1_cfg *s1_cfg = NULL; struct arm_smmu_s2_cfg *s2_cfg = NULL; struct arm_smmu_domain *smmu_domain = NULL; struct arm_smmu_cmdq_ent prefetch_cmd = { @@ -680,30 +683,54 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, smmu = master->smmu; } - if (smmu_domain) - s2_cfg = &smmu_domain->s2_cfg; + if (smmu_domain) { + switch (smmu_domain->stage) { + case ARM_SMMU_DOMAIN_NESTED: + s1_cfg = &smmu_domain->s1_cfg; + fallthrough; + case ARM_SMMU_DOMAIN_S2: + s2_cfg = &smmu_domain->s2_cfg; + break; + default: + break; + } + translate = !!s1_cfg || !!s2_cfg; + } if (val & STRTAB_STE_0_V) { switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { case STRTAB_STE_0_CFG_BYPASS: break; + case STRTAB_STE_0_CFG_S1_TRANS: + s1_live = true; + break; case STRTAB_STE_0_CFG_S2_TRANS: - ste_live = true; + s2_live = true; + break; + case STRTAB_STE_0_CFG_NESTED: + s1_live = true; + s2_live = true; break; case STRTAB_STE_0_CFG_ABORT: - BUG_ON(!disable_bypass); break; default: BUG(); /* STE corruption */ } } + ste_live = s1_live || s2_live; + /* Nuke the existing STE_0 value, as we're going to rewrite it */ val = STRTAB_STE_0_V; /* Bypass/fault */ - if (!smmu_domain || !(s2_cfg)) { - if (!smmu_domain && disable_bypass) + if (!smmu_domain) + abort = disable_bypass; + else + abort = smmu_domain->abort; + + if (abort || !translate) { + if (abort) val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); else val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); @@ -721,20 +748,56 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, return; } + if (ste_live) { + /* First invalidate the live STE */ + dst[0] = cpu_to_le64(STRTAB_STE_0_CFG_ABORT); + arm_smmu_sync_ste_for_sid(smmu, sid); + } + + if (s1_cfg) { + BUG_ON(s1_live); + dst[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | + FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | + FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1)); + + if (smmu->features & ARM_SMMU_FEAT_STALLS && + !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) + dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); + + val |= (s1_cfg->s1ctxptr & STRTAB_STE_0_S1CTXPTR_MASK) | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | + FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) | + FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt); + } + if (s2_cfg) { - BUG_ON(ste_live); - dst[2] = cpu_to_le64( + u64 vttbr = s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK; + u64 strtab = FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | #ifdef __BIG_ENDIAN STRTAB_STE_2_S2ENDI | #endif STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | - STRTAB_STE_2_S2R); + STRTAB_STE_2_S2R; + + if (s2_live) { + u64 s2ttb = le64_to_cpu(dst[3]) & STRTAB_STE_3_S2TTB_MASK; + BUG_ON(s2ttb != vttbr); + } + + BUG_ON(ste_live); + dst[2] = cpu_to_le64(strtab); - dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); + dst[3] = cpu_to_le64(vttbr); val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); + } else { + dst[2] = 0; + dst[3] = 0; } if (master->ats_enabled) @@ -788,10 +851,71 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return 0; } +static struct arm_smmu_master * +arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) +{ + struct rb_node *node; + struct arm_smmu_stream *stream; + + node = smmu->streams.rb_node; + while (node) { + stream = rb_entry(node, struct arm_smmu_stream, node); + if (stream->id < sid) + node = node->rb_right; + else if (stream->id > sid) + node = node->rb_left; + else + return stream->master; + } + + return NULL; +} + +static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) +{ + int ret; + struct arm_smmu_master *master; + u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]); + + switch (FIELD_GET(EVTQ_0_ID, evt[0])) { + case EVT_ID_TRANSLATION_FAULT: + break; + case EVT_ID_ADDR_SIZE_FAULT: + break; + case EVT_ID_ACCESS_FAULT: + break; + case EVT_ID_PERMISSION_FAULT: + break; + default: + return -EOPNOTSUPP; + } + + /* Stage-2 event */ + if (evt[1] & EVTQ_1_S2) + return -EFAULT; + + mutex_lock(&smmu->streams_mutex); + master = arm_smmu_find_master(smmu, sid); + if (!master) { + ret = -EINVAL; + goto out_unlock; + } + + ret = arm_vsmmu_handle_evt(master->domain->d, smmu->dev, evt); + if (ret) { + ret = -EINVAL; + goto out_unlock; + } + +out_unlock: + mutex_unlock(&smmu->streams_mutex); + return ret; +} + /* IRQ and event handlers */ static void arm_smmu_evtq_tasklet(void *dev) { - int i; + int i, ret; struct arm_smmu_device *smmu = dev; struct arm_smmu_queue *q = &smmu->evtq.q; struct arm_smmu_ll_queue *llq = &q->llq; @@ -801,6 +925,10 @@ static void arm_smmu_evtq_tasklet(void *dev) while (!queue_remove_raw(q, evt)) { u8 id = FIELD_GET(EVTQ_0_ID, evt[0]); + ret = arm_smmu_handle_evt(smmu, evt); + if (!ret) + continue; + dev_info(smmu->dev, "event 0x%02x received:\n", id); for (i = 0; i < ARRAY_SIZE(evt); ++i) dev_info(smmu->dev, "\t0x%016llx\n", @@ -1021,8 +1149,8 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, if (!master->ats_enabled) return 0; - for (i = 0; i < master->num_sids; i++) { - cmd->atc.sid = master->sids[i]; + for (i = 0; i < master->num_streams; i++) { + cmd->atc.sid = master->streams[i].id; arm_smmu_cmdq_issue_cmd(master->smmu, cmd); } @@ -1212,6 +1340,15 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain, { int ret; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED && + (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1) || + !(smmu->features & ARM_SMMU_FEAT_TRANS_S2))) { + dev_info(smmu_domain->smmu->dev, + "does not implement two stages\n"); + return -EINVAL; + } /* Restrict the stage to what we can actually support */ smmu_domain->stage = ARM_SMMU_DOMAIN_S2; @@ -1250,13 +1387,13 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) int i, j; struct arm_smmu_device *smmu = master->smmu; - for (i = 0; i < master->num_sids; ++i) { - u32 sid = master->sids[i]; + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); /* Bridged PCI devices may end up with duplicated IDs */ for (j = 0; j < i; j++) - if (master->sids[j] == sid) + if (master->streams[j].id == sid) break; if (j < i) continue; @@ -1465,12 +1602,86 @@ static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) return sid < limit; } + +static int arm_smmu_insert_master(struct arm_smmu_device *smmu, + struct arm_smmu_master *master) +{ + int i; + int ret = 0; + struct arm_smmu_stream *new_stream, *cur_stream; + struct rb_node **new_node, *parent_node = NULL; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); + + master->streams = _xzalloc_array(sizeof(*master->streams), sizeof(void *), + fwspec->num_ids); + if (!master->streams) + return -ENOMEM; + master->num_streams = fwspec->num_ids; + + mutex_lock(&smmu->streams_mutex); + for (i = 0; i < fwspec->num_ids; i++) { + u32 sid = fwspec->ids[i]; + + new_stream = &master->streams[i]; + new_stream->id = sid; + new_stream->master = master; + + /* + * Check the SIDs are in range of the SMMU and our stream table + */ + if (!arm_smmu_sid_in_range(smmu, sid)) { + ret = -ERANGE; + break; + } + + /* Ensure l2 strtab is initialised */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + ret = arm_smmu_init_l2_strtab(smmu, sid); + if (ret) + break; + } + + /* Insert into SID tree */ + new_node = &(smmu->streams.rb_node); + while (*new_node) { + cur_stream = rb_entry(*new_node, struct arm_smmu_stream, + node); + parent_node = *new_node; + if (cur_stream->id > new_stream->id) { + new_node = &((*new_node)->rb_left); + } else if (cur_stream->id < new_stream->id) { + new_node = &((*new_node)->rb_right); + } else { + dev_warn(master->dev, + "stream %u already in tree\n", + cur_stream->id); + ret = -EINVAL; + break; + } + } + if (ret) + break; + + rb_link_node(&new_stream->node, parent_node, new_node); + rb_insert_color(&new_stream->node, &smmu->streams); + } + + if (ret) { + for (i--; i >= 0; i--) + rb_erase(&master->streams[i].node, &smmu->streams); + xfree(master->streams); + } + mutex_unlock(&smmu->streams_mutex); + + return ret; +} + /* Forward declaration */ static struct arm_smmu_device *arm_smmu_get_by_dev(const struct device *dev); static int arm_smmu_add_device(u8 devfn, struct device *dev) { - int i, ret; + int ret; struct arm_smmu_device *smmu; struct arm_smmu_master *master; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); @@ -1488,26 +1699,11 @@ static int arm_smmu_add_device(u8 devfn, struct device *dev) master->dev = dev; master->smmu = smmu; - master->sids = fwspec->ids; - master->num_sids = fwspec->num_ids; dev_iommu_priv_set(dev, master); - /* Check the SIDs are in range of the SMMU and our stream table */ - for (i = 0; i < master->num_sids; i++) { - u32 sid = master->sids[i]; - - if (!arm_smmu_sid_in_range(smmu, sid)) { - ret = -ERANGE; - goto err_free_master; - } - - /* Ensure l2 strtab is initialised */ - if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { - ret = arm_smmu_init_l2_strtab(smmu, sid); - if (ret) - goto err_free_master; - } - } + ret = arm_smmu_insert_master(smmu, master); + if (ret) + goto err_free_master; /* * Note that PASID must be enabled before, and disabled after ATS: @@ -1728,6 +1924,9 @@ static int __init arm_smmu_init_structures(struct arm_smmu_device *smmu) { int ret; + mutex_init(&smmu->streams_mutex); + smmu->streams = RB_ROOT; + ret = arm_smmu_init_queues(smmu); if (ret) return ret; @@ -2202,11 +2401,14 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) break; } + if (reg & IDR0_S1P) + smmu->features |= ARM_SMMU_FEAT_TRANS_S1; + if (reg & IDR0_S2P) smmu->features |= ARM_SMMU_FEAT_TRANS_S2; - if (!(reg & IDR0_S2P)) { - dev_err(smmu->dev, "no stage-2 translation support!\n"); + if (!(reg & (IDR0_S1P | IDR0_S2P))) { + dev_err(smmu->dev, "no translation support!\n"); return -ENXIO; } @@ -2505,6 +2707,9 @@ static int __init arm_smmu_device_probe(struct platform_device *pdev) list_add(&smmu->devices, &arm_smmu_devices); spin_unlock(&arm_smmu_devices_lock); + /* Add to host IOMMU list to initialize vIOMMU for dom0 */ + add_to_host_iommu_list(ioaddr, iosize, dev_to_dt(pdev)); + return 0; @@ -2584,14 +2789,47 @@ static struct arm_smmu_device *arm_smmu_get_by_dev(const struct device *dev) return NULL; } +static struct iommu_domain *arm_smmu_get_domain_by_sid(struct domain *d, + u32 sid) +{ + int i; + unsigned long flags; + struct iommu_domain *io_domain; + struct arm_smmu_domain *smmu_domain; + struct arm_smmu_master *master; + struct arm_smmu_xen_domain *xen_domain = dom_iommu(d)->arch.priv; + + /* + * Loop through the &xen_domain->contexts to locate a context + * assigned to this SMMU + */ + list_for_each_entry(io_domain, &xen_domain->contexts, list) { + smmu_domain = to_smmu_domain(io_domain); + + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + for (i = 0; i < master->num_streams; i++) { + if (sid != master->streams[i].id) + continue; + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + return io_domain; + } + } + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + } + return NULL; +} + static struct iommu_domain *arm_smmu_get_domain(struct domain *d, struct device *dev) { + unsigned long flags; struct iommu_domain *io_domain; struct arm_smmu_domain *smmu_domain; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_xen_domain *xen_domain = dom_iommu(d)->arch.priv; struct arm_smmu_device *smmu = arm_smmu_get_by_dev(fwspec->iommu_dev); + struct arm_smmu_master *master; if (!smmu) return NULL; @@ -2602,8 +2840,15 @@ static struct iommu_domain *arm_smmu_get_domain(struct domain *d, */ list_for_each_entry(io_domain, &xen_domain->contexts, list) { smmu_domain = to_smmu_domain(io_domain); - if (smmu_domain->smmu == smmu) - return io_domain; + + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + if (master->dev == dev) { + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + return io_domain; + } + } + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } return NULL; } @@ -2734,6 +2979,53 @@ static void arm_smmu_iommu_xen_domain_teardown(struct domain *d) xfree(xen_domain); } +static int arm_smmu_attach_guest_config(struct domain *d, u32 sid, + struct iommu_guest_config *cfg) +{ + int ret = -EINVAL; + unsigned long flags; + struct arm_smmu_master *master; + struct arm_smmu_domain *smmu_domain; + struct arm_smmu_xen_domain *xen_domain = dom_iommu(d)->arch.priv; + struct iommu_domain *io_domain = arm_smmu_get_domain_by_sid(d, sid); + + if (!io_domain) + return -ENODEV; + + smmu_domain = to_smmu_domain(io_domain); + + spin_lock(&xen_domain->lock); + + switch (cfg->config) { + case ARM_SMMU_DOMAIN_ABORT: + smmu_domain->abort = true; + break; + case ARM_SMMU_DOMAIN_BYPASS: + smmu_domain->abort = false; + break; + case ARM_SMMU_DOMAIN_NESTED: + /* Enable Nested stage translation. */ + smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; + smmu_domain->s1_cfg.s1ctxptr = cfg->s1ctxptr; + smmu_domain->s1_cfg.s1fmt = cfg->s1fmt; + smmu_domain->s1_cfg.s1cdmax = cfg->s1cdmax; + smmu_domain->abort = false; + break; + default: + goto out; + } + + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) + arm_smmu_install_ste_for_dev(master); + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + + ret = 0; +out: + spin_unlock(&xen_domain->lock); + return ret; +} + static const struct iommu_ops arm_smmu_iommu_ops = { .page_sizes = PAGE_SIZE_4K, .init = arm_smmu_iommu_xen_domain_init, @@ -2746,6 +3038,7 @@ static const struct iommu_ops arm_smmu_iommu_ops = { .unmap_page = arm_iommu_unmap_page, .dt_xlate = arm_smmu_dt_xlate, .add_device = arm_smmu_add_device, + .attach_guest_config = arm_smmu_attach_guest_config }; static __init int arm_smmu_dt_init(struct dt_device_node *dev, @@ -2774,6 +3067,9 @@ static __init int arm_smmu_dt_init(struct dt_device_node *dev, platform_features &= smmu->features; + /* Set vIOMMU type to SMMUv3 */ + vsmmuv3_set_type(); + return 0; } diff --git a/xen/drivers/passthrough/arm/smmu-v3.h b/xen/drivers/passthrough/arm/smmu-v3.h index f09048812ca8..820271a41360 100644 --- a/xen/drivers/passthrough/arm/smmu-v3.h +++ b/xen/drivers/passthrough/arm/smmu-v3.h @@ -60,6 +60,12 @@ #define IDR5_VAX GENMASK(11, 10) #define IDR5_VAX_52_BIT 1 +#define ARM_SMMU_IIDR 0x18 +#define IIDR_PRODUCTID GENMASK(31, 20) +#define IIDR_VARIANT GENMASK(19, 16) +#define IIDR_REVISION GENMASK(15, 12) +#define IIDR_IMPLEMENTER GENMASK(11, 0) + #define ARM_SMMU_CR0 0x20 #define CR0_ATSCHK (1 << 4) #define CR0_CMDQEN (1 << 3) @@ -197,6 +203,7 @@ #define STRTAB_STE_0_CFG_BYPASS 4 #define STRTAB_STE_0_CFG_S1_TRANS 5 #define STRTAB_STE_0_CFG_S2_TRANS 6 +#define STRTAB_STE_0_CFG_NESTED 7 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) #define STRTAB_STE_0_S1FMT_LINEAR 0 @@ -347,6 +354,26 @@ #define EVTQ_0_ID GENMASK_ULL(7, 0) +#define EVT_ID_BAD_STREAMID 0x02 +#define EVT_ID_BAD_STE 0x04 +#define EVT_ID_TRANSLATION_FAULT 0x10 +#define EVT_ID_ADDR_SIZE_FAULT 0x11 +#define EVT_ID_ACCESS_FAULT 0x12 +#define EVT_ID_PERMISSION_FAULT 0x13 + +#define EVTQ_0_SSV (1UL << 11) +#define EVTQ_0_SSID GENMASK_ULL(31, 12) +#define EVTQ_0_SID GENMASK_ULL(63, 32) +#define EVTQ_1_STAG GENMASK_ULL(15, 0) +#define EVTQ_1_STALL (1UL << 31) +#define EVTQ_1_PnU (1UL << 33) +#define EVTQ_1_InD (1UL << 34) +#define EVTQ_1_RnW (1UL << 35) +#define EVTQ_1_S2 (1UL << 39) +#define EVTQ_1_CLASS GENMASK_ULL(41, 40) +#define EVTQ_1_TT_READ (1UL << 44) +#define EVTQ_2_ADDR GENMASK_ULL(63, 0) +#define EVTQ_3_IPA GENMASK_ULL(51, 12) /* PRI queue */ #define PRIQ_ENT_SZ_SHIFT 4 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3) @@ -391,6 +418,7 @@ enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S2, ARM_SMMU_DOMAIN_NESTED, ARM_SMMU_DOMAIN_BYPASS, + ARM_SMMU_DOMAIN_ABORT, }; /* Xen specific code. */ @@ -546,6 +574,12 @@ struct arm_smmu_strtab_l1_desc { dma_addr_t l2ptr_dma; }; +struct arm_smmu_s1_cfg { + u64 s1ctxptr; + u8 s1fmt; + u8 s1cdmax; +}; + struct arm_smmu_s2_cfg { u16 vmid; u64 vttbr; @@ -636,6 +670,15 @@ struct arm_smmu_device { struct tasklet evtq_irq_tasklet; struct tasklet priq_irq_tasklet; struct tasklet combined_irq_tasklet; + + struct rb_root streams; + struct mutex streams_mutex; +}; + +struct arm_smmu_stream { + u32 id; + struct arm_smmu_master *master; + struct rb_node node; }; /* SMMU private data for each master */ @@ -644,8 +687,8 @@ struct arm_smmu_master { struct device *dev; struct arm_smmu_domain *domain; struct list_head domain_head; - u32 *sids; - unsigned int num_sids; + struct arm_smmu_stream *streams; + unsigned int num_streams; bool ats_enabled; }; @@ -657,7 +700,9 @@ struct arm_smmu_domain { atomic_t nr_ats_masters; enum arm_smmu_domain_stage stage; + struct arm_smmu_s1_cfg s1_cfg; struct arm_smmu_s2_cfg s2_cfg; + bool abort; /* Xen domain associated with this SMMU domain */ struct domain *d; diff --git a/xen/drivers/passthrough/arm/viommu.c b/xen/drivers/passthrough/arm/viommu.c new file mode 100644 index 000000000000..a1d6a04ba9be --- /dev/null +++ b/xen/drivers/passthrough/arm/viommu.c @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ + +#include +#include +#include +#include +#include + +#include + +/* List of all host IOMMUs */ +LIST_HEAD(host_iommu_list); + +const struct viommu_desc __read_mostly *cur_viommu; + +/* Common function for adding to host_iommu_list */ +void add_to_host_iommu_list(paddr_t addr, paddr_t size, + const struct dt_device_node *node) +{ + struct host_iommu *iommu_data; + + iommu_data = xzalloc(struct host_iommu); + if ( !iommu_data ) + panic("vIOMMU: Cannot allocate memory for host IOMMU data\n"); + + iommu_data->addr = addr; + iommu_data->size = size; + iommu_data->dt_node = node; + iommu_data->irq = platform_get_irq(node, 0); + if ( iommu_data->irq < 0 ) + { + gdprintk(XENLOG_ERR, + "vIOMMU: Cannot find a valid IOMMU irq\n"); + return; + } + + printk("vIOMMU: Found IOMMU @0x%"PRIx64"\n", addr); + + list_add_tail(&iommu_data->entry, &host_iommu_list); +} + +/* By default viommu is disabled. */ +bool __read_mostly viommu_enabled; +boolean_param("viommu", viommu_enabled); + +int domain_viommu_init(struct domain *d, uint16_t viommu_type) +{ + /* Enable viommu when it has been enabled explicitly (viommu=on). */ + if ( !viommu_enabled ) + return 0; + + if ( viommu_type == XEN_DOMCTL_CONFIG_VIOMMU_NONE ) + return 0; + + if ( !cur_viommu ) + return -ENODEV; + + if ( cur_viommu->viommu_type != viommu_type ) + return -EINVAL; + + return cur_viommu->ops->domain_init(d); +} + +int viommu_relinquish_resources(struct domain *d) +{ + if ( !cur_viommu ) + return 0; + + return cur_viommu->ops->relinquish_resources(d); +} + +uint16_t viommu_get_type(void) +{ + if ( !cur_viommu ) + return XEN_DOMCTL_CONFIG_VIOMMU_NONE; + + return cur_viommu->viommu_type; +} + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/drivers/passthrough/arm/vsmmu-v3.c b/xen/drivers/passthrough/arm/vsmmu-v3.c new file mode 100644 index 000000000000..102d59a67bcd --- /dev/null +++ b/xen/drivers/passthrough/arm/vsmmu-v3.c @@ -0,0 +1,892 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ + +#include +#include +#include +#include +#include +#include +#include + +#include "smmu-v3.h" + +/* Register Definition */ +#define ARM_SMMU_IDR2 0x8 +#define ARM_SMMU_IDR3 0xc +#define ARM_SMMU_IDR4 0x10 +#define ARM_SMMU_IIDR_VAL 0x12 +#define IDR0_TERM_MODEL (1 << 26) +#define IDR3_RIL (1 << 10) +#define CR0_RESERVED 0xFFFFFC20 +#define SMMU_IDR1_SIDSIZE 16 +#define SMMU_CMDQS 19 +#define SMMU_EVTQS 19 +#define DWORDS_BYTES 8 + +/* Struct to hold the vIOMMU ops and vIOMMU type */ +extern const struct viommu_desc __read_mostly *cur_viommu; + +/* SMMUv3 command definitions */ +#define CMDQ_OP_PREFETCH_CFG 0x1 +#define CMDQ_OP_CFGI_STE 0x3 +#define CMDQ_OP_CFGI_ALL 0x4 +#define CMDQ_OP_CFGI_CD 0x5 +#define CMDQ_OP_CFGI_CD_ALL 0x6 +#define CMDQ_OP_TLBI_NH_ASID 0x11 +#define CMDQ_OP_TLBI_NH_VA 0x12 +#define CMDQ_OP_TLBI_NSNH_ALL 0x30 +#define CMDQ_OP_CMD_SYNC 0x46 + +/* Queue Handling */ +#define Q_BASE(q) ((q)->q_base & Q_BASE_ADDR_MASK) +#define Q_CONS_ENT(q) (Q_BASE(q) + Q_IDX(q, (q)->cons) * (q)->ent_size) +#define Q_PROD_ENT(q) (Q_BASE(q) + Q_IDX(q, (q)->prod) * (q)->ent_size) + +/* Helper Macros */ +#define smmu_get_cmdq_enabled(x) FIELD_GET(CR0_CMDQEN, x) +#define smmu_get_evtq_enabled(x) FIELD_GET(CR0_EVTQEN, x) +#define smmu_cmd_get_command(x) FIELD_GET(CMDQ_0_OP, x) +#define smmu_cmd_get_sid(x) FIELD_GET(CMDQ_PREFETCH_0_SID, x) +#define smmu_get_ste_s1cdmax(x) FIELD_GET(STRTAB_STE_0_S1CDMAX, x) +#define smmu_get_ste_s1fmt(x) FIELD_GET(STRTAB_STE_0_S1FMT, x) +#define smmu_get_ste_s1stalld(x) FIELD_GET(STRTAB_STE_1_S1STALLD, x) +#define smmu_get_ste_s1ctxptr(x) FIELD_PREP(STRTAB_STE_0_S1CTXPTR_MASK, \ + FIELD_GET(STRTAB_STE_0_S1CTXPTR_MASK, x)) + +/* event queue entry */ +struct arm_smmu_evtq_ent { + /* Common fields */ + uint8_t opcode; + uint32_t sid; + + /* Event-specific fields */ + union { + struct { + uint32_t ssid; + bool ssv; + } c_bad_ste_streamid; + + struct { + bool stall; + uint16_t stag; + uint32_t ssid; + bool ssv; + bool s2; + uint64_t addr; + bool rnw; + bool pnu; + bool ind; + uint8_t class; + uint64_t addr2; + } f_translation; + }; +}; + +/* stage-1 translation configuration */ +struct arm_vsmmu_s1_trans_cfg { + paddr_t s1ctxptr; + uint8_t s1fmt; + uint8_t s1cdmax; + bool bypassed; /* translation is bypassed */ + bool aborted; /* translation is aborted */ +}; + +/* virtual smmu queue */ +struct arm_vsmmu_queue { + uint64_t q_base; /* base register */ + uint32_t prod; + uint32_t cons; + uint8_t ent_size; + uint8_t max_n_shift; +}; + +struct virt_smmu { + struct domain *d; + struct list_head viommu_list; + paddr_t addr; + uint8_t sid_split; + uint32_t features; + uint32_t cr[3]; + uint32_t cr0ack; + uint32_t gerror; + uint32_t gerrorn; + uint32_t strtab_base_cfg; + uint64_t strtab_base; + uint32_t irq_ctrl; + uint32_t virq; + uint64_t gerror_irq_cfg0; + uint64_t evtq_irq_cfg0; + struct arm_vsmmu_queue evtq, cmdq; + spinlock_t cmd_queue_lock; +}; + +/* Queue manipulation functions */ +static bool queue_full(struct arm_vsmmu_queue *q) +{ + return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && + Q_WRP(q, q->prod) != Q_WRP(q, q->cons); +} + +static bool queue_empty(struct arm_vsmmu_queue *q) +{ + return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && + Q_WRP(q, q->prod) == Q_WRP(q, q->cons); +} + +static void queue_inc_cons(struct arm_vsmmu_queue *q) +{ + uint32_t cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; + q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); +} + +static void queue_inc_prod(struct arm_vsmmu_queue *q) +{ + u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1; + q->prod = Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); +} + +static void dump_smmu_command(uint64_t *command) +{ + gdprintk(XENLOG_ERR, "cmd 0x%02llx: %016lx %016lx\n", + smmu_cmd_get_command(command[0]), command[0], command[1]); +} + +static void arm_vsmmu_inject_irq(struct virt_smmu *smmu, bool is_gerror, + uint32_t gerror_err) +{ + uint32_t new_gerrors, pending; + + if ( is_gerror ) + { + /* trigger global error irq to guest */ + pending = smmu->gerror ^ smmu->gerrorn; + new_gerrors = ~pending & gerror_err; + + /* only toggle non pending errors */ + if (!new_gerrors) + return; + + smmu->gerror ^= new_gerrors; + } + + vgic_inject_irq(smmu->d, NULL, smmu->virq, true); +} + +static int arm_vsmmu_write_evtq(struct virt_smmu *smmu, uint64_t *evt) +{ + struct arm_vsmmu_queue *q = &smmu->evtq; + struct domain *d = smmu->d; + paddr_t addr; + int ret; + + if ( !smmu_get_evtq_enabled(smmu->cr[0]) ) + return -EINVAL; + + if ( queue_full(q) ) + return -EINVAL; + + addr = Q_PROD_ENT(q); + ret = access_guest_memory_by_gpa(d, addr, evt, + sizeof(*evt) * EVTQ_ENT_DWORDS, true); + if ( ret ) + return ret; + + queue_inc_prod(q); + + /* trigger eventq irq to guest */ + if ( !queue_empty(q) ) + arm_vsmmu_inject_irq(smmu, false, 0); + + return 0; +} + +void arm_vsmmu_send_event(struct virt_smmu *smmu, + struct arm_smmu_evtq_ent *ent) +{ + uint64_t evt[EVTQ_ENT_DWORDS]; + int ret; + + memset(evt, 0, 1 << EVTQ_ENT_SZ_SHIFT); + + if ( !smmu_get_evtq_enabled(smmu->cr[0]) ) + return; + + evt[0] |= FIELD_PREP(EVTQ_0_ID, ent->opcode); + evt[0] |= FIELD_PREP(EVTQ_0_SID, ent->sid); + + switch (ent->opcode) + { + case EVT_ID_BAD_STREAMID: + case EVT_ID_BAD_STE: + evt[0] |= FIELD_PREP(EVTQ_0_SSID, ent->c_bad_ste_streamid.ssid); + evt[0] |= FIELD_PREP(EVTQ_0_SSV, ent->c_bad_ste_streamid.ssv); + break; + case EVT_ID_TRANSLATION_FAULT: + case EVT_ID_ADDR_SIZE_FAULT: + case EVT_ID_ACCESS_FAULT: + case EVT_ID_PERMISSION_FAULT: + break; + default: + gdprintk(XENLOG_WARNING, "vSMMUv3: event opcode is bad\n"); + break; + } + + ret = arm_vsmmu_write_evtq(smmu, evt); + if ( ret ) + arm_vsmmu_inject_irq(smmu, true, GERROR_EVTQ_ABT_ERR); + + return; +} + +static struct virt_smmu *vsmmuv3_find_by_addr(struct domain *d, paddr_t paddr) +{ + struct virt_smmu *smmu; + + list_for_each_entry( smmu, &d->arch.viommu_list, viommu_list ) + { + if ( smmu->addr == paddr ) + return smmu; + } + + return NULL; +} + +int arm_vsmmu_handle_evt(struct domain *d, struct device *dev, uint64_t *evt) +{ + int ret; + struct virt_smmu *smmu; + + if ( is_hardware_domain(d) ) + { + paddr_t paddr; + /* Base address */ + ret = dt_device_get_address(dev_to_dt(dev), 0, &paddr, NULL); + if ( ret ) + return -EINVAL; + + smmu = vsmmuv3_find_by_addr(d, paddr); + if ( !smmu ) + return -ENODEV; + } + else + { + smmu = list_entry(d->arch.viommu_list.next, + struct virt_smmu, viommu_list); + } + + ret = arm_vsmmu_write_evtq(smmu, evt); + if ( ret ) + arm_vsmmu_inject_irq(smmu, true, GERROR_EVTQ_ABT_ERR); + + return 0; +} + +static int arm_vsmmu_find_ste(struct virt_smmu *smmu, uint32_t sid, + uint64_t *ste) +{ + paddr_t addr, strtab_base; + struct domain *d = smmu->d; + uint32_t log2size; + int strtab_size_shift; + int ret; + struct arm_smmu_evtq_ent ent = { + .sid = sid, + .c_bad_ste_streamid = { + .ssid = 0, + .ssv = false, + }, + }; + + log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, smmu->strtab_base_cfg); + + if ( sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE)) ) + { + ent.opcode = EVT_ID_BAD_STE; + arm_vsmmu_send_event(smmu, &ent); + return -EINVAL; + } + + if ( smmu->features & STRTAB_BASE_CFG_FMT_2LVL ) + { + int idx, max_l2_ste, span; + paddr_t l1ptr, l2ptr; + uint64_t l1std; + + strtab_size_shift = MAX(5, (int)log2size - smmu->sid_split - 1 + 3); + strtab_base = smmu->strtab_base & STRTAB_BASE_ADDR_MASK & + ~GENMASK_ULL(strtab_size_shift, 0); + idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS; + l1ptr = (paddr_t)(strtab_base + idx * sizeof(l1std)); + + ret = access_guest_memory_by_gpa(d, l1ptr, &l1std, + sizeof(l1std), false); + if ( ret ) + { + gdprintk(XENLOG_ERR, + "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr); + return ret; + } + + span = FIELD_GET(STRTAB_L1_DESC_SPAN, l1std); + if ( !span ) + { + gdprintk(XENLOG_ERR, "Bad StreamID span\n"); + return -EINVAL; + } + + max_l2_ste = (1 << span) - 1; + l2ptr = FIELD_PREP(STRTAB_L1_DESC_L2PTR_MASK, + FIELD_GET(STRTAB_L1_DESC_L2PTR_MASK, l1std)); + idx = sid & ((1 << smmu->sid_split) - 1); + if ( idx > max_l2_ste ) + { + gdprintk(XENLOG_ERR, "idx=%d > max_l2_ste=%d\n", + idx, max_l2_ste); + ent.opcode = EVT_ID_BAD_STREAMID; + arm_vsmmu_send_event(smmu, &ent); + return -EINVAL; + } + addr = l2ptr + idx * sizeof(*ste) * STRTAB_STE_DWORDS; + } + else + { + strtab_size_shift = log2size + 5; + strtab_base = smmu->strtab_base & STRTAB_BASE_ADDR_MASK & + ~GENMASK_ULL(strtab_size_shift, 0); + addr = strtab_base + sid * sizeof(*ste) * STRTAB_STE_DWORDS; + } + ret = access_guest_memory_by_gpa(d, addr, ste, sizeof(*ste), false); + if ( ret ) + { + gdprintk(XENLOG_ERR, + "Cannot fetch pte at address=0x%"PRIx64"\n", addr); + return -EINVAL; + } + + return 0; +} + +static int arm_vsmmu_decode_ste(struct virt_smmu *smmu, uint32_t sid, + struct arm_vsmmu_s1_trans_cfg *cfg, + uint64_t *ste) +{ + uint64_t val = ste[0]; + struct arm_smmu_evtq_ent ent = { + .opcode = EVT_ID_BAD_STE, + .sid = sid, + .c_bad_ste_streamid = { + .ssid = 0, + .ssv = false, + }, + }; + + if ( !(val & STRTAB_STE_0_V) ) + return -EAGAIN; + + switch ( FIELD_GET(STRTAB_STE_0_CFG, val) ) + { + case STRTAB_STE_0_CFG_BYPASS: + cfg->bypassed = true; + return 0; + case STRTAB_STE_0_CFG_ABORT: + cfg->aborted = true; + return 0; + case STRTAB_STE_0_CFG_S1_TRANS: + break; + case STRTAB_STE_0_CFG_S2_TRANS: + gdprintk(XENLOG_ERR, "vSMMUv3 does not support stage 2 yet\n"); + goto bad_ste; + default: + BUG(); /* STE corruption */ + } + + cfg->s1ctxptr = smmu_get_ste_s1ctxptr(val); + cfg->s1fmt = smmu_get_ste_s1fmt(val); + cfg->s1cdmax = smmu_get_ste_s1cdmax(val); + if ( cfg->s1cdmax != 0 ) + { + gdprintk(XENLOG_ERR, + "vSMMUv3 does not support multiple context descriptors\n"); + goto bad_ste; + } + + return 0; + +bad_ste: + arm_vsmmu_send_event(smmu, &ent); + return -EINVAL; +} + +static int arm_vsmmu_handle_cfgi_ste(struct virt_smmu *smmu, uint64_t *cmdptr) +{ + int ret; + uint64_t ste[STRTAB_STE_DWORDS]; + struct domain *d = smmu->d; + struct domain_iommu *hd = dom_iommu(d); + struct arm_vsmmu_s1_trans_cfg s1_cfg = {0}; + uint32_t sid = smmu_cmd_get_sid(cmdptr[0]); + struct iommu_guest_config guest_cfg = {0}; + + ret = arm_vsmmu_find_ste(smmu, sid, ste); + if ( ret ) + return ret; + + ret = arm_vsmmu_decode_ste(smmu, sid, &s1_cfg, ste); + if ( ret ) + return (ret == -EAGAIN ) ? 0 : ret; + + guest_cfg.s1ctxptr = s1_cfg.s1ctxptr; + guest_cfg.s1fmt = s1_cfg.s1fmt; + guest_cfg.s1cdmax = s1_cfg.s1cdmax; + + if ( s1_cfg.bypassed ) + guest_cfg.config = ARM_SMMU_DOMAIN_BYPASS; + else if ( s1_cfg.aborted ) + guest_cfg.config = ARM_SMMU_DOMAIN_ABORT; + else + guest_cfg.config = ARM_SMMU_DOMAIN_NESTED; + + ret = hd->platform_ops->attach_guest_config(d, sid, &guest_cfg); + if ( ret ) + return ret; + + return 0; +} + +static int arm_vsmmu_handle_cmds(struct virt_smmu *smmu) +{ + struct arm_vsmmu_queue *q = &smmu->cmdq; + struct domain *d = smmu->d; + uint64_t command[CMDQ_ENT_DWORDS]; + paddr_t addr; + + if ( !smmu_get_cmdq_enabled(smmu->cr[0]) ) + return 0; + + while ( !queue_empty(q) ) + { + int ret; + + addr = Q_CONS_ENT(q); + ret = access_guest_memory_by_gpa(d, addr, command, + sizeof(command), false); + if ( ret ) + return ret; + + switch ( smmu_cmd_get_command(command[0]) ) + { + case CMDQ_OP_CFGI_STE: + ret = arm_vsmmu_handle_cfgi_ste(smmu, command); + break; + case CMDQ_OP_PREFETCH_CFG: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_CD_ALL: + case CMDQ_OP_CFGI_ALL: + case CMDQ_OP_CMD_SYNC: + break; + case CMDQ_OP_TLBI_NH_ASID: + case CMDQ_OP_TLBI_NSNH_ALL: + case CMDQ_OP_TLBI_NH_VA: + if ( !iommu_iotlb_flush_all(smmu->d, 1) ) + break; + default: + gdprintk(XENLOG_ERR, "vSMMUv3: unhandled command\n"); + dump_smmu_command(command); + break; + } + + if ( ret ) + { + gdprintk(XENLOG_ERR, + "vSMMUv3: command error %d while handling command\n", + ret); + dump_smmu_command(command); + } + queue_inc_cons(q); + } + return 0; +} + +static int vsmmuv3_mmio_write(struct vcpu *v, mmio_info_t *info, + register_t r, void *priv) +{ + struct virt_smmu *smmu = priv; + uint64_t reg; + uint32_t reg32; + + switch ( info->gpa & 0xffff ) + { + case VREG32(ARM_SMMU_CR0): + reg32 = smmu->cr[0]; + vreg_reg32_update(®32, r, info); + smmu->cr[0] = reg32; + smmu->cr0ack = reg32 & ~CR0_RESERVED; + break; + + case VREG32(ARM_SMMU_CR1): + reg32 = smmu->cr[1]; + vreg_reg32_update(®32, r, info); + smmu->cr[1] = reg32; + break; + + case VREG32(ARM_SMMU_CR2): + reg32 = smmu->cr[2]; + vreg_reg32_update(®32, r, info); + smmu->cr[2] = reg32; + break; + + case VREG64(ARM_SMMU_STRTAB_BASE): + reg = smmu->strtab_base; + vreg_reg64_update(®, r, info); + smmu->strtab_base = reg; + break; + + case VREG32(ARM_SMMU_STRTAB_BASE_CFG): + reg32 = smmu->strtab_base_cfg; + vreg_reg32_update(®32, r, info); + smmu->strtab_base_cfg = reg32; + + smmu->sid_split = FIELD_GET(STRTAB_BASE_CFG_SPLIT, reg32); + smmu->features |= STRTAB_BASE_CFG_FMT_2LVL; + break; + + case VREG32(ARM_SMMU_CMDQ_BASE): + reg = smmu->cmdq.q_base; + vreg_reg64_update(®, r, info); + smmu->cmdq.q_base = reg; + smmu->cmdq.max_n_shift = FIELD_GET(Q_BASE_LOG2SIZE, smmu->cmdq.q_base); + if ( smmu->cmdq.max_n_shift > SMMU_CMDQS ) + smmu->cmdq.max_n_shift = SMMU_CMDQS; + break; + + case VREG32(ARM_SMMU_CMDQ_PROD): + spin_lock(&smmu->cmd_queue_lock); + reg32 = smmu->cmdq.prod; + vreg_reg32_update(®32, r, info); + smmu->cmdq.prod = reg32; + + if ( arm_vsmmu_handle_cmds(smmu) ) + gdprintk(XENLOG_ERR, "error handling vSMMUv3 commands\n"); + + spin_unlock(&smmu->cmd_queue_lock); + break; + + case VREG32(ARM_SMMU_CMDQ_CONS): + reg32 = smmu->cmdq.cons; + vreg_reg32_update(®32, r, info); + smmu->cmdq.cons = reg32; + break; + + case VREG32(ARM_SMMU_EVTQ_BASE): + reg = smmu->evtq.q_base; + vreg_reg64_update(®, r, info); + smmu->evtq.q_base = reg; + smmu->evtq.max_n_shift = FIELD_GET(Q_BASE_LOG2SIZE, smmu->evtq.q_base); + if ( smmu->cmdq.max_n_shift > SMMU_EVTQS ) + smmu->cmdq.max_n_shift = SMMU_EVTQS; + break; + + case VREG32(ARM_SMMU_EVTQ_PROD): + reg32 = smmu->evtq.prod; + vreg_reg32_update(®32, r, info); + smmu->evtq.prod = reg32; + break; + + case VREG32(ARM_SMMU_EVTQ_CONS): + reg32 = smmu->evtq.cons; + vreg_reg32_update(®32, r, info); + smmu->evtq.cons = reg32; + break; + + case VREG32(ARM_SMMU_IRQ_CTRL): + reg32 = smmu->irq_ctrl; + vreg_reg32_update(®32, r, info); + smmu->irq_ctrl = reg32; + break; + + case VREG64(ARM_SMMU_GERROR_IRQ_CFG0): + reg = smmu->gerror_irq_cfg0; + vreg_reg64_update(®, r, info); + smmu->gerror_irq_cfg0 = reg; + break; + + case VREG64(ARM_SMMU_EVTQ_IRQ_CFG0): + reg = smmu->evtq_irq_cfg0; + vreg_reg64_update(®, r, info); + smmu->evtq_irq_cfg0 = reg; + break; + + case VREG32(ARM_SMMU_GERRORN): + reg = smmu->gerrorn; + vreg_reg64_update(®, r, info); + smmu->gerrorn = reg; + break; + + default: + printk(XENLOG_G_ERR + "%pv: vSMMUv3: unhandled write r%d offset %"PRIpaddr"\n", + v, info->dabt.reg, (unsigned long)info->gpa & 0xffff); + return IO_ABORT; + } + + return IO_HANDLED; +} + +static int vsmmuv3_mmio_read(struct vcpu *v, mmio_info_t *info, + register_t *r, void *priv) +{ + struct virt_smmu *smmu = priv; + uint64_t reg; + + switch ( info->gpa & 0xffff ) + { + case VREG32(ARM_SMMU_IDR0): + reg = FIELD_PREP(IDR0_S1P, 1) | FIELD_PREP(IDR0_TTF, 2) | + FIELD_PREP(IDR0_COHACC, 0) | FIELD_PREP(IDR0_ASID16, 1) | + FIELD_PREP(IDR0_TTENDIAN, 0) | FIELD_PREP(IDR0_STALL_MODEL, 1) | + FIELD_PREP(IDR0_ST_LVL, 1) | FIELD_PREP(IDR0_TERM_MODEL, 1); + *r = vreg_reg32_extract(reg, info); + break; + + case VREG32(ARM_SMMU_IDR1): + reg = FIELD_PREP(IDR1_SIDSIZE, SMMU_IDR1_SIDSIZE) | + FIELD_PREP(IDR1_CMDQS, SMMU_CMDQS) | + FIELD_PREP(IDR1_EVTQS, SMMU_EVTQS); + *r = vreg_reg32_extract(reg, info); + break; + + case VREG32(ARM_SMMU_IDR2): + goto read_reserved; + + case VREG32(ARM_SMMU_IDR3): + reg = FIELD_PREP(IDR3_RIL, 0); + *r = vreg_reg32_extract(reg, info); + break; + + case VREG32(ARM_SMMU_IDR4): + goto read_impl_defined; + + case VREG32(ARM_SMMU_IDR5): + reg = FIELD_PREP(IDR5_GRAN4K, 1) | FIELD_PREP(IDR5_GRAN16K, 1) | + FIELD_PREP(IDR5_GRAN64K, 1) | FIELD_PREP(IDR5_OAS, IDR5_OAS_48_BIT); + *r = vreg_reg32_extract(reg, info); + break; + + case VREG32(ARM_SMMU_IIDR): + *r = vreg_reg32_extract(ARM_SMMU_IIDR_VAL, info); + break; + + case VREG32(ARM_SMMU_CR0): + *r = vreg_reg32_extract(smmu->cr[0], info); + break; + + case VREG32(ARM_SMMU_CR0ACK): + *r = vreg_reg32_extract(smmu->cr0ack, info); + break; + + case VREG32(ARM_SMMU_CR1): + *r = vreg_reg32_extract(smmu->cr[1], info); + break; + + case VREG32(ARM_SMMU_CR2): + *r = vreg_reg32_extract(smmu->cr[2], info); + break; + + case VREG32(ARM_SMMU_STRTAB_BASE): + *r = vreg_reg64_extract(smmu->strtab_base, info); + break; + + case VREG32(ARM_SMMU_STRTAB_BASE_CFG): + *r = vreg_reg32_extract(smmu->strtab_base_cfg, info); + break; + + case VREG32(ARM_SMMU_CMDQ_BASE): + *r = vreg_reg64_extract(smmu->cmdq.q_base, info); + break; + + case VREG32(ARM_SMMU_CMDQ_PROD): + *r = vreg_reg32_extract(smmu->cmdq.prod, info); + break; + + case VREG32(ARM_SMMU_CMDQ_CONS): + *r = vreg_reg32_extract(smmu->cmdq.cons, info); + break; + + case VREG32(ARM_SMMU_EVTQ_BASE): + *r = vreg_reg64_extract(smmu->evtq.q_base, info); + break; + + case VREG32(ARM_SMMU_EVTQ_PROD): + *r = vreg_reg32_extract(smmu->evtq.prod, info); + break; + + case VREG32(ARM_SMMU_EVTQ_CONS): + *r = vreg_reg32_extract(smmu->evtq.cons, info); + break; + + case VREG32(ARM_SMMU_IRQ_CTRL): + case VREG32(ARM_SMMU_IRQ_CTRLACK): + *r = vreg_reg32_extract(smmu->irq_ctrl, info); + break; + + case VREG64(ARM_SMMU_GERROR_IRQ_CFG0): + *r = vreg_reg64_extract(smmu->gerror_irq_cfg0, info); + break; + + case VREG64(ARM_SMMU_EVTQ_IRQ_CFG0): + *r = vreg_reg64_extract(smmu->evtq_irq_cfg0, info); + break; + + case VREG32(ARM_SMMU_GERROR): + *r = vreg_reg64_extract(smmu->gerror, info); + break; + + case VREG32(ARM_SMMU_GERRORN): + *r = vreg_reg64_extract(smmu->gerrorn, info); + break; + + default: + printk(XENLOG_G_ERR + "%pv: vSMMUv3: unhandled read r%d offset %"PRIpaddr"\n", + v, info->dabt.reg, (unsigned long)info->gpa & 0xffff); + return IO_ABORT; + } + + return IO_HANDLED; + + read_impl_defined: + printk(XENLOG_G_DEBUG + "%pv: vSMMUv3: RAZ on implementation defined register offset %"PRIpaddr"\n", + v, info->gpa & 0xffff); + *r = 0; + return IO_HANDLED; + + read_reserved: + printk(XENLOG_G_DEBUG + "%pv: vSMMUv3: RAZ on reserved register offset %"PRIpaddr"\n", + v, info->gpa & 0xffff); + *r = 0; + return IO_HANDLED; +} + +static const struct mmio_handler_ops vsmmuv3_mmio_handler = { + .read = vsmmuv3_mmio_read, + .write = vsmmuv3_mmio_write, +}; + +static int vsmmuv3_init_single(struct domain *d, paddr_t addr, + paddr_t size, uint32_t virq) +{ + int ret; + struct virt_smmu *smmu; + + smmu = xzalloc(struct virt_smmu); + if ( !smmu ) + return -ENOMEM; + + smmu->d = d; + smmu->virq = virq; + smmu->addr = addr; + smmu->cmdq.q_base = FIELD_PREP(Q_BASE_LOG2SIZE, SMMU_CMDQS); + smmu->cmdq.ent_size = CMDQ_ENT_DWORDS * DWORDS_BYTES; + smmu->evtq.q_base = FIELD_PREP(Q_BASE_LOG2SIZE, SMMU_EVTQS); + smmu->evtq.ent_size = EVTQ_ENT_DWORDS * DWORDS_BYTES; + + spin_lock_init(&smmu->cmd_queue_lock); + + ret = vgic_reserve_virq(d, virq); + if ( !ret ) + goto out; + + register_mmio_handler(d, &vsmmuv3_mmio_handler, addr, size, smmu); + + /* Register the vIOMMU to be able to clean it up later. */ + list_add_tail(&smmu->viommu_list, &d->arch.viommu_list); + + return 0; + +out: + xfree(smmu); + vgic_free_virq(d, virq); + return ret; +} + +int domain_vsmmuv3_init(struct domain *d) +{ + int ret; + INIT_LIST_HEAD(&d->arch.viommu_list); + + if ( is_hardware_domain(d) ) + { + struct host_iommu *hw_iommu; + + list_for_each_entry(hw_iommu, &host_iommu_list, entry) + { + ret = vsmmuv3_init_single(d, hw_iommu->addr, hw_iommu->size, + hw_iommu->irq); + if ( ret ) + return ret; + } + } + else + { + ret = vsmmuv3_init_single(d, GUEST_VSMMUV3_BASE, GUEST_VSMMUV3_SIZE, + GUEST_VSMMU_SPI); + if ( ret ) + return ret; + } + + return 0; +} + +int vsmmuv3_relinquish_resources(struct domain *d) +{ + struct virt_smmu *pos, *temp; + + /* Cope with unitialized vIOMMU */ + if ( list_head_is_null(&d->arch.viommu_list) ) + return 0; + + list_for_each_entry_safe(pos, temp, &d->arch.viommu_list, viommu_list ) + { + list_del(&pos->viommu_list); + xfree(pos); + } + + return 0; +} + +static const struct viommu_ops vsmmuv3_ops = { + .domain_init = domain_vsmmuv3_init, + .relinquish_resources = vsmmuv3_relinquish_resources, +}; + +static const struct viommu_desc vsmmuv3_desc = { + .ops = &vsmmuv3_ops, + .viommu_type = XEN_DOMCTL_CONFIG_VIOMMU_SMMUV3, +}; + +void __init vsmmuv3_set_type(void) +{ + const struct viommu_desc *desc = &vsmmuv3_desc; + + if ( !is_viommu_enabled() ) + return; + + if ( cur_viommu && (cur_viommu != desc) ) + { + printk("WARNING: Cannot set vIOMMU, already set to a different value\n"); + return; + } + + cur_viommu = desc; +} + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/drivers/passthrough/arm/vsmmu-v3.h b/xen/drivers/passthrough/arm/vsmmu-v3.h new file mode 100644 index 000000000000..c7bfd3fb5950 --- /dev/null +++ b/xen/drivers/passthrough/arm/vsmmu-v3.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +#ifndef __ARCH_ARM_VSMMU_V3_H__ +#define __ARCH_ARM_VSMMU_V3_H__ + +#include + +#ifdef CONFIG_VIRTUAL_ARM_SMMU_V3 + +void vsmmuv3_set_type(void); + +static inline int arm_vsmmu_handle_evt(struct domain *d, + struct device *dev, uint64_t *evt) +{ + return -EINVAL; +} + +#else + +static inline void vsmmuv3_set_type(void) +{ + return; +} + +static inline int arm_vsmmu_handle_evt(struct domain *d, + struct device *dev, uint64_t *evt) +{ + return -EINVAL; +} + +#endif /* CONFIG_VIRTUAL_ARM_SMMU_V3 */ + +#endif /* __ARCH_ARM_VSMMU_V3_H__ */ diff --git a/xen/include/public/arch-arm.h b/xen/include/public/arch-arm.h index e2412a17474e..1b03222b77da 100644 --- a/xen/include/public/arch-arm.h +++ b/xen/include/public/arch-arm.h @@ -327,12 +327,17 @@ DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t); #define XEN_DOMCTL_CONFIG_TEE_OPTEE 1 #define XEN_DOMCTL_CONFIG_TEE_FFA 2 +#define XEN_DOMCTL_CONFIG_VIOMMU_NONE 0 +#define XEN_DOMCTL_CONFIG_VIOMMU_SMMUV3 1 + struct xen_arch_domainconfig { /* IN/OUT */ uint8_t gic_version; /* IN - Contains SVE vector length divided by 128 */ uint8_t sve_vl; /* IN */ + uint8_t viommu_type; + /* IN */ uint16_t tee_type; /* IN */ uint32_t nr_spis; @@ -446,6 +451,10 @@ typedef uint64_t xen_callback_t; #define GUEST_GICV3_GICR0_BASE xen_mk_ullong(0x03020000) /* vCPU0..127 */ #define GUEST_GICV3_GICR0_SIZE xen_mk_ullong(0x01000000) +/* vsmmuv3 ITS mappings */ +#define GUEST_VSMMUV3_BASE xen_mk_ullong(0x04040000) +#define GUEST_VSMMUV3_SIZE xen_mk_ullong(0x00040000) + /* * 256 MB is reserved for VPCI configuration space based on calculation * 256 buses x 32 devices x 8 functions x 4 KB = 256 MB @@ -512,9 +521,10 @@ typedef uint64_t xen_callback_t; #define GUEST_EVTCHN_PPI 31 #define GUEST_VPL011_SPI 32 +#define GUEST_VSMMU_SPI 33 -#define GUEST_VIRTIO_MMIO_SPI_FIRST 33 -#define GUEST_VIRTIO_MMIO_SPI_LAST 43 +#define GUEST_VIRTIO_MMIO_SPI_FIRST 34 +#define GUEST_VIRTIO_MMIO_SPI_LAST 44 /* * SGI is the preferred delivery mechanism of FF-A pending notifications or diff --git a/xen/include/public/device_tree_defs.h b/xen/include/public/device_tree_defs.h index 9e80d0499dc3..7846a0425c6e 100644 --- a/xen/include/public/device_tree_defs.h +++ b/xen/include/public/device_tree_defs.h @@ -14,6 +14,7 @@ */ #define GUEST_PHANDLE_GIC (65000) #define GUEST_PHANDLE_IOMMU (GUEST_PHANDLE_GIC + 1) +#define GUEST_PHANDLE_VSMMUV3 (GUEST_PHANDLE_IOMMU + 1) #define GUEST_ROOT_ADDRESS_CELLS 2 #define GUEST_ROOT_SIZE_CELLS 2 diff --git a/xen/include/xen/iommu.h b/xen/include/xen/iommu.h index 77a514019cc6..753db9f82bac 100644 --- a/xen/include/xen/iommu.h +++ b/xen/include/xen/iommu.h @@ -240,6 +240,15 @@ int iommu_remove_dt_device(struct dt_device_node *np); #endif /* HAS_DEVICE_TREE */ +#ifdef CONFIG_ARM +struct iommu_guest_config { + paddr_t s1ctxptr; + uint8_t config; + uint8_t s1fmt; + uint8_t s1cdmax; +}; +#endif /* CONFIG_ARM */ + struct page_info; /* @@ -316,6 +325,11 @@ struct iommu_ops { #endif /* Inhibit all interrupt generation, to be used at shutdown. */ void (*quiesce)(void); + +#ifdef CONFIG_ARM + int (*attach_guest_config)(struct domain *d, u32 sid, + struct iommu_guest_config *cfg); +#endif }; /*