Sandro/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-09-18 23:09:29 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branches 'arm/smmu/updates', 'arm/smmu/bindings', 'mediatek', 'qualcomm/msm', 'rockchip', 'ti/omap', 'riscv', 'intel/vt-d', 'amd/amd-vi', 'core' and 'typos' into next

Browse Source
This commit is contained in:
Joerg Roedel
2026-08-11 09:12:19 +02:00
parent db2ddb8714 340e3c5165 02f1359b11 aebaa93f3d 535a200220 b10d5920ca 57f63e7082 a9c7b86b03 c54e4ae971 cb2860ad6c 5a9e89ea34 57bd32b2af
commit 3c6a2bac15
44 changed files with 1029 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/admin-guide/kernel-parameters.txt
+3
View File
@@ -565,6 +565,9 @@ Kernel parameters
arm64.nogcs [ARM64] Unconditionally disable Guarded Control Stack
support
arm64.nohaft [ARM64] Unconditionally disable Hardware managed Access
Flag for Table descriptors support
arm64.nomops [ARM64] Unconditionally disable Memory Copy and Memory
Set instructions support
Documentation/arch/arm64/silicon-errata.rst
+2
View File
@@ -316,6 +316,8 @@ stable kernels.
| | | T241-MPAM-4, | |
| | | T241-MPAM-6 | |
+----------------+-----------------+-----------------+-----------------------------+
| NVIDIA | T264 SMMU | T264-SMMU-3 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
+----------------+-----------------+-----------------+-----------------------------+
| Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 |
+----------------+-----------------+-----------------+-----------------------------+
Documentation/devicetree/bindings/iommu/arm,smmu.yaml
+8
View File
@@ -38,7 +38,9 @@ properties:
- qcom,eliza-smmu-500
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,ipq9650-smmu-500
- qcom,kaanapali-smmu-500
- qcom,maili-smmu-500
- qcom,milos-smmu-500
- qcom,nord-smmu-500
- qcom,qcm2290-smmu-500
@@ -95,6 +97,7 @@ properties:
- description: Qcom Adreno GPUs implementing "qcom,smmu-500" and "arm,mmu-500"
items:
- enum:
- qcom,eliza-smmu-500
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
@@ -108,6 +111,7 @@ properties:
- qcom,sc7280-smmu-500
- qcom,sc8180x-smmu-500
- qcom,sc8280xp-smmu-500
- qcom,shikra-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6125-smmu-500
- qcom,sm8150-smmu-500
@@ -543,6 +547,7 @@ allOf:
- enum:
- qcom,milos-smmu-500
- qcom,sar2130p-smmu-500
- qcom,shikra-smmu-500
- qcom,sm8550-smmu-500
- qcom,sm8650-smmu-500
- qcom,x1e80100-smmu-500
@@ -570,6 +575,7 @@ allOf:
compatible:
items:
- enum:
- qcom,eliza-smmu-500
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
@@ -620,7 +626,9 @@ allOf:
- qcom,eliza-smmu-500
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,ipq9650-smmu-500
- qcom,kaanapali-smmu-500
- qcom,maili-smmu-500
- qcom,milos-smmu-500
- qcom,nord-smmu-500
- qcom,qcs615-smmu-500
Documentation/devicetree/bindings/iommu/nvidia,tegra264-cmdqv.yaml
+1 -1
View File
@@ -38,5 +38,5 @@ examples:
cmdqv@5200000 {
compatible = "nvidia,tegra264-cmdqv";
reg = <0x5200000 0x830000>;
interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
interrupts = <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>;
};
Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt
-59
View File
@@ -1,59 +0,0 @@
OMAP2+ IOMMU
Required properties:
- compatible : Should be one of,
"ti,omap2-iommu" for OMAP2/OMAP3 IOMMU instances
"ti,omap4-iommu" for OMAP4/OMAP5 IOMMU instances
"ti,dra7-dsp-iommu" for DRA7xx DSP IOMMU instances
"ti,dra7-iommu" for DRA7xx IOMMU instances
- ti,hwmods : Name of the hwmod associated with the IOMMU instance
- reg : Address space for the configuration registers
- interrupts : Interrupt specifier for the IOMMU instance
- #iommu-cells : Should be 0. OMAP IOMMUs are all "single-master" devices,
and needs no additional data in the pargs specifier. Please
also refer to the generic bindings document for more info
on this property,
Documentation/devicetree/bindings/iommu/iommu.txt
Optional properties:
- ti,#tlb-entries : Number of entries in the translation look-aside buffer.
Should be either 8 or 32 (default: 32)
- ti,iommu-bus-err-back : Indicates the IOMMU instance supports throwing
back a bus error response on MMU faults.
- ti,syscon-mmuconfig : Should be a pair of the phandle to the DSP_SYSTEM
syscon node that contains the additional control
register for enabling the MMU, and the MMU instance
number (0-indexed) within the sub-system. This property
is required for DSP IOMMU instances on DRA7xx SoCs. The
instance number should be 0 for DSP MDMA MMUs and 1 for
DSP EDMA MMUs.
Example:
/* OMAP3 ISP MMU */
mmu_isp: mmu@480bd400 {
#iommu-cells = <0>;
compatible = "ti,omap2-iommu";
reg = <0x480bd400 0x80>;
interrupts = <24>;
ti,hwmods = "mmu_isp";
ti,#tlb-entries = <8>;
};
/* DRA74x DSP2 MMUs */
mmu0_dsp2: mmu@41501000 {
compatible = "ti,dra7-dsp-iommu";
reg = <0x41501000 0x100>;
interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
ti,hwmods = "mmu0_dsp2";
#iommu-cells = <0>;
ti,syscon-mmuconfig = <&dsp2_system 0x0>;
};
mmu1_dsp2: mmu@41502000 {
compatible = "ti,dra7-dsp-iommu";
reg = <0x41502000 0x100>;
interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
ti,hwmods = "mmu1_dsp2";
#iommu-cells = <0>;
ti,syscon-mmuconfig = <&dsp2_system 0x1>;
};
Documentation/devicetree/bindings/iommu/ti,omap-iommu.yaml
+114
View File
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/iommu/ti,omap-iommu.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: OMAP2+ IOMMU
maintainers:
- Bhargav Joshi <j.bhargav.u@gmail.com>
properties:
compatible:
enum:
- ti,omap2-iommu
- ti,omap4-iommu
- ti,dra7-dsp-iommu
- ti,dra7-iommu
reg:
maxItems: 1
interrupts:
maxItems: 1
'#iommu-cells':
const: 0
ti,#tlb-entries:
description: Number of entries in the translation look-aside buffer.
$ref: /schemas/types.yaml#/definitions/uint32
enum: [8, 32]
default: 32
ti,iommu-bus-err-back:
description:
Indicates the IOMMU instance supports throwing back a bus error response
on MMU faults.
type: boolean
ti,syscon-mmuconfig:
description:
Pair of the phandle to the DSP_SYSTEM syscon node that
contains the additional control register for enabling the MMU, and the
MMU instance number (0-indexed) within the sub-system. The instance number
should be 0 for DSP MDMA MMUs and 1 for DSP EDMA MMUs.
$ref: /schemas/types.yaml#/definitions/phandle-array
maxItems: 1
items:
items:
- description: phandle to the DSP_SYSTEM syscon node
- description: MMU instance number
enum: [0, 1]
ti,hwmods:
description: Name of the hwmod associated with the IOMMU instance
$ref: /schemas/types.yaml#/definitions/string
deprecated: true
required:
- compatible
- reg
- '#iommu-cells'
- interrupts
allOf:
- if:
properties:
compatible:
contains:
const: ti,dra7-dsp-iommu
then:
required:
- ti,syscon-mmuconfig
else:
properties:
ti,syscon-mmuconfig: false
additionalProperties: false
examples:
- |
mmu@480bd400 {
#iommu-cells = <0>;
compatible = "ti,omap2-iommu";
reg = <0x480bd400 0x80>;
interrupts = <24>;
ti,hwmods = "mmu_isp";
ti,#tlb-entries = <8>;
};
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
mmu@41501000 {
compatible = "ti,dra7-dsp-iommu";
reg = <0x41501000 0x100>;
interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
ti,hwmods = "mmu0_dsp2";
#iommu-cells = <0>;
ti,syscon-mmuconfig = <&dsp2_system 0x0>;
};
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
mmu@41502000 {
compatible = "ti,dra7-dsp-iommu";
reg = <0x41502000 0x100>;
interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
ti,hwmods = "mmu1_dsp2";
#iommu-cells = <0>;
ti,syscon-mmuconfig = <&dsp2_system 0x1>;
};
arch/arm64/kernel/pi/idreg-override.c
+2
View File
@@ -64,6 +64,7 @@ static const struct ftr_set_desc mmfr1 __prel64_initconst = {
.override = &id_aa64mmfr1_override,
.fields = {
FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter),
FIELD("hafdbs", ID_AA64MMFR1_EL1_HAFDBS_SHIFT, NULL),
{}
},
};
@@ -246,6 +247,7 @@ static const struct {
{ "arm64.nomte", "id_aa64pfr1.mte=0" },
{ "nokaslr", "arm64_sw.nokaslr=1" },
{ "rodata=off", "arm64_sw.rodataoff=1" },
{ "arm64.nohaft", "id_aa64mmfr1.hafdbs=2" },
{ "arm64.nolva", "id_aa64mmfr2.varange=0" },
{ "arm64.no32bit_el0", "id_aa64pfr0.el0=1" },
{ "arm64.nompam", "id_aa64pfr0.mpam=0 id_aa64pfr1.mpam_frac=0" },
drivers/iommu/amd/amd_iommu_types.h
+8 -1
View File
@@ -113,6 +113,7 @@
#define FEATURE_SNPAVICSUP_GAM(x) \
(FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
#define FEATURE_HT_RANGE_IGNORE BIT_ULL(11)
#define FEATURE_SNP_PAGE_MODE0_SUP BIT_ULL(13)
#define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8)
#define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
@@ -159,6 +160,8 @@
#define EVENT_FLAGS_SHIFT 0x10
#define EVENT_FLAG_RW 0x020
#define EVENT_FLAG_I 0x008
#define EVENT_FLAG_PPR_RX 0x001
#define EVENT_FLAG_PPR_GN 0x200
/* feature control bits */
#define CONTROL_IOMMU_EN 0
@@ -280,7 +283,8 @@
#define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
#define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
#define PPR_DEVID(x) ((x) & 0xffffULL)
#define PPR_TAG(x) (((x) >> 32) & 0x3ffULL)
#define PPR_TAG(x) (((x) >> 32) & 0x1ffULL)
#define PPR_TAG_LAST_PAGE(x) (((x) >> 32) & 0x200ULL)
#define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
#define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
#define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
@@ -416,6 +420,9 @@ extern bool amd_iommu_dump;
pr_info(format, ## arg); \
} while(0);
/* SNP page mode 0 support */
extern bool amd_iommu_snp_mode0_sup;
/* global flag if IOMMUs cache non-present entries */
extern bool amd_iommu_np_cache;
/* Only true if all IOMMUs support device IOTLBs */
drivers/iommu/amd/debugfs.c
+3 -9
View File
@@ -176,19 +176,13 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf,
kfree(srcid_ptr);
return -ENODEV;
}
break;
}
if (pci_seg->id != seg) {
sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid);
kfree(srcid_ptr);
return -EINVAL;
return cnt;
}
sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid);
kfree(srcid_ptr);
return cnt;
return -EINVAL;
}
static int devid_show(struct seq_file *m, void *unused)
drivers/iommu/amd/init.c
+72 -27
View File
@@ -180,6 +180,9 @@ bool amd_iommu_hatdis;
bool amd_iommu_snp_en;
EXPORT_SYMBOL(amd_iommu_snp_en);
/* SNP page mode 0 support */
bool amd_iommu_snp_mode0_sup;
LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */
LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */
LIST_HEAD(amd_ivhd_dev_flags_list); /* list of all IVHD device entry settings */
@@ -3085,20 +3088,44 @@ static void __init free_iommu_resources(void)
free_pci_segments();
}
/* SB IOAPIC is always on this device in AMD systems */
#define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
static bool __init check_sb_ioapic(int devid)
{
u8 bus = PCI_BUS_NUM(devid);
u8 devfn = devid & 0xff;
u16 val;
/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */
#define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0))
val = read_pci_config_16(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
PCI_CLASS_DEVICE);
/*
* The SB IOAPIC is integrated into the FCH (Southbridge), which is
* exposed as an SMBus or ISA bridge in PCI config space.
*/
return val == PCI_CLASS_SERIAL_SMBUS || val == PCI_CLASS_BRIDGE_ISA;
}
/*
* The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts
* 0 through 23).
*/
static int __init get_sb_ioapic_id(void)
{
int idx = mp_find_ioapic(0);
if (idx < 0)
return -ENODEV;
return mpc_ioapic_id(idx);
}
static bool __init check_ioapic_information(void)
{
const char *fw_bug = FW_BUG;
bool ret, has_sb_ioapic;
int idx;
int idx, sb_apicid;
has_sb_ioapic = false;
ret = false;
ret = true;
/*
* If we have map overrides on the kernel command line the
@@ -3108,6 +3135,16 @@ static bool __init check_ioapic_information(void)
if (cmdline_maps)
fw_bug = "";
sb_apicid = get_sb_ioapic_id();
if (sb_apicid < 0) {
/*
* Lack of SB IOAPIC registration is not a firmware bug,
* e.g. kernel booted with noapic or noacpi.
*/
fw_bug = "";
goto out;
}
for (idx = 0; idx < nr_ioapics; idx++) {
int devid, id = mpc_ioapic_id(idx);
@@ -3116,17 +3153,11 @@ static bool __init check_ioapic_information(void)
pr_err("%s: IOAPIC[%d] not in IVRS table\n",
fw_bug, id);
ret = false;
} else if (devid == IOAPIC_SB_DEVID ||
(boot_cpu_data.x86_vendor == X86_VENDOR_HYGON &&
boot_cpu_data.x86 == 0x18 &&
boot_cpu_data.x86_model >= 0x4 &&
boot_cpu_data.x86_model <= 0xf &&
devid == IOAPIC_SB_DEVID_FAM18H_M4H)) {
} else if (id == sb_apicid && check_sb_ioapic(devid)) {
has_sb_ioapic = true;
ret = true;
}
}
out:
if (!has_sb_ioapic) {
/*
* We expect the SB IOAPIC to be listed in the IVRS
@@ -3137,6 +3168,7 @@ static bool __init check_ioapic_information(void)
* device id for the IOAPIC in the system.
*/
pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
ret = false;
}
if (!ret)
@@ -3365,18 +3397,28 @@ static __init void iommu_snp_enable(void)
#ifdef CONFIG_KVM_AMD_SEV
if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
return;
/*
* The SNP support requires that IOMMU must be enabled, and is
* configured with V1 page table (DTE[Mode] = 0 is not supported).
*/
if (no_iommu || iommu_default_passthrough()) {
pr_warn("SNP: IOMMU disabled or configured in passthrough mode, SNP cannot be supported.\n");
/* SNP support required IOMMU to be ON */
if (no_iommu) {
pr_warn("SNP: IOMMU disabled, SNP cannot be supported.\n");
goto disable_snp;
}
if (amd_iommu_pgtable != PD_MODE_V1) {
pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n");
goto disable_snp;
amd_iommu_snp_mode0_sup = check_feature2(FEATURE_SNP_PAGE_MODE0_SUP);
/*
* If SNP page mode 0 is not enabled, then SNP support requires that IOMMU
* must be configured with V1 page table (DTE[Mode] != 0).
*/
if (!amd_iommu_snp_mode0_sup) {
if (iommu_default_passthrough()) {
pr_warn("SNP: IOMMU configured in passthrough mode, SNP cannot be supported.\n");
goto disable_snp;
}
if (amd_iommu_pgtable != PD_MODE_V1) {
pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n");
goto disable_snp;
}
}
amd_iommu_snp_en = check_feature(FEATURE_SNP);
@@ -3917,12 +3959,15 @@ bool amd_iommu_pasid_supported(void)
amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
return false;
if (!amd_iommu_gt_ppr_supported())
return false;
/*
* Since DTE[Mode]=0 is prohibited on SNP-enabled system
* (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
* setting up IOMMUv1 page table.
* If SNP page mode 0 is not supported, then DTE[Mode]=0 is prohibited
* on SNP-enabled system (i.e. EFR[SNPSup]=1). IOMMUv2 page table
* cannot be used without setting up IOMMUv1 page table.
*/
return amd_iommu_gt_ppr_supported() && !amd_iommu_snp_en;
return (!amd_iommu_snp_en) || (amd_iommu_snp_en && amd_iommu_snp_mode0_sup);
}
struct amd_iommu *get_amd_iommu(unsigned int idx)
drivers/iommu/amd/iommu.c
+59 -16
View File
@@ -92,6 +92,9 @@ static void clone_aliases(struct amd_iommu *iommu, struct device *dev);
static int iommu_completion_wait(struct amd_iommu *iommu);
static int __amd_iommu_complete_ppr(struct device *dev, u32 pasid,
int status, int tag, bool gn);
/****************************************************************************
*
* Helper functions
@@ -456,7 +459,7 @@ static void clone_aliases(struct amd_iommu *iommu, struct device *dev)
/*
* The IVRS alias stored in the alias table may not be
* part of the PCI DMA aliases if it's bus differs
* part of the PCI DMA aliases if its bus differs
* from the original device.
*/
clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], pdev);
@@ -505,7 +508,7 @@ static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)
}
/*
* Find or create an IOMMU group for a acpihid device.
* Find or create an IOMMU group for an acpihid device.
*/
static struct iommu_group *acpihid_device_group(struct device *dev)
{
@@ -905,10 +908,44 @@ out:
pci_dev_put(pdev);
}
static void amd_iommu_report_ppr_err(struct amd_iommu *iommu, volatile u32 *event,
u16 devid, u64 address, int flags)
{
struct pci_dev *pdev;
struct device *dev = iommu->iommu.dev;
u32 pasid = PPR_PASID(*((u64 *)event));
int tag = event[1] & 0x03FF;
bool gn;
dev_err_ratelimited(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x "
"pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n",
iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid), pasid, address, flags, tag);
/* Skip COMPLETE_PPR_REQUEST response if RX=1 */
if (flags & EVENT_FLAG_PPR_RX)
return;
pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
devid & 0xff);
if (!pdev)
return;
if (!dev_iommu_priv_get(&pdev->dev)) {
pci_dev_put(pdev);
return;
}
gn = (flags & EVENT_FLAG_PPR_GN);
__amd_iommu_complete_ppr(&pdev->dev, pasid, IOMMU_PAGE_RESP_FAILURE, tag, gn);
pci_dev_put(pdev);
}
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
{
struct device *dev = iommu->iommu.dev;
int type, devid, flags, tag;
int type, devid, flags;
volatile u32 *event = __evt;
int count = 0;
u64 address, ctrl;
@@ -982,11 +1019,7 @@ retry:
amd_iommu_report_rmp_hw_error(iommu, event);
break;
case EVENT_TYPE_INV_PPR_REQ:
pasid = PPR_PASID(*((u64 *)__evt));
tag = event[1] & 0x03FF;
dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n",
iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
pasid, address, flags, tag);
amd_iommu_report_ppr_err(iommu, event, devid, address, flags);
break;
default:
dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",
@@ -1268,7 +1301,7 @@ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
/*
* Builds an invalidation address which is suitable for one page or multiple
* pages. Sets the size bit (S) as needed is more than one page is flushed.
* pages. Sets the size bit (S) as needed if more than one page is flushed.
*/
static inline u64 build_inv_address(u64 address, u64 last)
{
@@ -1345,7 +1378,7 @@ static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
}
static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,
int status, int tag, u8 gn)
int status, int tag, bool gn)
{
memset(cmd, 0, sizeof(*cmd));
@@ -1850,7 +1883,8 @@ static void dev_flush_pasid_all(struct iommu_dev_data *dev_data,
amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, U64_MAX);
}
int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
static int __amd_iommu_complete_ppr(struct device *dev, u32 pasid,
int status, int tag, bool gn)
{
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
@@ -1859,12 +1893,21 @@ int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
dev_data = dev_iommu_priv_get(dev);
iommu = get_amd_iommu_from_dev(dev);
build_complete_ppr(&cmd, dev_data->devid, pasid, status,
tag, dev_data->pri_tlp);
build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, gn);
return iommu_queue_command(iommu, &cmd);
}
int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
bool gn;
gn = pdom_is_v2_pgtbl_mode(dev_data->domain);
return __amd_iommu_complete_ppr(dev, pasid, status, tag, gn);
}
/****************************************************************************
*
* The next functions belong to the domain allocation. A domain is
@@ -2920,9 +2963,9 @@ static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *de
{
/*
* Don't allow attaching a device to the identity domain if SNP is
* enabled.
* enabled and SNP Mode0 support is not present.
*/
if (amd_iommu_snp_en)
if (amd_iommu_snp_en && !amd_iommu_snp_mode0_sup)
return -EINVAL;
return amd_iommu_attach_device(dom, dev, old);
@@ -3697,7 +3740,7 @@ static void fill_msi_msg(struct msi_msg *msg, u32 index)
/*
* The struct msi_msg.dest_mode_logical is used to set the DM bit
* in MSI Message Address Register. For device w/ 2K int-remap support,
* this is bit must be set to 1 regardless of the actual destination
* this bit must be set to 1 regardless of the actual destination
* mode, which is signified by the IRTE[DM].
*/
if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
drivers/iommu/amd/ppr.c
+6 -6
View File
@@ -130,7 +130,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
event.fault.prm.perm = ppr_flag_to_fault_perm(PPR_FLAGS(raw[0]));
event.fault.prm.addr = (u64)(raw[1] & PAGE_MASK);
event.fault.prm.pasid = PPR_PASID(raw[0]);
event.fault.prm.grpid = PPR_TAG(raw[0]) & 0x1FF;
event.fault.prm.grpid = PPR_TAG(raw[0]);
/*
* PASID zero is used for requests from the I/O device without
@@ -140,25 +140,25 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
if (event.fault.prm.pasid == 0 ||
event.fault.prm.pasid >= dev_data->max_pasids) {
pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n",
event.fault.prm.pasid, pdev->dev.id);
event.fault.prm.pasid, dev_data->devid);
goto out;
}
event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
if (PPR_TAG(raw[0]) & 0x200)
if (PPR_TAG_LAST_PAGE(raw[0]))
event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
/* Submit event */
iommu_report_device_fault(&pdev->dev, &event);
pci_dev_put(pdev);
return;
out:
/* Nobody cared, abort */
amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]),
IOMMU_PAGE_RESP_FAILURE,
PPR_TAG(raw[0]) & 0x1FF);
IOMMU_PAGE_RESP_FAILURE, PPR_TAG(raw[0]));
pci_dev_put(pdev);
}
void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
+5 -2
View File
@@ -33,6 +33,9 @@ void *arm_smmu_hw_info(struct device *dev, u32 *length,
info->iidr = readl_relaxed(master->smmu->base + ARM_SMMU_IIDR);
info->aidr = readl_relaxed(master->smmu->base + ARM_SMMU_AIDR);
if (arm_smmu_erratum_repeat_tlbi_cfgi())
info->flags |= IOMMU_HW_INFO_ARM_SMMUV3_ERRATA_REPEAT_TLBI_CFGI;
*length = sizeof(*info);
*type = IOMMU_HW_INFO_TYPE_ARM_SMMUV3;
@@ -400,8 +403,8 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
continue;
/* FIXME always uses the main cmdq rather than trying to group by type */
ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &last->cmd,
cur - last, true);
ret = __arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &last->cmd,
cur - last, true);
if (ret) {
cur--;
goto out;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+14 -10
View File
@@ -92,16 +92,6 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) &
CTXDESC_CD_1_TTB0_MASK);
/*
* Enable Hardware Access and Dirty updates (DBM) if supported.
* This is safe to enable by default, as PTE_WRITE and PTE_DBM
* share the same bit.
*/
if (master->smmu->features & ARM_SMMU_FEAT_HA)
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA);
if (master->smmu->features & ARM_SMMU_FEAT_HD)
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD);
} else {
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0);
@@ -114,6 +104,17 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
target->data[0] &=
cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R));
}
/*
* Enable Hardware Access and Dirty updates (DBM) if supported. This is
* safe to enable by default, as PTE_WRITE and PTE_DBM share the same bit,
* while the EPD0 config can't get as far as fetching any PTEs anyway.
*/
if (master->smmu->features & ARM_SMMU_FEAT_HA)
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA);
if (master->smmu->features & ARM_SMMU_FEAT_HD)
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD);
if (master->smmu->features & ARM_SMMU_FEAT_HAFT && system_supports_haft())
target->data[1] |= cpu_to_le64(CTXDESC_CD_1_HAFT);
/*
* MAIR value is pretty much constant and global, so we can just get it
@@ -211,6 +212,9 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
if (system_supports_bbml2_noabort())
feat_mask |= ARM_SMMU_FEAT_BBML2;
if (system_supports_haft())
feat_mask |= ARM_SMMU_FEAT_HAFT;
if ((smmu->features & feat_mask) != feat_mask)
return false;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+149 -39
View File
@@ -18,6 +18,7 @@
#include <linux/interrupt.h>
#include <linux/io-pgtable.h>
#include <linux/iopoll.h>
#include <linux/jump_label.h>
#include <linux/module.h>
#include <linux/msi.h>
#include <linux/of.h>
@@ -42,6 +43,14 @@ MODULE_PARM_DESC(disable_msipolling,
static const struct iommu_ops arm_smmu_ops;
static struct iommu_dirty_ops arm_smmu_dirty_ops;
/*
* Repeat every {CFGI,TLBI};CMD_SYNC command sequence so that the second
* issue executes only after the first issue's CMD_SYNC has completed.
* Does not apply to ATC_INV. The key is global and is enabled from DT
* probe on affected hardware (currently Tegra264 only).
*/
static DEFINE_STATIC_KEY_FALSE(arm_smmu_erratum_repeat_tlbi_cfgi_key);
enum arm_smmu_msi_index {
EVTQ_MSI_INDEX,
GERROR_MSI_INDEX,
@@ -698,10 +707,10 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq,
* insert their own list of commands then all of the commands from one
* CPU will appear before any of the commands from the other CPU.
*/
int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync)
int __arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync)
{
struct arm_smmu_cmd cmd_sync;
u32 prod;
@@ -820,6 +829,43 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
return ret;
}
bool arm_smmu_erratum_repeat_tlbi_cfgi(void)
{
return static_branch_unlikely(&arm_smmu_erratum_repeat_tlbi_cfgi_key);
}
static bool arm_smmu_erratum_cmd_needs_repeating(struct arm_smmu_cmd *cmd)
{
u8 opcode;
if (!arm_smmu_erratum_repeat_tlbi_cfgi())
return false;
opcode = FIELD_GET(CMDQ_0_OP, cmd->data[0]);
return opcode >= CMDQ_OP_CFGI_STE && opcode < CMDQ_OP_ATC_INV;
}
int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync)
{
int ret = __arm_smmu_cmdq_issue_cmdlist(smmu, cmdq, cmds, n, sync);
/*
* A bare CMD_SYNC can be issued with n == 0 (e.g. an empty
* batch_submit()), in which case there is no cmds[0] to inspect
* and nothing to repeat.
*/
if (!n || ret || !sync)
return ret;
if (arm_smmu_erratum_cmd_needs_repeating(&cmds[0]))
ret = __arm_smmu_cmdq_issue_cmdlist(smmu, cmdq, cmds, n, sync);
return ret;
}
static int arm_smmu_cmdq_issue_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd, bool sync)
{
@@ -847,16 +893,35 @@ static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu,
cmds->cmdq = arm_smmu_get_cmdq(smmu, cmd);
}
static bool arm_smmu_cmdq_batch_force_sync(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd)
{
/* The batch's pre-assigned cmdq doesn't support the new command */
if (!arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd))
return true;
/* Arm erratum 2812531 */
if (cmds->num == CMDQ_BATCH_ENTRIES - 1 &&
(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC))
return true;
/*
* See the description at arm_smmu_erratum_repeat_tlbi_cfgi_key. Batches
* never mix CFGI/TLBI with others, so checking cmds[0] alone is enough.
*/
if (cmds->num == CMDQ_BATCH_ENTRIES &&
arm_smmu_erratum_cmd_needs_repeating(&cmds->cmds[0]))
return true;
return false;
}
static void arm_smmu_cmdq_batch_add_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd)
{
bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
bool unsupported_cmd;
unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
if (force_sync || unsupported_cmd) {
if (arm_smmu_cmdq_batch_force_sync(smmu, cmds, cmd)) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, true);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd);
@@ -1240,9 +1305,9 @@ VISIBLE_IF_KUNIT
void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
__le64 *safe_bits)
{
const __le64 eats_s1chk =
const u64 eats_s1chk =
FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
const __le64 eats_trans =
const u64 eats_trans =
FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
/*
@@ -2446,9 +2511,10 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
/* Determine how many chunks of 2^scale size we have */
num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
/* Keep the pre-DS 5-bit truncation when scale > 31 */
cmd->data[0] = orig_data0 |
FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) |
FIELD_PREP(CMDQ_TLBI_0_SCALE, scale);
FIELD_PREP(CMDQ_TLBI_0_SCALE, scale & 0x1f);
/* range is num * 2^scale * pgsize */
inv_range = num << (scale + tg);
@@ -2956,8 +3022,13 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master)
* ATC invalidation of PASID 0 causes the entire ATC to be flushed.
*/
arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
if (pci_enable_ats(pdev, stu))
dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
/*
* Since pci_prepare_ats() has already verified the HW capability
* and programmed the STE, pci_enable_ats() should not fail here.
*/
WARN(pci_enable_ats(pdev, stu),
"%s: Failed to enable ATS (STU %zu)\n", dev_name(master->dev), stu);
}
static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
@@ -3252,7 +3323,7 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
* 5. old domain updates its invs array, unreferencing master->build_invs
*
* For 1 and 5, prepare the two updated arrays in advance, handling any changes
* that can possibly failure. So the actual update of either 1 or 5 won't fail.
* that can possibly fail. So the actual update of either 1 or 5 won't fail.
* arm_smmu_asid_lock ensures that the old invs in the domains are intact while
* we are sequencing to update them.
*/
@@ -4047,9 +4118,9 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
}
/* Put the ids into order for sorted to_merge/to_unref arrays */
sort_nonatomic(master->streams, master->num_streams,
sizeof(master->streams[0]), arm_smmu_stream_id_cmp,
NULL);
sort(master->streams, master->num_streams,
sizeof(master->streams[0]), arm_smmu_stream_id_cmp,
NULL);
mutex_lock(&smmu->streams_mutex);
for (i = 0; i < fwspec->num_ids; i++) {
@@ -4398,6 +4469,20 @@ int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
return 0;
}
static void arm_smmu_free_iopf_action(void *data)
{
struct iopf_queue *queue = data;
iopf_queue_free(queue);
}
static void arm_smmu_destroy_vmid_map(void *data)
{
struct ida *ida = data;
ida_destroy(ida);
}
static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
{
int ret;
@@ -4425,6 +4510,11 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
if (!smmu->evtq.iopf)
return -ENOMEM;
ret = devm_add_action_or_reset(smmu->dev,
arm_smmu_free_iopf_action,
smmu->evtq.iopf);
if (ret)
return ret;
}
/* priq */
@@ -4503,7 +4593,8 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
ida_init(&smmu->vmid_map);
return 0;
return devm_add_action_or_reset(smmu->dev, arm_smmu_destroy_vmid_map,
&smmu->vmid_map);
}
static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
@@ -4533,8 +4624,9 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
u32 reg;
writel_relaxed(val, smmu->base + reg_off);
return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
1, ARM_SMMU_POLL_TIMEOUT_US);
return readl_relaxed_poll_timeout_atomic(smmu->base + ack_off, reg,
reg == val, 1,
ARM_SMMU_POLL_TIMEOUT_US);
}
/* GBPA is "special" */
@@ -4716,6 +4808,15 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
return ret;
}
static void arm_smmu_disable_action(void *data)
{
struct arm_smmu_device *smmu = data;
if (smmu->impl_ops && smmu->impl_ops->device_disable)
smmu->impl_ops->device_disable(smmu);
arm_smmu_device_disable(smmu);
}
static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
{
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
@@ -4921,10 +5022,14 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
{
u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD |
ARM_SMMU_FEAT_HAFT);
u32 hw_features = 0;
switch (FIELD_GET(IDR0_HTTU, reg)) {
case IDR0_HTTU_ACCESS_DIRTY_HAFT:
hw_features |= ARM_SMMU_FEAT_HAFT;
fallthrough;
case IDR0_HTTU_ACCESS_DIRTY:
hw_features |= ARM_SMMU_FEAT_HD;
fallthrough;
@@ -5098,6 +5203,9 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
/* Maximum number of outstanding stalls */
smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
if (reg & IDR5_DS)
smmu->features |= ARM_SMMU_FEAT_DS;
/* Page sizes */
if (reg & IDR5_GRAN64K)
smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
@@ -5256,6 +5364,9 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
smmu->features |= ARM_SMMU_FEAT_COHERENCY;
switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
case IDR0_HTTU_ACCESS_DIRTY_HAFT:
smmu->features |= ARM_SMMU_FEAT_HAFT;
fallthrough;
case IDR0_HTTU_ACCESS_DIRTY:
smmu->features |= ARM_SMMU_FEAT_HD;
fallthrough;
@@ -5292,8 +5403,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev,
if (of_dma_is_coherent(dev->of_node))
smmu->features |= ARM_SMMU_FEAT_COHERENCY;
if (of_device_is_compatible(dev->of_node, "nvidia,tegra264-smmu"))
if (of_device_is_compatible(dev->of_node, "nvidia,tegra264-smmu")) {
tegra_cmdqv_dt_probe(dev->of_node, smmu);
static_branch_enable(&arm_smmu_erratum_repeat_tlbi_cfgi_key);
}
return ret;
}
@@ -5472,7 +5585,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
/* Initialise in-memory data structures */
ret = arm_smmu_init_structures(smmu);
if (ret)
goto err_free_iopf;
return ret;
/* Record our private device structure */
platform_set_drvdata(pdev, smmu);
@@ -5482,30 +5595,30 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
/* Reset the device */
ret = arm_smmu_device_reset(smmu);
if (ret) {
arm_smmu_device_disable(smmu);
return ret;
}
/* Register last so it unwinds first, while the CMDQ is still up. */
ret = devm_add_action_or_reset(smmu->dev, arm_smmu_disable_action, smmu);
if (ret)
goto err_disable;
return ret;
/* And we're up. Go go go! */
ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
"smmu3.%pa", &ioaddr);
if (ret)
goto err_disable;
return ret;
ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
if (ret) {
dev_err(dev, "Failed to register iommu\n");
goto err_free_sysfs;
iommu_device_sysfs_remove(&smmu->iommu);
return ret;
}
return 0;
err_free_sysfs:
iommu_device_sysfs_remove(&smmu->iommu);
err_disable:
arm_smmu_device_disable(smmu);
err_free_iopf:
iopf_queue_free(smmu->evtq.iopf);
return ret;
}
static void arm_smmu_device_remove(struct platform_device *pdev)
@@ -5514,9 +5627,6 @@ static void arm_smmu_device_remove(struct platform_device *pdev)
iommu_device_unregister(&smmu->iommu);
iommu_device_sysfs_remove(&smmu->iommu);
arm_smmu_device_disable(smmu);
iopf_queue_free(smmu->evtq.iopf);
ida_destroy(&smmu->vmid_map);
}
static void arm_smmu_device_shutdown(struct platform_device *pdev)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+12 -1
View File
@@ -40,6 +40,7 @@ struct arm_vsmmu;
#define IDR0_HTTU GENMASK(7, 6)
#define IDR0_HTTU_ACCESS 1
#define IDR0_HTTU_ACCESS_DIRTY 2
#define IDR0_HTTU_ACCESS_DIRTY_HAFT 3
#define IDR0_COHACC (1 << 4)
#define IDR0_TTF GENMASK(3, 2)
#define IDR0_TTF_AARCH64 2
@@ -64,6 +65,7 @@ struct arm_vsmmu;
#define ARM_SMMU_IDR5 0x14
#define IDR5_STALL_MAX GENMASK(31, 16)
#define IDR5_DS (1 << 7)
#define IDR5_GRAN64K (1 << 6)
#define IDR5_GRAN16K (1 << 5)
#define IDR5_GRAN4K (1 << 4)
@@ -369,6 +371,7 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
#define CTXDESC_CD_0_ASET (1UL << 47)
#define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
#define CTXDESC_CD_1_HAFT (1UL << 3)
#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
/*
@@ -415,7 +418,7 @@ struct arm_smmu_cmd {
#define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
#define CMDQ_TLBI_RANGE_NUM_MAX 31
#define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
#define CMDQ_TLBI_0_SCALE GENMASK_ULL(25, 20)
#define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
#define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
#define CMDQ_TLBI_1_LEAF (1UL << 0)
@@ -871,6 +874,7 @@ struct arm_smmu_strtab_cfg {
struct arm_smmu_impl_ops {
int (*device_reset)(struct arm_smmu_device *smmu);
void (*device_disable)(struct arm_smmu_device *smmu);
void (*device_remove)(struct arm_smmu_device *smmu);
int (*init_structures)(struct arm_smmu_device *smmu);
struct arm_smmu_cmdq *(*get_secondary_cmdq)(
@@ -921,6 +925,8 @@ struct arm_smmu_device {
#define ARM_SMMU_FEAT_HD (1 << 22)
#define ARM_SMMU_FEAT_S2FWB (1 << 23)
#define ARM_SMMU_FEAT_BBML2 (1 << 24)
#define ARM_SMMU_FEAT_HAFT (1 << 25)
#define ARM_SMMU_FEAT_DS (1 << 26)
u32 features;
#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
@@ -1207,10 +1213,15 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state);
void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
const struct arm_smmu_ste *target);
int __arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync);
int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync);
bool arm_smmu_erratum_repeat_tlbi_cfgi(void);
#ifdef CONFIG_ARM_SMMU_V3_SVA
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+155 -69
View File
@@ -56,6 +56,8 @@
#define VINTF_ENABLED BIT(0)
#define TEGRA241_VINTF_SID_MATCH(s) (0x0040 + 0x4*(s))
#define VINTF_SID_MATCH_VIRT_SID GENMASK(20, 1)
#define VINTF_SID_MATCH_ENABLE BIT(0)
#define TEGRA241_VINTF_SID_REPLACE(s) (0x0080 + 0x4*(s))
#define TEGRA241_VINTF_LVCMDQ_ERR_MAP_64(m) \
@@ -311,55 +313,86 @@ static void tegra241_vintf_user_handle_error(struct tegra241_vintf *vintf)
static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf)
{
struct tegra241_cmdqv *cmdqv = vintf->cmdqv;
int i;
for (i = 0; i < LVCMDQ_ERR_MAP_NUM_64; i++) {
u64 map = readq_relaxed(REG_VINTF(vintf, LVCMDQ_ERR_MAP_64(i)));
while (map) {
unsigned long lidx = __ffs64(map);
struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
unsigned long map_bit = __ffs64(map);
unsigned long lidx = 64 * i + map_bit;
struct tegra241_vcmdq *vcmdq;
u32 gerror;
__arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq);
map &= ~BIT_ULL(map_bit);
/* A bit beyond the count means a HW error; skip it */
if (WARN_ON_ONCE(lidx >= cmdqv->num_lvcmdqs_per_vintf))
continue;
/* Pairs with smp_store_release() publishing it */
vcmdq = smp_load_acquire(&vintf->lvcmdqs[lidx]);
if (!vcmdq)
continue;
gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
__arm_smmu_cmdq_skip_err(&cmdqv->smmu, &vcmdq->cmdq);
writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN));
map &= ~BIT_ULL(lidx);
}
}
}
/*
* The CMDQV error interrupt is edge-triggered, so a pending VINTF error fires
* this ISR once and does not re-assert. An unacked guest therefore cannot
* storm the host. The HW latches and forwards each new error event on its
* own, so an already-set ERR_MAP bit does not suppress the interrupt for a
* new error.
*/
static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
{
struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid;
void __iomem *reg_vintf_map = REG_CMDQV(cmdqv, VINTF_ERR_MAP);
char err_str[256];
u64 vintf_map;
/* Use readl_relaxed() as register addresses are not 64-bit aligned */
vintf_map = (u64)readl_relaxed(reg_vintf_map + 0x4) << 32 |
(u64)readl_relaxed(reg_vintf_map);
snprintf(err_str, sizeof(err_str),
"vintf_map: %016llx, vcmdq_map %08x:%08x:%08x:%08x", vintf_map,
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(3))),
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(2))),
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(1))),
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(0))));
dev_warn(cmdqv->dev, "unexpected error reported. %s\n", err_str);
dev_warn_ratelimited(
cmdqv->dev,
"unexpected error reported. vintf_map: %016llx, vcmdq_map %08x:%08x:%08x:%08x\n",
vintf_map, readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(3))),
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(2))),
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(1))),
readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(0))));
/* Handle VINTF0 and its LVCMDQs */
if (vintf_map & BIT_ULL(0)) {
tegra241_vintf0_handle_error(cmdqv->vintfs[0]);
struct tegra241_vintf *vintf0;
vintf_map &= ~BIT_ULL(0);
/* NULL until tegra241_cmdqv_init_structures() publishes it */
vintf0 = smp_load_acquire(&cmdqv->vintfs[0]);
if (vintf0)
tegra241_vintf0_handle_error(vintf0);
}
/* Handle other user VINTFs and their LVCMDQs */
while (vintf_map) {
unsigned long idx = __ffs64(vintf_map);
struct tegra241_vintf *vintf;
tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]);
vintf_map &= ~BIT_ULL(idx);
/* A bit beyond the count means a HW error; skip it */
if (WARN_ON_ONCE(idx >= cmdqv->num_vintfs))
continue;
/* The slot may be published or torn down (NULL'd) concurrently */
vintf = smp_load_acquire(&cmdqv->vintfs[idx]);
if (vintf)
tegra241_vintf_user_handle_error(vintf);
}
return IRQ_HANDLED;
@@ -666,7 +699,6 @@ static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx,
vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx);
vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx);
vintf->lvcmdqs[lidx] = vcmdq;
return 0;
}
@@ -682,7 +714,7 @@ static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
dev_dbg(vintf->cmdqv->dev,
"%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64));
/* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */
if (vcmdq->vintf->hyp_own)
if (!vcmdq->vintf->idx)
kfree(vcmdq);
}
@@ -705,14 +737,15 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
/* Build an arm_smmu_cmdq for each LVCMDQ */
ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq);
if (ret)
goto deinit_lvcmdq;
goto free_vcmdq;
/* Pairs with the smp_load_acquire() in the error ISR */
smp_store_release(&vintf->lvcmdqs[lidx], vcmdq);
dev_dbg(cmdqv->dev,
"%sallocated\n", lvcmdq_error_header(vcmdq, header, 64));
return vcmdq;
deinit_lvcmdq:
tegra241_vintf_deinit_lvcmdq(vintf, lidx);
free_vcmdq:
kfree(vcmdq);
return ERR_PTR(ret);
@@ -723,8 +756,18 @@ free_vcmdq:
static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
{
kfree(cmdqv->vintfs[idx]->lvcmdqs);
/*
* Clear the slot and drain any in-flight ISR before returning idx to
* the IDA, so a concurrent create that reuses idx cannot have its
* freshly published VINTF erased here. A plain WRITE_ONCE() suffices
* since clearing the slot publishes no data. This also covers the
* init-failure unwind, which reaches deinit_vintf() without the
* destroy callback.
*/
WRITE_ONCE(cmdqv->vintfs[idx], NULL);
if (cmdqv->irq > 0)
synchronize_irq(cmdqv->irq);
ida_free(&cmdqv->vintf_ids, idx);
cmdqv->vintfs[idx] = NULL;
}
static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
@@ -750,7 +793,8 @@ static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
return -ENOMEM;
}
cmdqv->vintfs[idx] = vintf;
/* Pairs with the smp_load_acquire() in tegra241_cmdqv_isr() */
smp_store_release(&cmdqv->vintfs[idx], vintf);
return ret;
}
@@ -761,8 +805,6 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
struct tegra241_vintf *vintf = cmdqv->vintfs[idx];
u16 lidx;
tegra241_vintf_hw_deinit(vintf);
/* Remove LVCMDQ resources */
for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++)
if (vintf->lvcmdqs[lidx])
@@ -770,7 +812,7 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx);
tegra241_cmdqv_deinit_vintf(cmdqv, idx);
if (!vintf->hyp_own) {
if (vintf->idx) {
mutex_destroy(&vintf->lvcmdq_mutex);
ida_destroy(&vintf->sids);
/* Guest-owned VINTF is free-ed with viommu by iommufd core */
@@ -779,12 +821,31 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
}
}
static void tegra241_cmdqv_hw_disable(struct arm_smmu_device *smmu)
{
struct tegra241_cmdqv *cmdqv =
container_of(smmu, struct tegra241_cmdqv, smmu);
u16 idx;
for (idx = 0; idx < cmdqv->num_vintfs; idx++)
if (cmdqv->vintfs[idx])
tegra241_vintf_hw_deinit(cmdqv->vintfs[idx]);
}
static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
{
struct tegra241_cmdqv *cmdqv =
container_of(smmu, struct tegra241_cmdqv, smmu);
u16 idx;
/*
* Free the IRQ before tearing down the VINTFs. free_irq() waits for any
* in-flight tegra241_cmdqv_isr() to finish and blocks new ones, so the
* ISR cannot dereference a VINTF that is freed by the loop below.
*/
if (cmdqv->irq > 0)
free_irq(cmdqv->irq, cmdqv);
/* Remove VINTF resources */
for (idx = 0; idx < cmdqv->num_vintfs; idx++) {
if (cmdqv->vintfs[idx]) {
@@ -797,8 +858,6 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
/* Remove cmdqv resources */
ida_destroy(&cmdqv->vintf_ids);
if (cmdqv->irq > 0)
free_irq(cmdqv->irq, cmdqv);
iounmap(cmdqv->base);
kfree(cmdqv->vintfs);
put_device(cmdqv->dev); /* smmu->impl_dev */
@@ -844,6 +903,7 @@ static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = {
/* For in-kernel use */
.get_secondary_cmdq = tegra241_cmdqv_get_cmdq,
.device_reset = tegra241_cmdqv_hw_reset,
.device_disable = tegra241_cmdqv_hw_disable,
.device_remove = tegra241_cmdqv_remove,
/* For user-space use */
.hw_info = tegra241_cmdqv_hw_info,
@@ -869,6 +929,12 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf);
if (ret) {
dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret);
/*
* tegra241_cmdqv_init_vintf() failed to publish the vintf0 to
* cmdqv->vintfs[], so the probe unwind path that goes through
* cmdqv->vintfs[] would miss it. Free it here.
*/
kfree(vintf);
return ret;
}
@@ -890,16 +956,22 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
static struct dentry *cmdqv_debugfs_dir;
#endif
static struct arm_smmu_device *
__tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
int irq)
/*
* Probe the CMDQV and reallocate @smmu into the larger cmdqv->smmu.
*
* devm_krealloc() may relocate and free the original @smmu, so update *smmu to
* the new pointer once it succeeds. The error paths after it do the same, so a
* caller falling back keeps a live @smmu instead of the freed original.
*/
static int __tegra241_cmdqv_probe(struct arm_smmu_device **smmu,
struct resource *res, int irq)
{
static const struct arm_smmu_impl_ops init_ops = {
.init_structures = tegra241_cmdqv_init_structures,
.device_remove = tegra241_cmdqv_remove,
};
struct tegra241_cmdqv *cmdqv = NULL;
struct arm_smmu_device *new_smmu;
struct device *dev = (*smmu)->dev;
struct tegra241_cmdqv *cmdqv;
void __iomem *base;
u32 regval;
int ret;
@@ -908,38 +980,30 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
base = ioremap(res->start, resource_size(res));
if (!base) {
dev_err(smmu->dev, "failed to ioremap\n");
return NULL;
dev_err(dev, "failed to ioremap\n");
return -ENOMEM;
}
regval = readl(base + TEGRA241_CMDQV_CONFIG);
if (disable_cmdqv) {
dev_info(smmu->dev, "Detected disable_cmdqv=true\n");
dev_info(dev, "Detected disable_cmdqv=true\n");
writel(regval & ~CMDQV_EN, base + TEGRA241_CMDQV_CONFIG);
ret = -ENODEV;
goto iounmap;
}
cmdqv = devm_krealloc(smmu->dev, smmu, sizeof(*cmdqv), GFP_KERNEL);
if (!cmdqv)
cmdqv = devm_krealloc(dev, *smmu, sizeof(*cmdqv), GFP_KERNEL);
if (!cmdqv) {
ret = -ENOMEM;
goto iounmap;
new_smmu = &cmdqv->smmu;
}
*smmu = &cmdqv->smmu;
cmdqv->irq = irq;
cmdqv->base = base;
cmdqv->dev = smmu->impl_dev;
cmdqv->dev = (*smmu)->impl_dev;
cmdqv->base_phys = res->start;
if (cmdqv->irq > 0) {
ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
IRQF_ONESHOT, "tegra241-cmdqv",
cmdqv);
if (ret) {
dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
cmdqv->irq, ret);
goto iounmap;
}
}
regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM));
cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval);
cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval);
@@ -949,11 +1013,28 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
cmdqv->vintfs =
kzalloc_objs(*cmdqv->vintfs, cmdqv->num_vintfs);
if (!cmdqv->vintfs)
goto free_irq;
if (!cmdqv->vintfs) {
ret = -ENOMEM;
goto iounmap;
}
ida_init(&cmdqv->vintf_ids);
/*
* Request the IRQ only after cmdqv->vintfs is allocated and zeroed, so
* the ISR would not walk an uninitialized array.
*/
if (cmdqv->irq > 0) {
ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
IRQF_ONESHOT, "tegra241-cmdqv",
cmdqv);
if (ret) {
dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
cmdqv->irq, ret);
goto free_vintfs;
}
}
#ifdef CONFIG_IOMMU_DEBUGFS
if (!cmdqv_debugfs_dir) {
cmdqv_debugfs_dir =
@@ -964,24 +1045,23 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
#endif
/* Provide init-level ops only, until tegra241_cmdqv_init_structures */
new_smmu->impl_ops = &init_ops;
cmdqv->smmu.impl_ops = &init_ops;
return new_smmu;
return 0;
free_irq:
if (cmdqv->irq > 0)
free_irq(cmdqv->irq, cmdqv);
free_vintfs:
ida_destroy(&cmdqv->vintf_ids);
kfree(cmdqv->vintfs);
iounmap:
iounmap(base);
return NULL;
return ret;
}
struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
{
struct platform_device *pdev = to_platform_device(smmu->impl_dev);
struct arm_smmu_device *new_smmu;
struct resource *res;
int irq;
int irq, ret;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res) {
@@ -994,15 +1074,15 @@ struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
dev_warn(&pdev->dev,
"no interrupt. errors will not be reported\n");
new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);
if (new_smmu)
return new_smmu;
ret = __tegra241_cmdqv_probe(&smmu, res, irq);
if (!ret)
return smmu;
out_fallback:
dev_info(smmu->impl_dev, "Falling back to standard SMMU CMDQ\n");
smmu->options &= ~ARM_SMMU_OPT_TEGRA241_CMDQV;
put_device(smmu->impl_dev);
return ERR_PTR(-ENODEV);
return smmu;
}
/* User space VINTF and VCMDQ Functions */
@@ -1130,13 +1210,15 @@ static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue,
if (ret)
goto unmap_lvcmdq;
/* No lockless reader of a user VINTF's lvcmdqs[]; mutex-serialized */
vintf->lvcmdqs[lidx] = vcmdq;
hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user;
mutex_unlock(&vintf->lvcmdq_mutex);
return 0;
unmap_lvcmdq:
tegra241_vcmdq_unmap_lvcmdq(vcmdq);
tegra241_vintf_deinit_lvcmdq(vintf, lidx);
undepend_vcmdq:
if (vcmdq->prev)
iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core);
@@ -1152,6 +1234,7 @@ static void tegra241_cmdqv_destroy_vintf_user(struct iommufd_viommu *viommu)
if (vintf->mmap_offset)
iommufd_viommu_destroy_mmap(&vintf->vsmmu.core,
vintf->mmap_offset);
tegra241_vintf_hw_deinit(vintf);
tegra241_cmdqv_remove_vintf(vintf->cmdqv, vintf->idx);
}
@@ -1178,10 +1261,11 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
u64 virt_sid = vdev->virt_id;
int sidx;
if (virt_sid > UINT_MAX)
if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID))
return -EINVAL;
WARN_ON_ONCE(master->num_streams != 1);
if (master->num_streams != 1)
return -EOPNOTSUPP;
/* Find an empty pair of SID_REPLACE and SID_MATCH */
sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1,
@@ -1190,7 +1274,9 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
return sidx;
writel(stream->id, REG_VINTF(vintf, SID_REPLACE(sidx)));
writel(virt_sid << 1 | 0x1, REG_VINTF(vintf, SID_MATCH(sidx)));
writel(FIELD_PREP(VINTF_SID_MATCH_VIRT_SID, virt_sid) |
VINTF_SID_MATCH_ENABLE,
REG_VINTF(vintf, SID_MATCH(sidx)));
dev_dbg(vintf->cmdqv->dev,
"VINTF%u: allocated SID_REPLACE%d for pSID=%x, vSID=%x\n",
vintf->idx, sidx, stream->id, (u32)virt_sid);
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
+1
View File
@@ -425,6 +425,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
{ .compatible = "qcom,sm6375-mdss" },
{ .compatible = "qcom,sm8150-mdss" },
{ .compatible = "qcom,sm8250-mdss" },
{ .compatible = "qcom,sm8450-mdss" },
{ .compatible = "qcom,x1e80100-mdss" },
{ }
};
drivers/iommu/arm/arm-smmu/arm-smmu.c
+2 -7
View File
@@ -865,11 +865,8 @@ static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
"arm-smmu-context-fault", smmu_domain);
if (ret < 0) {
dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
cfg->irptndx, irq);
if (ret < 0)
cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
}
mutex_unlock(&smmu_domain->init_mutex);
@@ -2219,9 +2216,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
"arm-smmu global fault", smmu);
if (err)
return dev_err_probe(dev, err,
"failed to request global IRQ %d (%u)\n",
i, irq);
return err;
}
platform_set_drvdata(pdev, smmu);
drivers/iommu/arm/arm-smmu/qcom_iommu.c
+24 -15
View File
@@ -200,7 +200,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev)
fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
if (report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {
dev_err_ratelimited(ctx->dev,
"Unhandled context fault: fsr=0x%x, "
"iova=0x%016llx, fsynr=0x%x, cb=%d\n",
@@ -306,14 +306,14 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain,
ctx->domain = domain;
}
mutex_unlock(&qcom_domain->init_mutex);
/* Publish page table ops for map/unmap */
qcom_domain->pgtbl_ops = pgtbl_ops;
return 0;
mutex_unlock(&qcom_domain->init_mutex);
return 0;
out_clear_iommu:
free_io_pgtable_ops(pgtbl_ops);
qcom_domain->iommu = NULL;
out_unlock:
mutex_unlock(&qcom_domain->init_mutex);
@@ -704,18 +704,21 @@ static int qcom_iommu_ctx_probe(struct platform_device *pdev)
/* clear IRQs before registering fault handler, just in case the
* boot-loader left us a surprise:
*/
if (!ctx->secured_ctx)
if (!ctx->secured_ctx) {
ret = pm_runtime_resume_and_get(dev->parent);
if (ret)
return ret;
iommu_writel(ctx, ARM_SMMU_CB_FSR, iommu_readl(ctx, ARM_SMMU_CB_FSR));
pm_runtime_put_sync(dev->parent);
}
ret = devm_request_irq(dev, irq,
qcom_iommu_fault,
IRQF_SHARED,
"qcom-iommu-fault",
ctx);
if (ret) {
dev_err(dev, "failed to request IRQ %u\n", irq);
if (ret)
return ret;
}
ret = get_asid(dev->of_node);
if (ret < 0) {
@@ -836,38 +839,44 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, qcom_iommu);
pm_runtime_enable(dev);
ret = devm_pm_runtime_enable(dev);
if (ret)
return ret;
/* register context bank devices, which are child nodes: */
ret = devm_of_platform_populate(dev);
if (ret) {
dev_err(dev, "Failed to populate iommu contexts\n");
goto err_pm_disable;
return ret;
}
ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
dev_name(dev));
if (ret) {
dev_err(dev, "Failed to register iommu in sysfs\n");
goto err_pm_disable;
return ret;
}
ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);
if (ret) {
dev_err(dev, "Failed to register iommu\n");
goto err_pm_disable;
goto err_sysfs_remove;
}
if (qcom_iommu->local_base) {
pm_runtime_get_sync(dev);
ret = pm_runtime_resume_and_get(dev);
if (ret)
goto err_iommu_unregister;
writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
pm_runtime_put_sync(dev);
}
return 0;
err_pm_disable:
pm_runtime_disable(dev);
err_iommu_unregister:
iommu_device_unregister(&qcom_iommu->iommu);
err_sysfs_remove:
iommu_device_sysfs_remove(&qcom_iommu->iommu);
return ret;
}
drivers/iommu/dma-iommu.c
+13
View File
@@ -2204,6 +2204,19 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
dma_addr_t iova;
int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
size_t size = cookie_msi_granule(domain);
static DEFINE_MUTEX(msi_prepare_lock);
/*
* Normally a device's default domain is only ever attached to that
* device's own group, and the group mutex held by
* iommu_group_mutex_assert()'s callers is enough on its own. A VFIO
* type1 container is the one case that breaks that assumption: it
* can merge devices from different groups onto one domain, so two
* devices' group mutexes don't serialize each other here. A static
* lock is sufficient due to the expectation that this is a corner
* case that will never be contended in practice.
*/
guard(mutex)(&msi_prepare_lock);
msi_addr &= ~(phys_addr_t)(size - 1);
list_for_each_entry(msi_page, msi_page_list, list)
drivers/iommu/generic_pt/iommu_pt.h
+1 -1
View File
@@ -224,7 +224,7 @@ phys_addr_t DOMAIN_NS(iova_to_phys)(struct iommu_domain *domain,
ret = make_range(common_from_iommu(iommu_table), &range, iova, 1);
if (ret)
return ret;
return 0;
ret = pt_walk_range(&range, __iova_to_phys, &res);
/* PHYS_ADDR_MAX would be a better error code */
drivers/iommu/intel/cache.c
+2 -3
View File
@@ -454,9 +454,8 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
* affected by a change in S2. So just flush the entire
* device cache.
*/
addr = 0;
mask = MAX_AGAW_PFN_WIDTH;
fallthrough;
cache_tag_flush_devtlb_psi(domain, tag, 0, MAX_AGAW_PFN_WIDTH);
break;
case CACHE_TAG_DEVTLB:
cache_tag_flush_devtlb_psi(domain, tag, addr, mask);
break;
drivers/iommu/intel/debugfs.c
+4 -12
View File
@@ -690,19 +690,11 @@ static ssize_t dmar_perf_latency_write(struct file *filp,
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
int counting;
char buf[64];
int ret, counting;
if (cnt > 63)
cnt = 63;
if (copy_from_user(&buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
if (kstrtoint(buf, 0, &counting))
return -EINVAL;
ret = kstrtoint_from_user(ubuf, cnt, 0, &counting);
if (ret)
return ret;
switch (counting) {
case 0:
drivers/iommu/intel/dmar.c
+87 -11
View File
@@ -47,7 +47,7 @@ struct dmar_res_callback {
/*
* Assumptions:
* 1) The hotplug framework guarentees that DMAR unit will be hot-added
* 1) The hotplug framework guarantees that DMAR unit will be hot-added
* before IO devices managed by that unit.
* 2) The hotplug framework guarantees that DMAR unit will be hot-removed
* after IO devices managed by that unit.
@@ -915,31 +915,106 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
return 0;
}
/*
* Centralized helper for deciding the force_on policy
*
* dmar off policies (for DMA Remapping) are defined from stronger
* (more negative values) to weaker (less negative values).
*
* When a force_on type is passed in, it is associated to a reference
* level for comparison. force_on is permitted when dmar is in a
* off policy less negative than the reference level (if the policy is
* on then the check is always true).
*
* For supported force_on types:
*
* - DMAR_FORCEON_TBOOT: tboot strictly requires DMA remapping for secure
* boot hence supersedes any user opts ("iommu=off" or "intel_iommu=off")
* and weaker off policies. But if firmware forces DMA remapping off (by
* setting DMAR_REMAP_OPT_OUT in the DMAR table), no force_on is allowed.
* Firmware settings must be changed to unblock tboot.
*
* - DMAR_FORCEON_PLATFORM: external-facing devices requires DMA
* remapping to prevent malicious downstream external devices from
* composing DMA attacks. force_on is permitted only if dmar policy is
* off by build configurations (CONFIG_INTEL_IOMMU_DEFAULT_ON=off).
*
* In a nutshell, "trusted boot environment" is considered stronger than
* "user choices", which in turn is stronger than "platform opt-in hint".
* But they are all meaningless when it's forced off by "firmware".
*/
bool dmar_can_force_on(enum dmar_force_on force_on)
{
int level;
switch (force_on) {
case DMAR_FORCEON_TBOOT:
level = DMAR_USER_OFF;
break;
case DMAR_FORCEON_PLATFORM:
level = DMAR_DEFAULT_OFF;
break;
default:
level = INT_MAX;
pr_warn("Unsupported force_on type (%d)\n", force_on);
break;
}
return dmar_policy >= level;
}
static bool dmar_required(void)
{
if (dmar_policy_on())
return true;
if (!intel_iommu_tboot_noforce && tboot_enabled())
return dmar_can_force_on(DMAR_FORCEON_TBOOT);
if (dmar_platform_optin())
return dmar_can_force_on(DMAR_FORCEON_PLATFORM);
return false;
}
void __init detect_intel_iommu(void)
{
int ret;
struct dmar_res_callback validate_drhd_cb = {
.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
.ignore_unhandled = true,
};
struct acpi_table_dmar *dmar;
int ret;
down_write(&dmar_global_lock);
if (no_iommu)
dmar_policy = DMAR_USER_OFF;
ret = dmar_table_detect();
if (!ret)
ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
&validate_drhd_cb);
if (!ret && !no_iommu && !iommu_detected &&
(!dmar_disabled || dmar_platform_optin())) {
if (!ret) {
dmar = (struct acpi_table_dmar *)dmar_tbl;
ret = dmar_walk_dmar_table(dmar, &validate_drhd_cb);
}
if (ret)
goto out;
if (dmar->flags & DMAR_REMAP_OPT_OUT) {
dmar_policy = DMAR_FW_OFF;
pr_info("Firmware forces DMA remapping off\n");
pr_info("Any user opt or tboot/platform force_on will be ignored\n");
}
if (!iommu_detected && dmar_required()) {
iommu_detected = 1;
/* Make sure ACS will be enabled */
pci_request_acs();
}
if (!ret) {
x86_init.iommu.iommu_init = intel_iommu_init;
x86_platform.iommu_shutdown = intel_iommu_shutdown;
}
x86_init.iommu.iommu_init = intel_iommu_init;
x86_platform.iommu_shutdown = intel_iommu_shutdown;
out:
if (dmar_tbl) {
acpi_put_table(dmar_tbl);
dmar_tbl = NULL;
@@ -1099,6 +1174,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
spin_lock_init(&iommu->lock);
ida_init(&iommu->domain_ida);
mutex_init(&iommu->did_lock);
iommu->max_domain_id = cap_ndoms(iommu->cap);
ver = readl(iommu->reg + DMAR_VER_REG);
pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
drivers/iommu/intel/iommu.c
+92 -62
View File
@@ -53,12 +53,10 @@ static int rwbf_quirk;
#define rwbf_required(iommu) (rwbf_quirk || cap_rwbf((iommu)->cap))
/*
* set to 1 to panic kernel if can't successfully enable VT-d
* (used when kernel is launched w/ TXT)
* Skip forcing iommu on and avoid tboot-related kernel panics during
* initialization when set to 1 (via intel_iommu=tboot_noforce).
*/
static int force_on = 0;
static int intel_iommu_tboot_noforce;
static int no_platform_optin;
int intel_iommu_tboot_noforce;
#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
@@ -199,7 +197,11 @@ static LIST_HEAD(dmar_satc_units);
static void intel_iommu_domain_free(struct iommu_domain *domain);
int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
int dmar_policy = DMAR_ON;
#else
int dmar_policy = DMAR_DEFAULT_OFF;
#endif
int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
int intel_iommu_enabled = 0;
@@ -240,11 +242,10 @@ static int __init intel_iommu_setup(char *str)
while (*str) {
if (!strncmp(str, "on", 2)) {
dmar_disabled = 0;
dmar_policy = DMAR_ON;
pr_info("IOMMU enabled\n");
} else if (!strncmp(str, "off", 3)) {
dmar_disabled = 1;
no_platform_optin = 1;
dmar_policy = DMAR_USER_OFF;
pr_info("IOMMU disabled\n");
} else if (!strncmp(str, "igfx_off", 8)) {
disable_igfx_iommu = 1;
@@ -876,8 +877,14 @@ static void iommu_enable_pci_ats(struct device_domain_info *info)
if (!pci_ats_page_aligned(pdev))
return;
if (!pci_enable_ats(pdev, VTD_PAGE_SHIFT))
info->ats_enabled = 1;
/*
* pci_enable_ats() should not fail here because earlier checks
* have already verified support and configuration.
*/
if (WARN_ON(pci_enable_ats(pdev, VTD_PAGE_SHIFT)))
return;
info->ats_enabled = 1;
}
static void iommu_disable_pci_ats(struct device_domain_info *info)
@@ -1046,7 +1053,7 @@ int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
}
num = ida_alloc_range(&iommu->domain_ida, IDA_START_DID,
cap_ndoms(iommu->cap) - 1, GFP_KERNEL);
iommu->max_domain_id - 1, GFP_KERNEL);
if (num < 0) {
pr_err("%s: No free domain ids\n", iommu->name);
goto err_unlock;
@@ -1108,9 +1115,10 @@ static void copied_context_tear_down(struct intel_iommu *iommu,
assert_spin_locked(&iommu->lock);
did_old = context_domain_id(context);
context_clear_entry(context);
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
if (did_old < cap_ndoms(iommu->cap)) {
if (did_old < iommu->max_domain_id) {
iommu->flush.flush_context(iommu, did_old,
PCI_DEVID(bus, devfn),
DMA_CCMD_MASK_NOBIT,
@@ -1119,6 +1127,9 @@ static void copied_context_tear_down(struct intel_iommu *iommu,
DMA_TLB_DSI_FLUSH);
}
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
clear_context_copied(iommu, bus, devfn);
}
@@ -1246,7 +1257,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
intel_context_flush_no_pasid(info, context, did);
intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn));
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
@@ -1446,7 +1457,7 @@ static int copy_context_table(struct intel_iommu *iommu,
struct context_entry **tbl,
int bus, bool ext)
{
int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did;
struct context_entry *new_ce = NULL, ce;
struct context_entry *old_ce = NULL;
struct root_entry re;
@@ -1462,10 +1473,9 @@ static int copy_context_table(struct intel_iommu *iommu,
if (idx == 0) {
/* First save what we may have and clean up */
if (new_ce) {
tbl[tbl_idx] = new_ce;
tbl[tbl_idx + tbl_slot] = new_ce;
__iommu_flush_cache(iommu, new_ce,
VTD_PAGE_SIZE);
pos = 1;
}
if (old_ce)
@@ -1487,6 +1497,9 @@ static int copy_context_table(struct intel_iommu *iommu,
}
}
/* Track if saving UCTP or LCTP entries in scalable mode */
tbl_slot = ext && devfn >= 0x80 ? 1 : 0;
ret = -ENOMEM;
old_ce = memremap(old_ce_phys, PAGE_SIZE,
MEMREMAP_WB);
@@ -1508,14 +1521,14 @@ static int copy_context_table(struct intel_iommu *iommu,
continue;
did = context_domain_id(&ce);
if (did >= 0 && did < cap_ndoms(iommu->cap))
if (did >= 0 && did < iommu->max_domain_id)
ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);
set_context_copied(iommu, bus, devfn);
new_ce[idx] = ce;
}
tbl[tbl_idx + pos] = new_ce;
tbl[tbl_idx + tbl_slot] = new_ce;
__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
@@ -1554,12 +1567,16 @@ static int copy_translation_tables(struct intel_iommu *iommu)
return -ENOMEM;
old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
if (!old_rt_phys)
return -EINVAL;
if (!old_rt_phys) {
ret = -EINVAL;
goto err_free_bitmap;
}
old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
if (!old_rt)
return -ENOMEM;
if (!old_rt) {
ret = -ENOMEM;
goto err_free_bitmap;
}
/* This is too big for the stack - allocate it from slab */
ctxt_table_entries = ext ? 512 : 256;
@@ -1603,11 +1620,14 @@ static int copy_translation_tables(struct intel_iommu *iommu)
__iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
ret = 0;
memunmap(old_rt);
return 0;
out_unmap:
memunmap(old_rt);
err_free_bitmap:
bitmap_free(iommu->copied_tables);
iommu->copied_tables = NULL;
return ret;
}
@@ -1706,7 +1726,7 @@ static int __init init_dmars(void)
* we always have to disable PMRs or DMA may fail on
* this device
*/
if (force_on)
if (dmar_policy_force_on())
iommu_disable_protect_mem_regions(iommu);
continue;
}
@@ -1798,7 +1818,7 @@ static int init_iommu_hw(void)
* we always have to disable PMRs or DMA may fail on
* this device
*/
if (force_on)
if (dmar_policy_force_on())
iommu_disable_protect_mem_regions(iommu);
continue;
}
@@ -1859,7 +1879,7 @@ static void iommu_resume(void *data)
unsigned long flag;
if (init_iommu_hw()) {
if (force_on)
if (dmar_policy_force_on())
panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
else
WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
@@ -2127,7 +2147,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
/*
* we always have to disable PMRs or DMA may fail on this device
*/
if (force_on)
if (dmar_policy_force_on())
iommu_disable_protect_mem_regions(iommu);
return 0;
}
@@ -2365,7 +2385,7 @@ void intel_iommu_shutdown(void)
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu = NULL;
if (no_iommu || dmar_disabled)
if (dmar_policy_off())
return;
/*
@@ -2428,7 +2448,7 @@ static ssize_t domains_supported_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct intel_iommu *iommu = dev_to_intel_iommu(dev);
return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));
return sysfs_emit(buf, "%ld\n", iommu->max_domain_id);
}
static DEVICE_ATTR_RO(domains_supported);
@@ -2439,7 +2459,7 @@ static ssize_t domains_used_show(struct device *dev,
unsigned int count = 0;
int id;
for (id = 0; id < cap_ndoms(iommu->cap); id++)
for (id = 0; id < iommu->max_domain_id; id++)
if (ida_exists(&iommu->domain_ida, id))
count++;
@@ -2480,25 +2500,25 @@ static bool has_external_pci(void)
return false;
}
static int __init platform_optin_force_iommu(void)
static void __init platform_optin_force_iommu(void)
{
if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
return 0;
if (!dmar_platform_optin() || !dmar_can_force_on(DMAR_FORCEON_PLATFORM))
return;
if (no_iommu || dmar_disabled)
pr_info("Intel-IOMMU force enabled due to platform opt in\n");
if (!has_external_pci())
return;
/*
* If Intel-IOMMU is disabled by default, we will apply identity
* map for all devices except those marked as being untrusted.
*/
if (dmar_disabled)
if (dmar_policy_off()) {
pr_info("Intel-IOMMU force enabled due to platform opt in\n");
iommu_set_default_passthrough(false);
}
dmar_disabled = 0;
no_iommu = 0;
return 1;
/* No concurrent access to dmar_policy at this point. */
dmar_policy = DMAR_FORCE_ON;
}
static int __init probe_acpi_namespace_devices(void)
@@ -2538,18 +2558,20 @@ static int __init probe_acpi_namespace_devices(void)
return 0;
}
static __init int tboot_force_iommu(void)
static __init void tboot_force_iommu(void)
{
if (!tboot_enabled())
return 0;
if (!tboot_enabled() || intel_iommu_tboot_noforce)
return;
if (no_iommu || dmar_disabled)
if (!dmar_can_force_on(DMAR_FORCEON_TBOOT))
panic("tboot: Failed to force IOMMU on\n");
if (dmar_policy_off())
pr_warn("Forcing Intel-IOMMU to enabled\n");
dmar_disabled = 0;
/* No concurrent access to dmar_policy at this point. */
dmar_policy = DMAR_FORCE_ON;
no_iommu = 0;
return 1;
}
int __init intel_iommu_init(void)
@@ -2562,18 +2584,19 @@ int __init intel_iommu_init(void)
* Intel IOMMU is required for a TXT/tboot launch or platform
* opt in, so enforce that.
*/
force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
platform_optin_force_iommu();
tboot_force_iommu();
if (!dmar_policy_force_on())
platform_optin_force_iommu();
down_write(&dmar_global_lock);
if (dmar_table_init()) {
if (force_on)
if (dmar_policy_force_on())
panic("tboot: Failed to initialize DMAR table\n");
goto out_free_dmar;
}
if (dmar_dev_scope_init() < 0) {
if (force_on)
if (dmar_policy_force_on())
panic("tboot: Failed to initialize DMAR device scope\n");
goto out_free_dmar;
}
@@ -2591,7 +2614,7 @@ int __init intel_iommu_init(void)
if (!no_iommu)
intel_iommu_debugfs_init();
if (no_iommu || dmar_disabled) {
if (dmar_policy_off()) {
/*
* We exit the function here to ensure IOMMU's remapping and
* mempool aren't setup, which means that the IOMMU's PMRs
@@ -2627,7 +2650,7 @@ int __init intel_iommu_init(void)
ret = init_dmars();
if (ret) {
if (force_on)
if (dmar_policy_force_on())
panic("tboot: Failed to initialize DMARs\n");
pr_err("Initialization failed\n");
goto out_free_dmar;
@@ -3135,13 +3158,13 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
if (ret)
return ret;
ret = iopf_for_domain_set(domain, dev);
ret = iopf_for_domain_replace(domain, old, dev);
if (ret)
return ret;
ret = dmar_domain_attach_device(to_dmar_domain(domain), dev);
if (ret)
iopf_for_domain_remove(domain, dev);
iopf_for_domain_replace(old, domain, dev);
return ret;
}
@@ -3292,7 +3315,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
dev_iommu_priv_set(dev, info);
if (pdev && pci_ats_supported(pdev)) {
pci_prepare_ats(pdev, VTD_PAGE_SHIFT);
ret = pci_prepare_ats(pdev, VTD_PAGE_SHIFT);
if (ret)
goto free;
ret = device_rbtree_insert(iommu, info);
if (ret)
goto free;
@@ -3316,6 +3342,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
return &iommu->iommu;
free_table:
intel_pasid_teardown_sm_context(dev);
intel_pasid_free_table(dev);
clear_rbtree:
device_rbtree_remove(info);
@@ -3844,10 +3871,13 @@ static int identity_domain_attach_dev(struct iommu_domain *domain,
return 0;
/*
* No PRI support with the global identity domain. No need to enable or
* disable PRI in this path as the iommu has been put in the blocking
* state.
* The identity domain has no iopf_handler, so no IOPF reference is
* taken for it. The reference held by the old domain must still be
* released here; putting the device in the blocking state above does
* not affect the IOPF reference count.
*/
iopf_for_domain_remove(old, dev);
if (sm_supported(iommu))
ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
else
drivers/iommu/intel/iommu.h
+57 -16
View File
@@ -700,6 +700,7 @@ struct intel_iommu {
/* mutex to protect domain_ida */
struct mutex did_lock;
struct ida domain_ida; /* domain id allocator */
unsigned long max_domain_id;
unsigned long *copied_tables; /* bitmap of copied tables */
spinlock_t lock; /* protect context, domain ids */
struct root_entry *root_entry; /* virtual address */
@@ -1240,7 +1241,7 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
unsigned long end);
void intel_context_flush_no_pasid(struct device_domain_info *info,
struct context_entry *context, u16 did);
struct context_entry *context, u16 did, u16 sid);
int intel_iommu_enable_prq(struct intel_iommu *iommu);
int intel_iommu_finish_prq(struct intel_iommu *iommu);
@@ -1340,7 +1341,6 @@ static inline bool intel_domain_is_ss_paging(struct dmar_domain *domain)
return domain->domain.ops == &intel_ss_paging_domain_ops;
}
#ifdef CONFIG_INTEL_IOMMU
extern int intel_iommu_sm;
int iommu_calculate_agaw(struct intel_iommu *iommu);
int iommu_calculate_max_sagaw(struct intel_iommu *iommu);
@@ -1352,21 +1352,62 @@ static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu)
DMA_ECMD_ECCAP3_ESSENTIAL;
}
extern int dmar_disabled;
enum dmar_force_on {
DMAR_FORCEON_PLATFORM,
DMAR_FORCEON_TBOOT
};
/*
* On policies are positive, with more positive value being stronger.
* Off policies are negative, with more negative value being stronger.
*
* 'dmar' here refers to DMA remapping instead of the dmar/iommu unit.
*
* - DMAR_FORCE_ON:
* force to turn on (e.g. by tboot or platform opt-in).
*
* - DMAR_ON:
* turn on by build configuration (CONFIG_INTEL_IOMMU_DEFAULT_ON=on)
* or user opts ("intel_iommu=on").
*
* - DMAR_DEFAULT_OFF
* turn off by build configuration (CONFIG_INTEL_IOMMU_DEFAULT_ON=off).
*
* - DMAR_USER_OFF
* turn off by user opts ("intel_iommu=off" or "iommu=off").
*
* - DMAR_FW_OFF
* turn off due to firmware opt-out (DMAR_REMAP_OPT_OUT)
*
* - '0' is invalid, compared to decide the on/off policy
*
*/
#define DMAR_FORCE_ON 2
#define DMAR_ON 1
#define DMAR_DEFAULT_OFF -1
#define DMAR_USER_OFF -2
#define DMAR_FW_OFF -3
extern int dmar_policy;
static inline bool dmar_policy_on(void)
{
return dmar_policy > 0;
}
static inline bool dmar_policy_off(void)
{
return dmar_policy < 0;
}
static inline bool dmar_policy_force_on(void)
{
return dmar_policy == DMAR_FORCE_ON;
}
bool dmar_can_force_on(enum dmar_force_on force_on);
extern int intel_iommu_enabled;
#else
static inline int iommu_calculate_agaw(struct intel_iommu *iommu)
{
return 0;
}
static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
{
return 0;
}
#define dmar_disabled (1)
#define intel_iommu_enabled (0)
#define intel_iommu_sm (0)
#endif
extern int intel_iommu_tboot_noforce;
static inline const char *decode_prq_descriptor(char *str, size_t size,
u64 dw0, u64 dw1, u64 dw2, u64 dw3)
drivers/iommu/intel/pasid.c
+6 -3
View File
@@ -751,7 +751,7 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn)
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
intel_context_flush_no_pasid(info, context, did);
intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn));
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
@@ -955,9 +955,12 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info)
* This helper can only be used when IOMMU is working in the legacy mode or
* IOMMU is in scalable mode but all PASID table entries of the device are
* non-present.
*
* @sid identifies the context entry that was modified, which may be a DMA
* alias of @info->dev rather than its own requester ID.
*/
void intel_context_flush_no_pasid(struct device_domain_info *info,
struct context_entry *context, u16 did)
struct context_entry *context, u16 did, u16 sid)
{
struct intel_iommu *iommu = info->iommu;
@@ -967,7 +970,7 @@ void intel_context_flush_no_pasid(struct device_domain_info *info,
* when operating in scalable mode. Therefore the @did value doesn't
* matter in scalable mode.
*/
iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn),
iommu->flush.flush_context(iommu, did, sid,
DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL);
/*
drivers/iommu/intel/prq.c
+1 -1
View File
@@ -223,7 +223,7 @@ bad_req:
goto prq_advance;
}
if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) {
if (unlikely(req->pm_req && (req->rd_req || req->wr_req))) {
pr_err("IOMMU: %s: Page request in Privilege Mode\n",
iommu->name);
goto bad_req;
drivers/iommu/intel/svm.c
+1 -1
View File
@@ -115,7 +115,7 @@ static int intel_iommu_sva_supported(struct device *dev)
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu;
if (!info || dmar_disabled)
if (!info || dmar_policy_off())
return -EINVAL;
iommu = info->iommu;
drivers/iommu/iommu-sva.c
+1 -1
View File
@@ -145,8 +145,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
list_add(&domain->next, &iommu_mm->sva_domains);
out:
refcount_set(&handle->users, 1);
mutex_unlock(&iommu_sva_lock);
handle->dev = dev;
mutex_unlock(&iommu_sva_lock);
return handle;
out_free_domain:
drivers/iommu/iommu.c
+2 -2
View File
@@ -1345,11 +1345,11 @@ int iommu_group_add_device(struct iommu_group *group, struct device *dev)
EXPORT_SYMBOL_GPL(iommu_group_add_device);
/**
* iommu_group_remove_device - remove a device from it's current group
* iommu_group_remove_device - remove a device from its current group
* @dev: device to be removed
*
* This function is called by an iommu driver to remove the device from
* it's current group. This decrements the iommu group reference count.
* its current group. This decrements the iommu group reference count.
*/
void iommu_group_remove_device(struct device *dev)
{
drivers/iommu/iommufd/device.c
+2 -2
View File
@@ -1307,8 +1307,8 @@ void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
xa_lock(&ioas->iopt.access_list);
xa_for_each(&ioas->iopt.access_list, index, access) {
if (!iommufd_lock_obj(&access->obj) ||
iommufd_access_is_internal(access))
if (iommufd_access_is_internal(access) ||
!iommufd_lock_obj(&access->obj))
continue;
xa_unlock(&ioas->iopt.access_list);
drivers/iommu/iommufd/ioas.c
+2
View File
@@ -427,6 +427,8 @@ static int iommufd_take_all_iova_rwsem(struct iommufd_ctx *ictx,
rc = xa_err(xa_store(ioas_list, index, ioas, GFP_KERNEL));
if (rc) {
up_write(&ioas->iopt.iova_rwsem);
refcount_dec(&ioas->obj.users);
iommufd_release_all_iova_rwsem(ictx, ioas_list);
return rc;
}
drivers/iommu/iova.c
+16 -15
View File
@@ -538,7 +538,7 @@ reserve_iova(struct iova_domain *iovad,
break;
}
/* We are here either because this is the first reserver node
/* We are here either because this is the first reserved node
* or need to insert remaining non overlap addr range
*/
iova = __insert_new_range(iovad, pfn_lo, pfn_hi);
@@ -621,6 +621,9 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
unsigned long flags;
int i;
if (!mag)
return;
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
for (i = 0 ; i < mag->size; ++i) {
@@ -739,12 +742,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad)
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
spin_lock_init(&cpu_rcache->lock);
cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
if (!cpu_rcache->loaded || !cpu_rcache->prev) {
ret = -ENOMEM;
goto out_err;
}
}
}
@@ -777,19 +774,23 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
spin_lock_irqsave(&cpu_rcache->lock, flags);
if (!iova_magazine_full(cpu_rcache->loaded)) {
if (cpu_rcache->loaded && !iova_magazine_full(cpu_rcache->loaded)) {
can_insert = true;
} else if (!iova_magazine_full(cpu_rcache->prev)) {
} else if (cpu_rcache->prev && !iova_magazine_full(cpu_rcache->prev)) {
swap(cpu_rcache->prev, cpu_rcache->loaded);
can_insert = true;
} else {
struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
if (new_mag) {
spin_lock(&rcache->lock);
iova_depot_push(rcache, cpu_rcache->loaded);
spin_unlock(&rcache->lock);
schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
if (cpu_rcache->loaded && !cpu_rcache->prev) {
cpu_rcache->prev = cpu_rcache->loaded;
} else if (cpu_rcache->loaded) {
spin_lock(&rcache->lock);
iova_depot_push(rcache, cpu_rcache->loaded);
spin_unlock(&rcache->lock);
schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
}
cpu_rcache->loaded = new_mag;
can_insert = true;
@@ -831,9 +832,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
spin_lock_irqsave(&cpu_rcache->lock, flags);
if (!iova_magazine_empty(cpu_rcache->loaded)) {
if (cpu_rcache->loaded && !iova_magazine_empty(cpu_rcache->loaded)) {
has_pfn = true;
} else if (!iova_magazine_empty(cpu_rcache->prev)) {
} else if (cpu_rcache->prev && !iova_magazine_empty(cpu_rcache->prev)) {
swap(cpu_rcache->prev, cpu_rcache->loaded);
has_pfn = true;
} else {
drivers/iommu/msm_iommu.c
+12 -3
View File
@@ -622,6 +622,9 @@ static int insert_iommu_master(struct device *dev,
return 0;
}
if (master->num_mids >= MAX_NUM_MIDS)
return -ENOSPC;
master->mids[master->num_mids++] = spec->args[0];
return 0;
}
@@ -720,7 +723,7 @@ static int msm_iommu_probe(struct platform_device *pdev)
iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
if (!iommu)
return -ENODEV;
return -ENOMEM;
iommu->dev = &pdev->dev;
INIT_LIST_HEAD(&iommu->ctx_list);
@@ -784,19 +787,25 @@ static int msm_iommu_probe(struct platform_device *pdev)
"msm-smmu.%pa", &ioaddr);
if (ret) {
pr_err("Could not add msm-smmu at %pa to sysfs\n", &ioaddr);
return ret;
goto err_remove_list;
}
ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev);
if (ret) {
pr_err("Could not register msm-smmu at %pa\n", &ioaddr);
return ret;
goto err_remove_sysfs;
}
pr_info("device mapped at %p, irq %d with %d ctx banks\n",
iommu->base, iommu->irq, iommu->ncb);
return ret;
err_remove_sysfs:
iommu_device_sysfs_remove(&iommu->iommu);
err_remove_list:
list_del(&iommu->dev_node);
return ret;
}
static const struct of_device_id msm_iommu_dt_match[] = {
drivers/iommu/mtk_iommu.c
+1 -1
View File
@@ -220,7 +220,7 @@ struct mtk_iommu_plat_data {
unsigned int iova_region_nr;
const struct mtk_iommu_iova_region *iova_region;
/*
* Indicate the correspondance between larbs, ports and regions.
* Indicate the correspondence between larbs, ports and regions.
*
* The index is the same as iova_region and larb port numbers are
* described as bit positions.
drivers/iommu/mtk_iommu_v1.c
+1 -1
View File
@@ -88,7 +88,7 @@ struct dma_iommu_mapping {
/* MTK generation one iommu HW only support 4K size mapping */
#define MT2701_IOMMU_PAGE_SHIFT 12
#define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT)
#define MT2701_LARB_NR_MAX 3
#define MT2701_LARB_NR_MAX 4
/*
* MTK m4u support 4GB iova address space, and only support 4K page
drivers/iommu/riscv/iommu-pci.c
+3 -3
View File
@@ -109,9 +109,9 @@ static void riscv_iommu_pci_shutdown(struct pci_dev *pdev)
}
static const struct pci_device_id riscv_iommu_pci_tbl[] = {
{PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_RISCV_IOMMU), 0},
{PCI_VDEVICE(RIVOS, PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA), 0},
{0,}
{ PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_RISCV_IOMMU) },
{ PCI_VDEVICE(RIVOS, PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA) },
{ }
};
static struct pci_driver riscv_iommu_pci_driver = {
drivers/iommu/rockchip-iommu.c
+59 -58
View File
@@ -84,6 +84,14 @@
*/
#define RK_IOMMU_PGSIZE_BITMAP 0x007ff000
struct rk_iommu_ops {
phys_addr_t (*pt_address)(u32 dte);
u32 (*mk_dtentries)(dma_addr_t pt_dma);
u32 (*mk_ptentries)(phys_addr_t page, int prot);
u64 dma_bit_mask;
gfp_t gfp_flags;
};
struct rk_iommu_domain {
struct list_head iommus;
u32 *dt; /* page directory table */
@@ -91,23 +99,11 @@ struct rk_iommu_domain {
spinlock_t iommus_lock; /* lock for iommus list */
spinlock_t dt_lock; /* lock for modifying page directory table */
struct device *dma_dev;
const struct rk_iommu_ops *rk_ops;
struct iommu_domain domain;
};
/* list of clocks required by IOMMU */
static const char * const rk_iommu_clocks[] = {
"aclk", "iface",
};
struct rk_iommu_ops {
phys_addr_t (*pt_address)(u32 dte);
u32 (*mk_dtentries)(dma_addr_t pt_dma);
u32 (*mk_ptentries)(phys_addr_t page, int prot);
u64 dma_bit_mask;
gfp_t gfp_flags;
};
struct rk_iommu {
struct device *dev;
void __iomem **bases;
@@ -119,6 +115,7 @@ struct rk_iommu {
struct iommu_device iommu;
struct list_head node; /* entry in rk_iommu_domain.iommus */
struct iommu_domain *domain; /* domain to which iommu is attached */
const struct rk_iommu_ops *rk_ops;
};
struct rk_iommudata {
@@ -126,7 +123,6 @@ struct rk_iommudata {
struct rk_iommu *iommu;
};
static const struct rk_iommu_ops *rk_ops;
static struct iommu_domain rk_identity_domain;
static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma,
@@ -422,6 +418,19 @@ static int rk_iommu_enable_stall(struct rk_iommu *iommu)
if (!rk_iommu_is_paging_enabled(iommu))
return 0;
/*
* Boot firmware can leave a bank in PAGE_FAULT_ACTIVE with no handler
* (PAGE_FAULT_ACTIVE & !STALL_ACTIVE & IDLE). Such a bank ignores
* CMD_ENABLE_STALL and never reaches STALL_ACTIVE, timing out the poll
* below. Acknowledge any stale fault first so every bank starts clean.
*/
for (i = 0; i < iommu->num_mmu; i++) {
if (rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &
RK_MMU_STATUS_PAGE_FAULT_ACTIVE)
writel(RK_MMU_CMD_PAGE_FAULT_DONE,
iommu->bases[i] + RK_MMU_COMMAND);
}
rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL);
ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,
@@ -512,7 +521,7 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu)
* and verifying that upper 5 (v1) or 7 (v2) nybbles are read back.
*/
for (i = 0; i < iommu->num_mmu; i++) {
dte_addr = rk_ops->pt_address(DTE_ADDR_DUMMY);
dte_addr = iommu->rk_ops->pt_address(DTE_ADDR_DUMMY);
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr);
if (dte_addr != rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR)) {
@@ -553,7 +562,7 @@ static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
page_offset = rk_iova_page_offset(iova);
mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR);
mmu_dte_addr_phys = rk_ops->pt_address(mmu_dte_addr);
mmu_dte_addr_phys = iommu->rk_ops->pt_address(mmu_dte_addr);
dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index);
dte_addr = phys_to_virt(dte_addr_phys);
@@ -562,14 +571,14 @@ static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
if (!rk_dte_is_pt_valid(dte))
goto print_it;
pte_addr_phys = rk_ops->pt_address(dte) + (pte_index * 4);
pte_addr_phys = iommu->rk_ops->pt_address(dte) + (pte_index * 4);
pte_addr = phys_to_virt(pte_addr_phys);
pte = *pte_addr;
if (!rk_pte_is_page_valid(pte))
goto print_it;
page_addr_phys = rk_ops->pt_address(pte) + page_offset;
page_addr_phys = iommu->rk_ops->pt_address(pte) + page_offset;
page_flags = pte & RK_PTE_PAGE_FLAGS_MASK;
print_it:
@@ -665,13 +674,13 @@ static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain,
if (!rk_dte_is_pt_valid(dte))
goto out;
pt_phys = rk_ops->pt_address(dte);
pt_phys = rk_domain->rk_ops->pt_address(dte);
page_table = (u32 *)phys_to_virt(pt_phys);
pte = page_table[rk_iova_pte_index(iova)];
if (!rk_pte_is_page_valid(pte))
goto out;
phys = rk_ops->pt_address(pte) + rk_iova_page_offset(iova);
phys = rk_domain->rk_ops->pt_address(pte) + rk_iova_page_offset(iova);
out:
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
@@ -732,7 +741,7 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
if (rk_dte_is_pt_valid(dte))
goto done;
page_table = iommu_alloc_pages_sz(GFP_ATOMIC | rk_ops->gfp_flags,
page_table = iommu_alloc_pages_sz(GFP_ATOMIC | rk_domain->rk_ops->gfp_flags,
SPAGE_SIZE);
if (!page_table)
return ERR_PTR(-ENOMEM);
@@ -744,13 +753,13 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
return ERR_PTR(-ENOMEM);
}
dte = rk_ops->mk_dtentries(pt_dma);
dte = rk_domain->rk_ops->mk_dtentries(pt_dma);
*dte_addr = dte;
rk_table_flush(rk_domain,
rk_domain->dt_dma + dte_index * sizeof(u32), 1);
done:
pt_phys = rk_ops->pt_address(dte);
pt_phys = rk_domain->rk_ops->pt_address(dte);
return (u32 *)phys_to_virt(pt_phys);
}
@@ -792,7 +801,7 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
if (rk_pte_is_page_valid(pte))
goto unwind;
pte_addr[pte_count] = rk_ops->mk_ptentries(paddr, prot);
pte_addr[pte_count] = rk_domain->rk_ops->mk_ptentries(paddr, prot);
paddr += SPAGE_SIZE;
}
@@ -814,7 +823,7 @@ unwind:
pte_count * SPAGE_SIZE);
iova += pte_count * SPAGE_SIZE;
page_phys = rk_ops->pt_address(pte_addr[pte_count]);
page_phys = rk_domain->rk_ops->pt_address(pte_addr[pte_count]);
pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n",
&iova, &page_phys, &paddr, prot);
@@ -851,7 +860,7 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
pte_index = rk_iova_pte_index(iova);
pte_addr = &page_table[pte_index];
pte_dma = rk_ops->pt_address(dte_index) + pte_index * sizeof(u32);
pte_dma = rk_domain->rk_ops->pt_address(dte_index) + pte_index * sizeof(u32);
ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova,
paddr, size, prot);
@@ -889,7 +898,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
return 0;
}
pt_phys = rk_ops->pt_address(dte);
pt_phys = rk_domain->rk_ops->pt_address(dte);
pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);
pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32);
unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size);
@@ -948,7 +957,7 @@ static int rk_iommu_enable(struct rk_iommu *iommu)
for (i = 0; i < iommu->num_mmu; i++) {
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
rk_ops->mk_dtentries(rk_domain->dt_dma));
iommu->rk_ops->mk_dtentries(rk_domain->dt_dma));
rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);
@@ -1035,6 +1044,9 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
if (iommu->domain == domain)
return 0;
if (rk_domain->rk_ops != iommu->rk_ops)
return -EINVAL;
ret = rk_iommu_identity_attach(&rk_identity_domain, dev, old);
if (ret)
return ret;
@@ -1076,17 +1088,19 @@ static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev)
if (!rk_domain)
return NULL;
iommu = rk_iommu_from_dev(dev);
rk_domain->rk_ops = iommu->rk_ops;
/*
* rk32xx iommus use a 2 level pagetable.
* Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
* Allocate one 4 KiB page for each table.
*/
rk_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | rk_ops->gfp_flags,
rk_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | rk_domain->rk_ops->gfp_flags,
SPAGE_SIZE);
if (!rk_domain->dt)
goto err_free_domain;
iommu = rk_iommu_from_dev(dev);
rk_domain->dma_dev = iommu->dev;
rk_domain->dt_dma = dma_map_single(rk_domain->dma_dev, rk_domain->dt,
SPAGE_SIZE, DMA_TO_DEVICE);
@@ -1125,7 +1139,7 @@ static void rk_iommu_domain_free(struct iommu_domain *domain)
for (i = 0; i < NUM_DT_ENTRIES; i++) {
u32 dte = rk_domain->dt[i];
if (rk_dte_is_pt_valid(dte)) {
phys_addr_t pt_phys = rk_ops->pt_address(dte);
phys_addr_t pt_phys = rk_domain->rk_ops->pt_address(dte);
u32 *page_table = phys_to_virt(pt_phys);
dma_unmap_single(rk_domain->dma_dev, pt_phys,
SPAGE_SIZE, DMA_TO_DEVICE);
@@ -1205,7 +1219,6 @@ static int rk_iommu_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct rk_iommu *iommu;
struct resource *res;
const struct rk_iommu_ops *ops;
int num_res = pdev->num_resources;
int err, i;
@@ -1219,16 +1232,9 @@ static int rk_iommu_probe(struct platform_device *pdev)
iommu->dev = dev;
iommu->num_mmu = 0;
ops = of_device_get_match_data(dev);
if (!rk_ops)
rk_ops = ops;
/*
* That should not happen unless different versions of the
* hardware block are embedded the same SoC
*/
if (WARN_ON(rk_ops != ops))
return -EINVAL;
iommu->rk_ops = of_device_get_match_data(dev);
if (!iommu->rk_ops)
return -ENOENT;
iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases),
GFP_KERNEL);
@@ -1245,7 +1251,7 @@ static int rk_iommu_probe(struct platform_device *pdev)
iommu->num_mmu++;
}
if (iommu->num_mmu == 0)
return PTR_ERR(iommu->bases[0]);
return -ENODEV;
iommu->num_irq = platform_irq_count(pdev);
if (iommu->num_irq < 0)
@@ -1254,25 +1260,20 @@ static int rk_iommu_probe(struct platform_device *pdev)
iommu->reset_disabled = device_property_read_bool(dev,
"rockchip,disable-mmu-reset");
iommu->num_clocks = ARRAY_SIZE(rk_iommu_clocks);
iommu->clocks = devm_kcalloc(iommu->dev, iommu->num_clocks,
sizeof(*iommu->clocks), GFP_KERNEL);
if (!iommu->clocks)
return -ENOMEM;
for (i = 0; i < iommu->num_clocks; ++i)
iommu->clocks[i].id = rk_iommu_clocks[i];
/*
* iommu clocks should be present for all new devices and devicetrees
* but there are older devicetrees without clocks out in the wild.
* So clocks as optional for the time being.
* Take every clock the devicetree provides. Most IOMMU instances
* need exactly "aclk" + "iface", but e.g. the RK3576 NPU IOMMUs sit
* behind additional gates (CBUF/DSU) whose clocks must be running
* for register writes to land. Clocks stay optional because there
* are older devicetrees without clocks out in the wild.
*/
err = devm_clk_bulk_get(iommu->dev, iommu->num_clocks, iommu->clocks);
err = devm_clk_bulk_get_all(iommu->dev, &iommu->clocks);
if (err == -ENOENT)
iommu->num_clocks = 0;
else if (err)
else if (err < 0)
return err;
else
iommu->num_clocks = err;
err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
if (err)
@@ -1294,7 +1295,7 @@ static int rk_iommu_probe(struct platform_device *pdev)
goto err_pm_disable;
}
dma_set_mask_and_coherent(dev, rk_ops->dma_bit_mask);
dma_set_mask_and_coherent(dev, iommu->rk_ops->dma_bit_mask);
err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
if (err)
drivers/pci/ats.c
+10 -3
View File
@@ -40,10 +40,13 @@ void pci_ats_init(struct pci_dev *dev)
*/
bool pci_ats_supported(struct pci_dev *dev)
{
if (!dev->ats_cap)
if (!dev->ats_cap || dev->untrusted)
return false;
return (dev->untrusted == 0);
if (dev->is_virtfn)
return pci_ats_supported(pci_physfn(dev));
return true;
}
EXPORT_SYMBOL_GPL(pci_ats_supported);
@@ -70,8 +73,12 @@ int pci_prepare_ats(struct pci_dev *dev, int ps)
if (ps < PCI_ATS_MIN_STU)
return -EINVAL;
if (dev->is_virtfn)
if (dev->is_virtfn) {
if (pci_physfn(dev)->ats_stu != ps)
return -EINVAL;
return 0;
}
dev->ats_stu = ps;
ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
include/linux/dmar.h
+1
View File
@@ -24,6 +24,7 @@ struct acpi_dmar_header;
#define DMAR_INTR_REMAP 0x1
#define DMAR_X2APIC_OPT_OUT 0x2
#define DMAR_PLATFORM_OPT_IN 0x4
#define DMAR_REMAP_OPT_OUT 0x8
struct intel_iommu;
include/uapi/linux/iommufd.h
+14 -3
View File
@@ -574,11 +574,22 @@ struct iommu_hw_info_vtd {
__aligned_u64 ecap_reg;
};
/**
* enum iommu_hw_info_arm_smmuv3_flags - Flags for ARM SMMUv3 hw_info
* @IOMMU_HW_INFO_ARM_SMMUV3_ERRATA_REPEAT_TLBI_CFGI:
* If set, user space must issue TLBI/CFGI+SYNC commands twice due to
* hardware erratum T264-SMMU-3. See the description at
* arm_smmu_erratum_repeat_tlbi_cfgi_key.
*/
enum iommu_hw_info_arm_smmuv3_flags {
IOMMU_HW_INFO_ARM_SMMUV3_ERRATA_REPEAT_TLBI_CFGI = 1 << 0,
};
/**
* struct iommu_hw_info_arm_smmuv3 - ARM SMMUv3 hardware information
* (IOMMU_HW_INFO_TYPE_ARM_SMMUV3)
*
* @flags: Must be set to 0
* @flags: Combination of enum iommu_hw_info_arm_smmuv3_flags
* @__reserved: Must be 0
* @idr: Implemented features for ARM SMMU Non-secure programming interface
* @iidr: Information about the implementation and implementer of ARM SMMU,
@@ -594,7 +605,7 @@ struct iommu_hw_info_vtd {
* idr[0]: ST_LEVEL, TERM_MODEL, STALL_MODEL, TTENDIAN , CD2L, ASID16, TTF
* idr[1]: SIDSIZE, SSIDSIZE
* idr[3]: BBML, RIL
* idr[5]: VAX, GRAN64K, GRAN16K, GRAN4K
* idr[5]: VAX, GRAN64K, GRAN16K, GRAN4K, DS
*
* - S1P should be assumed to be true if a NESTED HWPT can be created
* - VFIO/iommufd only support platforms with COHACC, it should be assumed to be
@@ -602,7 +613,7 @@ struct iommu_hw_info_vtd {
* - ATS is a per-device property. If the VMM describes any devices as ATS
* capable in ACPI/DT it should set the corresponding idr.
*
* This list may expand in future (eg E0PD, AIE, PBHA, D128, DS etc). It is
* This list may expand in future (eg E0PD, AIE, PBHA, D128 etc). It is
* important that VMMs do not read bits outside the list to allow for
* compatibility with future kernels. Several features in the SMMUv3
* architecture are not currently supported by the kernel for nesting: HTTU,
rust/kernel/iommu/pgtable.rs
+1 -2
View File
@@ -102,8 +102,7 @@ impl<F: IoPageTableFmt> IoPageTable<F> {
coherent_walk: config.coherent_walk,
tlb: &raw const NOOP_FLUSH_OPS,
iommu_dev: dev.as_raw(),
// SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`.
..unsafe { core::mem::zeroed() }
..Zeroable::zeroed()
};
// SAFETY: